FD.io VPP  v17.01-9-ge7dcee4
Vector Packet Processing
buffer_node.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer_node.h: VLIB buffer handling node helper macros/inlines
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_node_h
41 #define included_vlib_buffer_node_h
42 
43 /** \file
44  vlib buffer/node functions
45 */
46 
47 /** \brief Finish enqueueing two buffers forward in the graph.
48  Standard dual loop boilerplate element. This is a MACRO,
49  with MULTIPLE SIDE EFFECTS. In the ideal case,
50  <code>next_index == next0 == next1</code>,
51  which means that the speculative enqueue at the top of the dual loop
52  has correctly dealt with both packets. In that case, the macro does
53  nothing at all.
54 
55  @param vm vlib_main_t pointer, varies by thread
56  @param node current node vlib_node_runtime_t pointer
57  @param next_index speculated next index used for both packets
58  @param to_next speculated vector pointer used for both packets
59  @param n_left_to_next number of slots left in speculated vector
60  @param bi0 first buffer index
61  @param bi1 second buffer index
62  @param next0 actual next index to be used for the first packet
63  @param next1 actual next index to be used for the second packet
64 
65  @return @c next_index -- speculative next index to be used for future packets
66  @return @c to_next -- speculative frame to be used for future packets
67  @return @c n_left_to_next -- number of slots left in speculative frame
68 */
69 
70 #define vlib_validate_buffer_enqueue_x2(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,next0,next1) \
71 do { \
72  int enqueue_code = (next0 != next_index) + 2*(next1 != next_index); \
73  \
74  if (PREDICT_FALSE (enqueue_code != 0)) \
75  { \
76  switch (enqueue_code) \
77  { \
78  case 1: \
79  /* A B A */ \
80  to_next[-2] = bi1; \
81  to_next -= 1; \
82  n_left_to_next += 1; \
83  vlib_set_next_frame_buffer (vm, node, next0, bi0); \
84  break; \
85  \
86  case 2: \
87  /* A A B */ \
88  to_next -= 1; \
89  n_left_to_next += 1; \
90  vlib_set_next_frame_buffer (vm, node, next1, bi1); \
91  break; \
92  \
93  case 3: \
94  /* A B B or A B C */ \
95  to_next -= 2; \
96  n_left_to_next += 2; \
97  vlib_set_next_frame_buffer (vm, node, next0, bi0); \
98  vlib_set_next_frame_buffer (vm, node, next1, bi1); \
99  if (next0 == next1) \
100  { \
101  vlib_put_next_frame (vm, node, next_index, \
102  n_left_to_next); \
103  next_index = next1; \
104  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
105  } \
106  } \
107  } \
108 } while (0)
109 
110 
111 /** \brief Finish enqueueing four buffers forward in the graph.
112  Standard quad loop boilerplate element. This is a MACRO,
113  with MULTIPLE SIDE EFFECTS. In the ideal case,
114  <code>next_index == next0 == next1 == next2 == next3</code>,
115  which means that the speculative enqueue at the top of the quad loop
116  has correctly dealt with all four packets. In that case, the macro does
117  nothing at all.
118 
119  @param vm vlib_main_t pointer, varies by thread
120  @param node current node vlib_node_runtime_t pointer
121  @param next_index speculated next index used for both packets
122  @param to_next speculated vector pointer used for both packets
123  @param n_left_to_next number of slots left in speculated vector
124  @param bi0 first buffer index
125  @param bi1 second buffer index
126  @param bi2 third buffer index
127  @param bi3 fourth buffer index
128  @param next0 actual next index to be used for the first packet
129  @param next1 actual next index to be used for the second packet
130  @param next2 actual next index to be used for the third packet
131  @param next3 actual next index to be used for the fourth packet
132 
133  @return @c next_index -- speculative next index to be used for future packets
134  @return @c to_next -- speculative frame to be used for future packets
135  @return @c n_left_to_next -- number of slots left in speculative frame
136 */
137 
138 #define vlib_validate_buffer_enqueue_x4(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,bi2,bi3,next0,next1,next2,next3) \
139 do { \
140  /* After the fact: check the [speculative] enqueue to "next" */ \
141  u32 fix_speculation = next_index != next0 || next_index != next1 \
142  || next_index != next2 || next_index != next3; \
143  if (PREDICT_FALSE(fix_speculation)) \
144  { \
145  /* rewind... */ \
146  to_next -= 4; \
147  n_left_to_next += 4; \
148  \
149  /* If bi0 belongs to "next", send it there */ \
150  if (next_index == next0) \
151  { \
152  to_next[0] = bi0; \
153  to_next++; \
154  n_left_to_next --; \
155  } \
156  else /* send it where it needs to go */ \
157  vlib_set_next_frame_buffer (vm, node, next0, bi0); \
158  \
159  if (next_index == next1) \
160  { \
161  to_next[0] = bi1; \
162  to_next++; \
163  n_left_to_next --; \
164  } \
165  else \
166  vlib_set_next_frame_buffer (vm, node, next1, bi1); \
167  \
168  if (next_index == next2) \
169  { \
170  to_next[0] = bi2; \
171  to_next++; \
172  n_left_to_next --; \
173  } \
174  else \
175  vlib_set_next_frame_buffer (vm, node, next2, bi2); \
176  \
177  if (next_index == next3) \
178  { \
179  to_next[0] = bi3; \
180  to_next++; \
181  n_left_to_next --; \
182  } \
183  else \
184  vlib_set_next_frame_buffer (vm, node, next3, bi3); \
185  \
186  /* Change speculation: last 2 packets went to the same node */ \
187  if (next2 == next3) \
188  { \
189  vlib_put_next_frame (vm, node, next_index, n_left_to_next); \
190  next_index = next3; \
191  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
192  } \
193  } \
194  } while(0);
195 
196 /** \brief Finish enqueueing one buffer forward in the graph.
197  Standard single loop boilerplate element. This is a MACRO,
198  with MULTIPLE SIDE EFFECTS. In the ideal case,
199  <code>next_index == next0</code>,
200  which means that the speculative enqueue at the top of the single loop
201  has correctly dealt with the packet in hand. In that case, the macro does
202  nothing at all.
203 
204  @param vm vlib_main_t pointer, varies by thread
205  @param node current node vlib_node_runtime_t pointer
206  @param next_index speculated next index used for both packets
207  @param to_next speculated vector pointer used for both packets
208  @param n_left_to_next number of slots left in speculated vector
209  @param bi0 first buffer index
210  @param next0 actual next index to be used for the first packet
211 
212  @return @c next_index -- speculative next index to be used for future packets
213  @return @c to_next -- speculative frame to be used for future packets
214  @return @c n_left_to_next -- number of slots left in speculative frame
215 */
216 #define vlib_validate_buffer_enqueue_x1(vm,node,next_index,to_next,n_left_to_next,bi0,next0) \
217 do { \
218  if (PREDICT_FALSE (next0 != next_index)) \
219  { \
220  vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); \
221  next_index = next0; \
222  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
223  \
224  to_next[0] = bi0; \
225  to_next += 1; \
226  n_left_to_next -= 1; \
227  } \
228 } while (0)
229 
232  vlib_node_runtime_t * node,
233  vlib_frame_t * frame,
234  uword sizeof_trace,
235  void *opaque1,
236  uword opaque2,
237  void (*two_buffers) (vlib_main_t * vm,
238  void *opaque1,
239  uword opaque2,
240  vlib_buffer_t * b0,
241  vlib_buffer_t * b1,
242  u32 * next0, u32 * next1),
243  void (*one_buffer) (vlib_main_t * vm,
244  void *opaque1, uword opaque2,
245  vlib_buffer_t * b0,
246  u32 * next0))
247 {
248  u32 n_left_from, *from, *to_next;
249  u32 next_index;
250 
251  from = vlib_frame_vector_args (frame);
252  n_left_from = frame->n_vectors;
253  next_index = node->cached_next_index;
254 
255  if (node->flags & VLIB_NODE_FLAG_TRACE)
256  vlib_trace_frame_buffers_only (vm, node, from, frame->n_vectors,
257  /* stride */ 1, sizeof_trace);
258 
259  while (n_left_from > 0)
260  {
261  u32 n_left_to_next;
262 
263  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
264 
265  while (n_left_from >= 4 && n_left_to_next >= 2)
266  {
267  vlib_buffer_t *p0, *p1;
268  u32 pi0, next0;
269  u32 pi1, next1;
270 
271  /* Prefetch next iteration. */
272  {
273  vlib_buffer_t *p2, *p3;
274 
275  p2 = vlib_get_buffer (vm, from[2]);
276  p3 = vlib_get_buffer (vm, from[3]);
277 
278  vlib_prefetch_buffer_header (p2, LOAD);
279  vlib_prefetch_buffer_header (p3, LOAD);
280 
281  CLIB_PREFETCH (p2->data, 64, LOAD);
282  CLIB_PREFETCH (p3->data, 64, LOAD);
283  }
284 
285  pi0 = to_next[0] = from[0];
286  pi1 = to_next[1] = from[1];
287  from += 2;
288  to_next += 2;
289  n_left_from -= 2;
290  n_left_to_next -= 2;
291 
292  p0 = vlib_get_buffer (vm, pi0);
293  p1 = vlib_get_buffer (vm, pi1);
294 
295  two_buffers (vm, opaque1, opaque2, p0, p1, &next0, &next1);
296 
297  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
298  to_next, n_left_to_next,
299  pi0, pi1, next0, next1);
300  }
301 
302  while (n_left_from > 0 && n_left_to_next > 0)
303  {
304  vlib_buffer_t *p0;
305  u32 pi0, next0;
306 
307  pi0 = from[0];
308  to_next[0] = pi0;
309  from += 1;
310  to_next += 1;
311  n_left_from -= 1;
312  n_left_to_next -= 1;
313 
314  p0 = vlib_get_buffer (vm, pi0);
315 
316  one_buffer (vm, opaque1, opaque2, p0, &next0);
317 
318  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
319  to_next, n_left_to_next,
320  pi0, next0);
321  }
322 
323  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
324  }
325 
326  return frame->n_vectors;
327 }
328 
329 #endif /* included_vlib_buffer_node_h */
330 
331 /*
332  * fd.io coding-style-patch-verification: ON
333  *
334  * Local Variables:
335  * eval: (c-set-style "gnu")
336  * End:
337  */
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:459
#define always_inline
Definition: clib.h:84
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:216
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:350
u16 n_vectors
Definition: node.h:344
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
u16 cached_next_index
Definition: node.h:463
unsigned int u32
Definition: types.h:88
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:259
static uword generic_buffer_node_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, uword sizeof_trace, void *opaque1, uword opaque2, void(*two_buffers)(vlib_main_t *vm, void *opaque1, uword opaque2, vlib_buffer_t *b0, vlib_buffer_t *b1, u32 *next0, u32 *next1), void(*one_buffer)(vlib_main_t *vm, void *opaque1, uword opaque2, vlib_buffer_t *b0, u32 *next0))
Definition: buffer_node.h:231
u64 uword
Definition: types.h:112
void vlib_trace_frame_buffers_only(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, uword n_buffers, uword next_buffer_stride, uword n_buffer_data_bytes_in_trace)
Definition: trace.c:45
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:253
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:170
u8 data[0]
Packet data.
Definition: buffer.h:158
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57