FD.io VPP  v19.04-6-g6f05f72
Vector Packet Processing
pipeline.h
Go to the documentation of this file.
1 /*
2  * vnet/pipeline.h: software pipeline
3  *
4  * Copyright (c) 2012 Cisco and/or its affiliates.
5  * Licensed under the Apache License, Version 2.0 (the "License");
6  * you may not use this file except in compliance with the License.
7  * You may obtain a copy of the License at:
8  *
9  * http://www.apache.org/licenses/LICENSE-2.0
10  *
11  * Unless required by applicable law or agreed to in writing, software
12  * distributed under the License is distributed on an "AS IS" BASIS,
13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14  * See the License for the specific language governing permissions and
15  * limitations under the License.
16  */
17 
18 /*
19  * Usage example.
20  *
21  * #define NSTAGES 3 or whatever
22  *
23  * If using an aux data vector - to hold bihash keys or some such:
24  *
25  * #define AUX_DATA_TYPE my_aux_data_t
26  *
27  * <Define pipeline stages>
28  *
29  * #include <vnet/pipeline.h>
30  *
31  * static uword my_node_fn (vlib_main_t * vm,
32  * vlib_node_runtime_t * node,
33  * vlib_frame_t * frame)
34  * {
35  * return dispatch_pipeline (vm, node, frame);
36  * }
37  *
38  */
39 
40 #ifndef NSTAGES
41 #error files which #include <vnet/pipeline.h> must define NSTAGES
42 #endif
43 
44 #ifndef STAGE_INLINE
45 #define STAGE_INLINE inline
46 #endif
47 
48 /* Unless the user wants the aux data scheme, don't configure it */
49 #ifndef AUX_DATA_TYPE
50 #define AUX_DATA_ARG
51 #define AUX_DATA_DECL
52 #define AUX_DATA_PTR(pi)
53 #else
54 #define AUX_DATA_ARG ,##AUX_DATA_TYPE *ap
55 #define AUX_DATA_DECL AUX_DATA_TYPE aux_data[VLIB_FRAME_SIZE]
56 #define AUX_DATA_PTR(pi) ,aux_data +(pi)
57 #endif
58 
59 /*
60  * A prefetch stride of 2 is quasi-equivalent to doubling the number
61  * of stages with every other pipeline stage empty.
62  */
63 
64 /*
65  * This is a typical first pipeline stage, which prefetches
66  * buffer metadata and the first line of pkt data.
67  *
68  * To use it:
69  * #define stage0 generic_stage0
70  *
71  * This implementation won't use the aux data argument
72  */
73 static STAGE_INLINE void
76 {
77  vlib_prefetch_buffer_header (b, STORE);
78  CLIB_PREFETCH (b->data, CLIB_CACHE_LINE_BYTES, STORE);
79 }
80 
81 #if NSTAGES == 2
82 
83 static STAGE_INLINE uword
84 dispatch_pipeline (vlib_main_t * vm,
85  vlib_node_runtime_t * node, vlib_frame_t * frame)
86 {
87  u32 *from;
88  u32 n_left_from;
89  int pi;
91  u16 nexts[VLIB_FRAME_SIZE];
93 
94  n_left_from = frame->n_vectors;
95  from = vlib_frame_vector_args (frame);
96  vlib_get_buffers (vm, from, bufs, n_left_from);
97 
98  for (pi = 0; pi < NSTAGES - 1; pi++)
99  {
100  if (pi == n_left_from)
101  break;
102  stage0 (vm, node, bufs[pi] AUX_DATA_PTR (pi));
103  }
104 
105  for (; pi < n_left_from; pi++)
106  {
107  stage0 (vm, node, bufs[pi]);
108  nexts[pi - 1] =
109  last_stage (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
110  }
111 
112  for (; pi < (n_left_from + (NSTAGES - 1)); pi++)
113  {
114  if (((pi - 1) >= 0) && ((pi - 1) < n_left_from))
115  nexts[pi - 1] =
116  last_stage (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
117  }
118 
119  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
120  return frame->n_vectors;
121 }
122 #endif
123 
124 #if NSTAGES == 3
125 static STAGE_INLINE uword
126 dispatch_pipeline (vlib_main_t * vm,
127  vlib_node_runtime_t * node, vlib_frame_t * frame)
128 {
129  u32 *from;
130  u32 n_left_from;
131  int pi;
133  u16 nexts[VLIB_FRAME_SIZE];
135 
136  n_left_from = frame->n_vectors;
137  from = vlib_frame_vector_args (frame);
138  vlib_get_buffers (vm, from, bufs, n_left_from);
139 
140  for (pi = 0; pi < NSTAGES - 1; pi++)
141  {
142  if (pi == n_left_from)
143  break;
144  stage0 (vm, node, bufs[pi] AUX_DATA_PTR (pi));
145  if (pi - 1 >= 0)
146  stage1 (vm, node, bufs[pi - 1]);
147  }
148 
149  for (; pi < n_left_from; pi++)
150  {
151  stage0 (vm, node, bufs[pi] AUX_DATA_PTR (pi));
152  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
153  nexts[pi - 2] =
154  last_stage (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
155  }
156 
157  for (; pi < (n_left_from + (NSTAGES - 1)); pi++)
158  {
159  if (((pi - 1) >= 0) && ((pi - 1) < n_left_from))
160  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
161  if (((pi - 2) >= 0) && ((pi - 2) < n_left_from))
162  nexts[pi - 2] =
163  last_stage (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
164  }
165 
166  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
167  return frame->n_vectors;
168 }
169 #endif
170 
171 #if NSTAGES == 4
172 static STAGE_INLINE uword
173 dispatch_pipeline (vlib_main_t * vm,
174  vlib_node_runtime_t * node, vlib_frame_t * frame)
175 {
176  u32 *from;
177  u32 n_left_from;
178  int pi;
180  u16 nexts[VLIB_FRAME_SIZE];
182 
183  n_left_from = frame->n_vectors;
184  from = vlib_frame_vector_args (frame);
185  vlib_get_buffers (vm, from, bufs, n_left_from);
186 
187  for (pi = 0; pi < NSTAGES - 1; pi++)
188  {
189  if (pi == n_left_from)
190  break;
191  stage0 (vm, node, bufs[pi] AUX_DATA_PTR (pi));
192  if (pi - 1 >= 0)
193  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
194  if (pi - 2 >= 0)
195  stage2 (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
196  }
197 
198  for (; pi < n_left_from; pi++)
199  {
200  stage0 (vm, node, bufs[pi] AUX_DATA_PTR (pi));
201  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
202  stage2 (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
203  nexts[pi - 3] =
204  last_stage (vm, node, bufs[pi - 3] AUX_DATA_PTR (pi - 3));
205  }
206 
207  for (; pi < (n_left_from + (NSTAGES - 1)); pi++)
208  {
209  if (((pi - 1) >= 0) && ((pi - 1) < n_left_from))
210  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
211  if (((pi - 2) >= 0) && ((pi - 2) < n_left_from))
212  stage2 (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
213  if (((pi - 3) >= 0) && ((pi - 3) < n_left_from))
214  nexts[pi - 3] =
215  last_stage (vm, node, bufs[pi - 3] AUX_DATA_PTR (pi - 3));
216  }
217 
218  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
219  return frame->n_vectors;
220 }
221 #endif
222 
223 #if NSTAGES == 5
224 static STAGE_INLINE uword
225 dispatch_pipeline (vlib_main_t * vm,
226  vlib_node_runtime_t * node, vlib_frame_t * frame)
227 {
228  u32 *from;
229  u32 n_left_from;
230  int pi;
232  u16 nexts[VLIB_FRAME_SIZE];
234 
235  n_left_from = frame->n_vectors;
236  from = vlib_frame_vector_args (frame);
237  vlib_get_buffers (vm, from, bufs, n_left_from);
238 
239  for (pi = 0; pi < NSTAGES - 1; pi++)
240  {
241  if (pi == n_left_from)
242  break;
243  stage0 (vm, node, bufs[pi] AUX_DATA_PTR (pi));
244  if (pi - 1 >= 0)
245  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
246  if (pi - 2 >= 0)
247  stage2 (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
248  if (pi - 3 >= 0)
249  stage3 (vm, node, bufs[pi - 3] AUX_DATA_PTR (pi - 3));
250  }
251 
252  for (; pi < n_left_from; pi++)
253  {
254  stage0 (vm, node, bufs[pi] AUX_DATA_PTR (pi));
255  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
256  stage2 (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
257  stage3 (vm, node, bufs[pi - 3] AUX_DATA_PTR (pi - 3));
258  nexts[pi - 4] =
259  last_stage (vm, node, bufs[pi - 4] AUX_DATA_PTR (pi - 4));
260  }
261 
262  for (; pi < (n_left_from + (NSTAGES - 1)); pi++)
263  {
264  if (((pi - 1) >= 0) && ((pi - 1) < n_left_from))
265  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
266  if (((pi - 2) >= 0) && ((pi - 2) < n_left_from))
267  stage2 (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
268  if (((pi - 3) >= 0) && ((pi - 3) < n_left_from))
269  stage3 (vm, node, bufs[pi - 3] AUX_DATA_PTR (pi - 3));
270  if (((pi - 4) >= 0) && ((pi - 4) < n_left_from))
271  nexts[pi - 4] =
272  last_stage (vm, node, bufs[pi - 4] AUX_DATA_PTR (pi - 4));
273  }
274 
275  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
276  return frame->n_vectors;
277 }
278 #endif
279 
280 #if NSTAGES == 6
281 static STAGE_INLINE uword
282 dispatch_pipeline (vlib_main_t * vm,
283  vlib_node_runtime_t * node, vlib_frame_t * frame)
284 {
285  u32 *from;
286  u32 n_left_from;
287  int pi;
289  u16 nexts[VLIB_FRAME_SIZE];
291 
292  n_left_from = frame->n_vectors;
293  from = vlib_frame_vector_args (frame);
294  vlib_get_buffers (vm, from, bufs, n_left_from);
295 
296  for (pi = 0; pi < NSTAGES - 1; pi++)
297  {
298  if (pi == n_left_from)
299  break;
300  stage0 (vm, node, bufs[pi] AUX_DATA_PTR (pi));
301  if (pi - 1 >= 0)
302  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
303  if (pi - 2 >= 0)
304  stage2 (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
305  if (pi - 3 >= 0)
306  stage3 (vm, node, bufs[pi - 3] AUX_DATA_PTR (pi - 3));
307  if (pi - 4 >= 0)
308  stage4 (vm, node, bufs[pi - 4] AUX_DATA_PTR (pi - 4));
309  }
310 
311  for (; pi < n_left_from; pi++)
312  {
313  stage0 (vm, node, bufs[pi] AUX_DATA_PTR (pi));
314  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
315  stage2 (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
316  stage3 (vm, node, bufs[pi - 3] AUX_DATA_PTR (pi - 3));
317  stage4 (vm, node, bufs[pi - 4] AUX_DATA_PTR (pi - 4));
318  nexts[pi - 5] =
319  last_stage (vm, node, bufs[pi - 5] AUX_DATA_PTR (pi - 5));
320  }
321 
322  for (; pi < (n_left_from + (NSTAGES - 1)); pi++)
323  {
324  if (((pi - 1) >= 0) && ((pi - 1) < n_left_from))
325  stage1 (vm, node, bufs[pi - 1] AUX_DATA_PTR (pi - 1));
326  if (((pi - 2) >= 0) && ((pi - 2) < n_left_from))
327  stage2 (vm, node, bufs[pi - 2] AUX_DATA_PTR (pi - 2));
328  if (((pi - 3) >= 0) && ((pi - 3) < n_left_from))
329  stage3 (vm, node, bufs[pi - 3] AUX_DATA_PTR (pi - 3));
330  if (((pi - 4) >= 0) && ((pi - 4) < n_left_from))
331  stage4 (vm, node, bufs[pi - 4] AUX_DATA_PTR (pi - 4));
332  if (((pi - 5) >= 0) && ((pi - 5) < n_left_from))
333  nexts[pi - 5] =
334  last_stage (vm, node, bufs[pi - 5] AUX_DATA_PTR (pi - 5));
335  }
336 
337  vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
338  return frame->n_vectors;
339 }
340 #endif
341 
342 /*
343  * fd.io coding-style-patch-verification: ON
344  *
345  * Local Variables:
346  * eval: (c-set-style "gnu")
347  * End:
348  */
static STAGE_INLINE void generic_stage0(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b AUX_DATA_ARG)
Definition: pipeline.h:74
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:376
#define NSTAGES
Definition: decap.c:57
unsigned short u16
Definition: types.h:57
static void stage1(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b)
Definition: decap.c:68
u16 n_vectors
Definition: node.h:395
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:332
#define STAGE_INLINE
Definition: pipeline.h:45
static u32 last_stage(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b)
Definition: decap.c:118
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:274
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define AUX_DATA_PTR(pi)
Definition: pipeline.h:52
#define AUX_DATA_DECL
Definition: pipeline.h:51
static void stage0(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b)
Definition: decap.c:60
#define AUX_DATA_ARG
Definition: pipeline.h:50