FD.io VPP  v21.06-1-gbb7418cf9
Vector Packet Processing
buffer_funcs.c
Go to the documentation of this file.
1 /* SPDX-License-Identifier: Apache-2.0
2  * Copyright(c) 2021 Cisco Systems, Inc.
3  */
4 
5 #include <vppinfra/clib.h>
6 #include <vlib/vlib.h>
8 
11  u16 next_index, u32 *buffers, u16 *nexts, u32 n_buffers,
12  u32 n_left, u32 *tmp)
13 {
14  u64 match_bmp[VLIB_FRAME_SIZE / 64];
15  vlib_frame_t *f;
16  u32 n_extracted, n_free;
17  u32 *to;
18 
19  f = vlib_get_next_frame_internal (vm, node, next_index, 0);
20 
21  n_free = VLIB_FRAME_SIZE - f->n_vectors;
22 
23  /* if frame contains enough space for worst case scenario, we can avoid
24  * use of tmp */
25  if (n_free >= n_left)
26  to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
27  else
28  to = tmp;
29 
30  clib_mask_compare_u16 (next_index, nexts, match_bmp, n_buffers);
31 
32  n_extracted = clib_compress_u32 (to, buffers, match_bmp, n_buffers);
33 
34  for (int i = 0; i < ARRAY_LEN (match_bmp); i++)
35  used_elt_bmp[i] |= match_bmp[i];
36 
37  if (to != tmp)
38  {
39  /* indices already written to frame, just close it */
40  vlib_put_next_frame (vm, node, next_index, n_free - n_extracted);
41  }
42  else if (n_free >= n_extracted)
43  {
44  /* enough space in the existing frame */
45  to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
46  vlib_buffer_copy_indices (to, tmp, n_extracted);
47  vlib_put_next_frame (vm, node, next_index, n_free - n_extracted);
48  }
49  else
50  {
51  /* full frame */
52  to = (u32 *) vlib_frame_vector_args (f) + f->n_vectors;
53  vlib_buffer_copy_indices (to, tmp, n_free);
54  vlib_put_next_frame (vm, node, next_index, 0);
55 
56  /* second frame */
57  u32 n_2nd_frame = n_extracted - n_free;
58  f = vlib_get_next_frame_internal (vm, node, next_index, 1);
59  to = vlib_frame_vector_args (f);
60  vlib_buffer_copy_indices (to, tmp + n_free, n_2nd_frame);
61  vlib_put_next_frame (vm, node, next_index,
62  VLIB_FRAME_SIZE - n_2nd_frame);
63  }
64 
65  return n_left - n_extracted;
66 }
67 
68 void __clib_section (".vlib_buffer_enqueue_to_next_fn")
69 CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_next_fn)
71  uword count)
72 {
74  u32 n_left;
76 
77  while (count >= VLIB_FRAME_SIZE)
78  {
79  u64 used_elt_bmp[VLIB_FRAME_SIZE / 64] = {};
80  n_left = VLIB_FRAME_SIZE;
81  u32 off = 0;
82 
83  next_index = nexts[0];
84  n_left = enqueue_one (vm, node, used_elt_bmp, next_index, buffers, nexts,
85  VLIB_FRAME_SIZE, n_left, tmp);
86 
87  while (n_left)
88  {
89  while (PREDICT_FALSE (used_elt_bmp[off] == ~0))
90  off++;
91 
92  next_index =
93  nexts[off * 64 + count_trailing_zeros (~used_elt_bmp[off])];
94  n_left = enqueue_one (vm, node, used_elt_bmp, next_index, buffers,
95  nexts, VLIB_FRAME_SIZE, n_left, tmp);
96  }
97 
98  buffers += VLIB_FRAME_SIZE;
101  }
102 
103  if (count)
104  {
105  u64 used_elt_bmp[VLIB_FRAME_SIZE / 64] = {};
106  next_index = nexts[0];
107  n_left = count;
108  u32 off = 0;
109 
110  n_left = enqueue_one (vm, node, used_elt_bmp, next_index, buffers, nexts,
111  count, n_left, tmp);
112 
113  while (n_left)
114  {
115  while (PREDICT_FALSE (used_elt_bmp[off] == ~0))
116  off++;
117 
118  next_index =
119  nexts[off * 64 + count_trailing_zeros (~used_elt_bmp[off])];
120  n_left = enqueue_one (vm, node, used_elt_bmp, next_index, buffers,
121  nexts, count, n_left, tmp);
122  }
123  }
124 }
125 
126 CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_next_fn);
127 
128 void __clib_section (".vlib_buffer_enqueue_to_single_next_fn")
129 CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_single_next_fn)
131  u32 count)
132 {
133  u32 *to_next, n_left_to_next, n_enq;
134 
135  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
136 
137  if (PREDICT_TRUE (n_left_to_next >= count))
138  {
139  vlib_buffer_copy_indices (to_next, buffers, count);
140  n_left_to_next -= count;
141  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
142  return;
143  }
144 
145  n_enq = n_left_to_next;
146 next:
147  vlib_buffer_copy_indices (to_next, buffers, n_enq);
148  n_left_to_next -= n_enq;
149 
150  if (PREDICT_FALSE (count > n_enq))
151  {
152  count -= n_enq;
153  buffers += n_enq;
154 
155  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
156  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
157  n_enq = clib_min (n_left_to_next, count);
158  goto next;
159  }
160  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
161 }
162 CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_single_next_fn);
163 
164 u32 __clib_section (".vlib_buffer_enqueue_to_thread_fn")
165 CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_thread_fn)
166 (vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index,
167  u32 *buffer_indices, u16 *thread_indices, u32 n_packets,
168  int drop_on_congestion)
169 {
173  u32 n_left = n_packets;
174  u32 drop_list[VLIB_FRAME_SIZE], *dbi = drop_list, n_drop = 0;
175  vlib_frame_queue_elt_t *hf = 0;
176  u32 n_left_to_next_thread = 0, *to_next_thread = 0;
177  u32 next_thread_index, current_thread_index = ~0;
178  int i;
179 
180  fqm = vec_elt_at_index (tm->frame_queue_mains, frame_queue_index);
182 
183  while (n_left)
184  {
185  next_thread_index = thread_indices[0];
186 
187  if (next_thread_index != current_thread_index)
188  {
189  if (drop_on_congestion &&
191  frame_queue_index, next_thread_index, fqm->queue_hi_thresh,
193  {
194  dbi[0] = buffer_indices[0];
195  dbi++;
196  n_drop++;
197  goto next;
198  }
199 
200  if (hf)
201  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_thread;
202 
204  frame_queue_index, next_thread_index,
206 
207  n_left_to_next_thread = VLIB_FRAME_SIZE - hf->n_vectors;
208  to_next_thread = &hf->buffer_index[hf->n_vectors];
209  current_thread_index = next_thread_index;
210  }
211 
212  to_next_thread[0] = buffer_indices[0];
213  to_next_thread++;
214  n_left_to_next_thread--;
215 
216  if (n_left_to_next_thread == 0)
217  {
220  vlib_get_main_by_index (current_thread_index)->check_frame_queues =
221  1;
222  current_thread_index = ~0;
224  hf = 0;
225  }
226 
227  /* next */
228  next:
229  thread_indices += 1;
230  buffer_indices += 1;
231  n_left -= 1;
232  }
233 
234  if (hf)
235  hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_thread;
236 
237  /* Ship frames to the thread nodes */
238  for (i = 0; i < vec_len (ptd->handoff_queue_elt_by_thread_index); i++)
239  {
241  {
243  /*
244  * It works better to let the handoff node
245  * rate-adapt, always ship the handoff queue element.
246  */
247  if (1 || hf->n_vectors == hf->last_n_vectors)
248  {
252  }
253  else
254  hf->last_n_vectors = hf->n_vectors;
255  }
257  (vlib_frame_queue_t *) (~0);
258  }
259 
260  if (drop_on_congestion && n_drop)
261  vlib_buffer_free (vm, drop_list, n_drop);
262 
263  return n_packets - n_drop;
264 }
265 
266 CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_thread_fn);
267 
268 /*
269  * Check the frame queue to see if any frames are available.
270  * If so, pull the packets off the frames and put them to
271  * the handoff node.
272  */
273 u32 __clib_section (".vlib_frame_queue_dequeue_fn")
274 CLIB_MULTIARCH_FN (vlib_frame_queue_dequeue_fn)
276 {
277  u32 thread_id = vm->thread_index;
278  vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id];
280  u32 *from, *to;
281  vlib_frame_t *f;
282  int msg_type;
283  int processed = 0;
284  u32 vectors = 0;
285 
286  ASSERT (fq);
287  ASSERT (vm == vlib_global_main.vlib_mains[thread_id]);
288 
289  if (PREDICT_FALSE (fqm->node_index == ~0))
290  return 0;
291  /*
292  * Gather trace data for frame queues
293  */
294  if (PREDICT_FALSE (fq->trace))
295  {
296  frame_queue_trace_t *fqt;
298  u32 elix;
299 
300  fqt = &fqm->frame_queue_traces[thread_id];
301 
302  fqt->nelts = fq->nelts;
303  fqt->head = fq->head;
304  fqt->head_hint = fq->head_hint;
305  fqt->tail = fq->tail;
306  fqt->threshold = fq->vector_threshold;
307  fqt->n_in_use = fqt->tail - fqt->head;
308  if (fqt->n_in_use >= fqt->nelts)
309  {
310  // if beyond max then use max
311  fqt->n_in_use = fqt->nelts - 1;
312  }
313 
314  /* Record the number of elements in use in the histogram */
315  fqh = &fqm->frame_queue_histogram[thread_id];
316  fqh->count[fqt->n_in_use]++;
317 
318  /* Record a snapshot of the elements in use */
319  for (elix = 0; elix < fqt->nelts; elix++)
320  {
321  elt = fq->elts + ((fq->head + 1 + elix) & (fq->nelts - 1));
322  if (1 || elt->valid)
323  {
324  fqt->n_vectors[elix] = elt->n_vectors;
325  }
326  }
327  fqt->written = 1;
328  }
329 
330  while (1)
331  {
332  vlib_buffer_t *b;
333  if (fq->head == fq->tail)
334  {
335  fq->head_hint = fq->head;
336  return processed;
337  }
338 
339  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
340 
341  if (!elt->valid)
342  {
343  fq->head_hint = fq->head;
344  return processed;
345  }
346 
347  from = elt->buffer_index;
348  msg_type = elt->msg_type;
349 
351  ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE);
352 
353  f = vlib_get_frame_to_node (vm, fqm->node_index);
354 
355  /* If the first vector is traced, set the frame trace flag */
356  b = vlib_get_buffer (vm, from[0]);
357  if (b->flags & VLIB_BUFFER_IS_TRACED)
359 
360  to = vlib_frame_vector_args (f);
361 
362  vlib_buffer_copy_indices (to, from, elt->n_vectors);
363 
364  vectors += elt->n_vectors;
365  f->n_vectors = elt->n_vectors;
366  vlib_put_frame_to_node (vm, fqm->node_index, f);
367 
368  elt->valid = 0;
369  elt->n_vectors = 0;
370  elt->msg_type = 0xfefefefe;
372  fq->head++;
373  processed++;
374 
375  /*
376  * Limit the number of packets pushed into the graph
377  */
378  if (vectors >= fq->vector_threshold)
379  {
380  fq->head_hint = fq->head;
381  return processed;
382  }
383  }
384  ASSERT (0);
385  return processed;
386 }
387 CLIB_MARCH_FN_REGISTRATION (vlib_frame_queue_dequeue_fn);
388 
389 #ifndef CLIB_MARCH_VARIANT
391 
392 static clib_error_t *
394 {
397  CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_next_fn);
399  CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_single_next_fn);
401  CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_thread_fn);
403  CLIB_MARCH_FN_POINTER (vlib_frame_queue_dequeue_fn);
404  return 0;
405 }
406 
408 #endif
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:133
#define CLIB_MARCH_FN_POINTER(fn)
Definition: cpu.h:84
#define clib_min(x, y)
Definition: clib.h:342
u32 n_free
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:982
vlib_frame_t * vlib_get_next_frame_internal(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 allocate_new_next_frame)
Definition: main.c:384
static_always_inline u32 clib_compress_u32(u32 *dst, u32 *src, u64 *mask, u32 n_elts)
Compare array of 32-bit elemments into destination array based on mask.
Definition: vector_funcs.h:210
vlib_buffer_copy_indices(to, tmp, n_free)
#define PREDICT_TRUE(x)
Definition: clib.h:125
unsigned long u64
Definition: types.h:89
vlib_buffer_enqueue_to_single_next_fn_t * buffer_enqueue_to_single_next_fn
Definition: buffer_funcs.h:73
u16 nexts[VLIB_FRAME_SIZE]
u32 thread_index
Definition: main.h:213
static_always_inline u32 enqueue_one(vlib_main_t *vm, vlib_node_runtime_t *node, u64 *used_elt_bmp, u16 next_index, u32 *buffers, u16 *nexts, u32 n_buffers, u32 n_left, u32 *tmp)
Definition: buffer_funcs.c:10
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:81
volatile u32 valid
Definition: threads.h:75
static vlib_frame_queue_t * is_vlib_frame_queue_congested(u32 frame_queue_index, u32 index, u32 queue_hi_thresh, vlib_frame_queue_t **handoff_queue_by_worker_index)
Definition: threads.h:552
vlib_buffer_t ** b
#define count_trailing_zeros(x)
Definition: clib.h:161
unsigned int u32
Definition: types.h:88
volatile uword check_frame_queues
Definition: main.h:257
vlib_frame_t * f
#define static_always_inline
Definition: clib.h:112
u64 count[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:779
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:172
vlib_buffer_func_main_t vlib_buffer_func_main
Definition: buffer_funcs.c:390
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
Definition: threads.h:144
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:184
vlib_frame_queue_elt_t * elts
Definition: threads.h:137
static_always_inline void clib_mask_compare_u16(u16 v, u16 *a, u64 *mask, u32 n_elts)
Compare 16-bit elemments with provied value and return bitmap.
Definition: vector_funcs.h:67
#define VLIB_FRAME_SIZE
Definition: node.h:369
vlib_buffer_enqueue_to_next_fn_t * buffer_enqueue_to_next_fn
Definition: buffer_funcs.h:72
u16 * next
u32 * to
volatile u64 head
Definition: threads.h:127
u16 frame_flags
Definition: node.h:376
unsigned short u16
Definition: types.h:57
vlib_main_t ** vlib_mains
Definition: main.h:279
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:218
u32 * tmp
#define PREDICT_FALSE(x)
Definition: clib.h:124
static vlib_frame_queue_elt_t * vlib_get_worker_handoff_queue_elt(u32 frame_queue_index, u32 vlib_worker_index, vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index)
Definition: threads.h:584
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:395
u32 n_left
CLIB_MARCH_FN_REGISTRATION(vlib_buffer_enqueue_to_next_fn)
i32 n_vectors[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:774
volatile u64 tail
Definition: threads.h:122
u16 n_vectors
Definition: node.h:388
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
static clib_error_t * vlib_buffer_funcs_init(vlib_main_t *vm)
Definition: buffer_funcs.c:393
#define ARRAY_LEN(x)
Definition: clib.h:70
vlib_frame_queue_per_thread_data_t * per_thread_data
Definition: threads.h:155
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
Definition: threads.h:145
#define ASSERT(truth)
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:310
app_rx_mq_elt_t * elt
Definition: application.c:488
vlib_put_next_frame(vm, node, next_index, 0)
u32 n_buffers
nat44_ei_hairpin_src_next_t next_index
vlib_frame_queue_dequeue_fn_t * frame_queue_dequeue_fn
Definition: buffer_funcs.h:75
static vlib_main_t * vlib_get_main_by_index(u32 thread_index)
Definition: global_funcs.h:29
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
vlib_main_t vlib_node_runtime_t * node
Definition: nat44_ei.c:3047
VLIB buffer representation.
Definition: buffer.h:111
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:301
volatile u64 head_hint
Definition: threads.h:133
static void vlib_put_frame_queue_elt(vlib_frame_queue_elt_t *hf)
Definition: threads.h:514
vlib_buffer_enqueue_to_thread_fn_t * buffer_enqueue_to_thread_fn
Definition: buffer_funcs.h:74
u32 off
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:56
static u32 next_thread_index(vnet_main_t *vnm, u32 thread_index)
Definition: rx_queue.c:30
u8 count
Definition: dhcp.api:208
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:137
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:292
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:111
vlib_global_main_t vlib_global_main
Definition: main.c:1786
#define CLIB_MULTIARCH_FN(fn)
Definition: cpu.h:53