FD.io VPP  v19.04-6-g6f05f72
Vector Packet Processing
buffer_funcs.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer_funcs.h: VLIB buffer related functions/inlines
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
42 
43 #include <vppinfra/hash.h>
44 #include <vppinfra/fifo.h>
45 #include <vlib/buffer.h>
46 #include <vlib/physmem_funcs.h>
47 #include <vlib/main.h>
48 #include <vlib/node.h>
49 
50 /** \file
51  vlib buffer access methods.
52 */
53 
54 always_inline void
56 {
59 
60  /* reference count in allocated buffer always must be 1 or higher */
61  ASSERT (b->ref_count > 0);
62 
63  /* verify that buffer pool index is valid */
65  ASSERT (pointer_to_uword (b) >= bp->start);
66  ASSERT (pointer_to_uword (b) < bp->start + bp->size -
67  (bp->data_size + sizeof (vlib_buffer_t)));
68 }
69 
70 always_inline void *
71 vlib_buffer_ptr_from_index (uword buffer_mem_start, u32 buffer_index,
72  uword offset)
73 {
74  offset += ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
75  return uword_to_pointer (buffer_mem_start + offset, vlib_buffer_t *);
76 }
77 
78 /** \brief Translate buffer index into buffer pointer
79 
80  @param vm - (vlib_main_t *) vlib main data structure pointer
81  @param buffer_index - (u32) buffer index
82  @return - (vlib_buffer_t *) buffer pointer
83 */
85 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
86 {
88  vlib_buffer_t *b;
89 
90  b = vlib_buffer_ptr_from_index (bm->buffer_mem_start, buffer_index, 0);
91  vlib_buffer_validate (vm, b);
92  return b;
93 }
94 
97 {
98  return vm->buffer_main->default_data_size;
99 }
100 
103 {
104 #if defined(CLIB_HAVE_VEC512)
105  while (n_indices >= 16)
106  {
107  u32x16_store_unaligned (u32x16_load_unaligned (src), dst);
108  dst += 16;
109  src += 16;
110  n_indices -= 16;
111  }
112 #endif
113 
114 #if defined(CLIB_HAVE_VEC256)
115  while (n_indices >= 8)
116  {
117  u32x8_store_unaligned (u32x8_load_unaligned (src), dst);
118  dst += 8;
119  src += 8;
120  n_indices -= 8;
121  }
122 #endif
123 
124 #if defined(CLIB_HAVE_VEC128)
125  while (n_indices >= 4)
126  {
127  u32x4_store_unaligned (u32x4_load_unaligned (src), dst);
128  dst += 4;
129  src += 4;
130  n_indices -= 4;
131  }
132 #endif
133 
134  while (n_indices)
135  {
136  dst[0] = src[0];
137  dst += 1;
138  src += 1;
139  n_indices -= 1;
140  }
141 }
142 
143 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, template_end, 64);
146 {
147 #if defined CLIB_HAVE_VEC512
148  b->as_u8x64[0] = bt->as_u8x64[0];
149 #elif defined (CLIB_HAVE_VEC256)
150  b->as_u8x32[0] = bt->as_u8x32[0];
151  b->as_u8x32[1] = bt->as_u8x32[1];
152 #elif defined (CLIB_HAVE_VEC128)
153  b->as_u8x16[0] = bt->as_u8x16[0];
154  b->as_u8x16[1] = bt->as_u8x16[1];
155  b->as_u8x16[2] = bt->as_u8x16[2];
156  b->as_u8x16[3] = bt->as_u8x16[3];
157 #else
158  clib_memcpy_fast (b, bt, 64);
159 #endif
160 }
161 
164 {
165  ASSERT (numa_node < VLIB_BUFFER_MAX_NUMA_NODES);
166  return vm->buffer_main->default_buffer_pool_index_for_numa[numa_node];
167 }
168 
169 /** \brief Translate array of buffer indices into buffer pointers with offset
170 
171  @param vm - (vlib_main_t *) vlib main data structure pointer
172  @param bi - (u32 *) array of buffer indices
173  @param b - (void **) array to store buffer pointers
174  @param count - (uword) number of elements
175  @param offset - (i32) offset applied to each pointer
176 */
179  i32 offset)
180 {
181  uword buffer_mem_start = vm->buffer_main->buffer_mem_start;
182 #ifdef CLIB_HAVE_VEC256
183  u64x4 off = u64x4_splat (buffer_mem_start + offset);
184  /* if count is not const, compiler will not unroll while loop
185  se we maintain two-in-parallel variant */
186  while (count >= 8)
187  {
188  u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
189  u64x4 b1 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi + 4));
190  /* shift and add to get vlib_buffer_t pointer */
191  u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
192  u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
193  b += 8;
194  bi += 8;
195  count -= 8;
196  }
197 #endif
198  while (count >= 4)
199  {
200 #ifdef CLIB_HAVE_VEC256
201  u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
202  /* shift and add to get vlib_buffer_t pointer */
203  u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
204 #elif defined (CLIB_HAVE_VEC128)
205  u64x2 off = u64x2_splat (buffer_mem_start + offset);
206  u32x4 bi4 = u32x4_load_unaligned (bi);
207  u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
208 #if defined (__aarch64__)
210 #else
211  bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
212  u64x2 b1 = u32x4_extend_to_u64x2 ((u32x4) bi4);
213 #endif
214  u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
215  u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
216 #else
217  b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
218  b[1] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[1], offset);
219  b[2] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[2], offset);
220  b[3] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[3], offset);
221 #endif
222  b += 4;
223  bi += 4;
224  count -= 4;
225  }
226  while (count)
227  {
228  b[0] = vlib_buffer_ptr_from_index (buffer_mem_start, bi[0], offset);
229  b += 1;
230  bi += 1;
231  count -= 1;
232  }
233 }
234 
235 /** \brief Translate array of buffer indices into buffer pointers
236 
237  @param vm - (vlib_main_t *) vlib main data structure pointer
238  @param bi - (u32 *) array of buffer indices
239  @param b - (vlib_buffer_t **) array to store buffer pointers
240  @param count - (uword) number of elements
241 */
242 
245 {
246  vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0);
247 }
248 
249 /** \brief Translate buffer pointer into buffer index
250 
251  @param vm - (vlib_main_t *) vlib main data structure pointer
252  @param p - (void *) buffer pointer
253  @return - (u32) buffer index
254 */
255 
258 {
262  ASSERT (offset < bm->buffer_mem_size);
263  ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
264  return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
265 }
266 
267 /** \brief Translate array of buffer pointers into buffer indices with offset
268 
269  @param vm - (vlib_main_t *) vlib main data structure pointer
270  @param b - (void **) array of buffer pointers
271  @param bi - (u32 *) array to store buffer indices
272  @param count - (uword) number of elements
273  @param offset - (i32) offset applied to each pointer
274 */
278 {
279 #ifdef CLIB_HAVE_VEC256
280  u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
281  u64x4 off4 = u64x4_splat (vm->buffer_main->buffer_mem_start - offset);
282 
283  while (count >= 8)
284  {
285  /* load 4 pointers into 256-bit register */
286  u64x4 v0 = u64x4_load_unaligned (b);
287  u64x4 v1 = u64x4_load_unaligned (b + 4);
288  u32x8 v2, v3;
289 
290  v0 -= off4;
291  v1 -= off4;
292 
295 
296  /* permute 256-bit register so lower u32s of each buffer index are
297  * placed into lower 128-bits */
298  v2 = u32x8_permute ((u32x8) v0, mask);
299  v3 = u32x8_permute ((u32x8) v1, mask);
300 
301  /* extract lower 128-bits and save them to the array of buffer indices */
302  u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
303  u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
304  bi += 8;
305  b += 8;
306  count -= 8;
307  }
308 #endif
309  while (count >= 4)
310  {
311  /* equivalent non-nector implementation */
312  bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
313  bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset);
314  bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset);
315  bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset);
316  bi += 4;
317  b += 4;
318  count -= 4;
319  }
320  while (count)
321  {
322  bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
323  bi += 1;
324  b += 1;
325  count -= 1;
326  }
327 }
328 
329 /** \brief Translate array of buffer pointers into buffer indices
330 
331  @param vm - (vlib_main_t *) vlib main data structure pointer
332  @param b - (vlib_buffer_t **) array of buffer pointers
333  @param bi - (u32 *) array to store buffer indices
334  @param count - (uword) number of elements
335 */
338  uword count)
339 {
340  vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0);
341 }
342 
343 /** \brief Get next buffer in buffer linklist, or zero for end of list.
344 
345  @param vm - (vlib_main_t *) vlib main data structure pointer
346  @param b - (void *) buffer pointer
347  @return - (vlib_buffer_t *) next buffer, or NULL
348 */
351 {
352  return (b->flags & VLIB_BUFFER_NEXT_PRESENT
353  ? vlib_get_buffer (vm, b->next_buffer) : 0);
354 }
355 
357  vlib_buffer_t * b_first);
358 
359 /** \brief Get length in bytes of the buffer chain
360 
361  @param vm - (vlib_main_t *) vlib main data structure pointer
362  @param b - (void *) buffer pointer
363  @return - (uword) length of buffer chain
364 */
367 {
368  uword len = b->current_length;
369 
370  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
371  return len;
372 
373  if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
375 
377 }
378 
379 /** \brief Get length in bytes of the buffer index buffer chain
380 
381  @param vm - (vlib_main_t *) vlib main data structure pointer
382  @param bi - (u32) buffer index
383  @return - (uword) length of buffer chain
384 */
387 {
388  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
389  return vlib_buffer_length_in_chain (vm, b);
390 }
391 
392 /** \brief Copy buffer contents to memory
393 
394  @param vm - (vlib_main_t *) vlib main data structure pointer
395  @param buffer_index - (u32) buffer index
396  @param contents - (u8 *) memory, <strong>must be large enough</strong>
397  @return - (uword) length of buffer chain
398 */
400 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
401 {
402  uword content_len = 0;
403  uword l;
404  vlib_buffer_t *b;
405 
406  while (1)
407  {
408  b = vlib_get_buffer (vm, buffer_index);
409  l = b->current_length;
410  clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
411  content_len += l;
412  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
413  break;
414  buffer_index = b->next_buffer;
415  }
416 
417  return content_len;
418 }
419 
422 {
423  return vlib_physmem_get_pa (vm, b->data);
424 }
425 
428 {
429  return vlib_buffer_get_pa (vm, b) + b->current_data;
430 }
431 
432 /** \brief Prefetch buffer metadata by buffer index
433  The first 64 bytes of buffer contains most header information
434 
435  @param vm - (vlib_main_t *) vlib main data structure pointer
436  @param bi - (u32) buffer index
437  @param type - LOAD, STORE. In most cases, STORE is the right answer
438 */
439 /* Prefetch buffer header given index. */
440 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
441  do { \
442  vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
443  vlib_prefetch_buffer_header (_b, type); \
444  } while (0)
445 
446 typedef enum
447 {
448  /* Index is unknown. */
450 
451  /* Index is known and free/allocated. */
455 
456 void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
457  uword n_buffers,
459  expected_state);
460 
463 {
465 
467  uword *p = hash_get (bm->buffer_known_hash, buffer_index);
469  return p ? p[0] : VLIB_BUFFER_UNKNOWN;
470 }
471 
472 /* Validates sanity of a single buffer.
473  Returns format'ed vector with error message if any. */
474 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
475  uword follow_chain);
476 
478 vlib_get_buffer_pool (vlib_main_t * vm, u8 buffer_pool_index)
479 {
481  return vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
482 }
483 
485 vlib_buffer_pool_get (vlib_main_t * vm, u8 buffer_pool_index, u32 * buffers,
486  u32 n_buffers)
487 {
488  vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
489  u32 len;
490 
491  ASSERT (bp->buffers);
492 
493  clib_spinlock_lock (&bp->lock);
494  len = vec_len (bp->buffers);
495  if (PREDICT_TRUE (n_buffers < len))
496  {
497  len -= n_buffers;
498  vlib_buffer_copy_indices (buffers, bp->buffers + len, n_buffers);
499  _vec_len (bp->buffers) = len;
500  clib_spinlock_unlock (&bp->lock);
501  return n_buffers;
502  }
503  else
504  {
505  vlib_buffer_copy_indices (buffers, bp->buffers, len);
506  _vec_len (bp->buffers) = 0;
507  clib_spinlock_unlock (&bp->lock);
508  return len;
509  }
510 }
511 
512 
513 /** \brief Allocate buffers from specific pool into supplied array
514 
515  @param vm - (vlib_main_t *) vlib main data structure pointer
516  @param buffers - (u32 * ) buffer index array
517  @param n_buffers - (u32) number of buffers requested
518  @return - (u32) number of buffers actually allocated, may be
519  less than the number requested or zero
520 */
521 
523 vlib_buffer_alloc_from_pool (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
524  u8 buffer_pool_index)
525 {
527  vlib_buffer_pool_t *bp;
529  u32 *src, *dst, len, n_left;
530 
531  bp = vec_elt_at_index (bm->buffer_pools, buffer_pool_index);
532  bpt = vec_elt_at_index (bp->threads, vm->thread_index);
533 
534  dst = buffers;
535  n_left = n_buffers;
536  len = vec_len (bpt->cached_buffers);
537 
538  /* per-thread cache contains enough buffers */
539  if (len >= n_buffers)
540  {
541  src = bpt->cached_buffers + len - n_buffers;
542  vlib_buffer_copy_indices (dst, src, n_buffers);
543  _vec_len (bpt->cached_buffers) -= n_buffers;
544 
545  if (CLIB_DEBUG > 0)
546  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
548  return n_buffers;
549  }
550 
551  /* take everything available in the cache */
552  if (len)
553  {
554  vlib_buffer_copy_indices (dst, bpt->cached_buffers, len);
555  _vec_len (bpt->cached_buffers) = 0;
556  dst += len;
557  n_left -= len;
558  }
559 
560  len = round_pow2 (n_left, 32);
562  len = vlib_buffer_pool_get (vm, buffer_pool_index, bpt->cached_buffers,
563  len);
564  _vec_len (bpt->cached_buffers) = len;
565 
566  if (len)
567  {
568  u32 n_copy = clib_min (len, n_left);
569  src = bpt->cached_buffers + len - n_copy;
570  vlib_buffer_copy_indices (dst, src, n_copy);
571  _vec_len (bpt->cached_buffers) -= n_copy;
572  n_left -= n_copy;
573  }
574 
575  n_buffers -= n_left;
576 
577  /* Verify that buffers are known free. */
578  if (CLIB_DEBUG > 0)
579  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
581 
582  return n_buffers;
583 }
584 
585 /** \brief Allocate buffers from specific numa node into supplied array
586 
587  @param vm - (vlib_main_t *) vlib main data structure pointer
588  @param buffers - (u32 * ) buffer index array
589  @param n_buffers - (u32) number of buffers requested
590  @param numa_node - (u32) numa node
591  @return - (u32) number of buffers actually allocated, may be
592  less than the number requested or zero
593 */
595 vlib_buffer_alloc_on_numa (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
596  u32 numa_node)
597 {
598  u8 index = vlib_buffer_pool_get_default_for_numa (vm, numa_node);
599  return vlib_buffer_alloc_from_pool (vm, buffers, n_buffers, index);
600 }
601 
602 /** \brief Allocate buffers into supplied array
603 
604  @param vm - (vlib_main_t *) vlib main data structure pointer
605  @param buffers - (u32 * ) buffer index array
606  @param n_buffers - (u32) number of buffers requested
607  @return - (u32) number of buffers actually allocated, may be
608  less than the number requested or zero
609 */
610 
612 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
613 {
614  return vlib_buffer_alloc_on_numa (vm, buffers, n_buffers, vm->numa_node);
615 }
616 
617 /** \brief Allocate buffers into ring
618 
619  @param vm - (vlib_main_t *) vlib main data structure pointer
620  @param buffers - (u32 * ) buffer index ring
621  @param start - (u32) first slot in the ring
622  @param ring_size - (u32) ring size
623  @param n_buffers - (u32) number of buffers requested
624  @return - (u32) number of buffers actually allocated, may be
625  less than the number requested or zero
626 */
629  u32 ring_size, u32 n_buffers)
630 {
631  u32 n_alloc;
632 
633  ASSERT (n_buffers <= ring_size);
634 
635  if (PREDICT_TRUE (start + n_buffers <= ring_size))
636  return vlib_buffer_alloc (vm, ring + start, n_buffers);
637 
638  n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);
639 
640  if (PREDICT_TRUE (n_alloc == ring_size - start))
641  n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);
642 
643  return n_alloc;
644 }
645 
646 /** \brief Allocate buffers into ring from specific buffer pool
647 
648  @param vm - (vlib_main_t *) vlib main data structure pointer
649  @param buffers - (u32 * ) buffer index ring
650  @param start - (u32) first slot in the ring
651  @param ring_size - (u32) ring size
652  @param n_buffers - (u32) number of buffers requested
653  @return - (u32) number of buffers actually allocated, may be
654  less than the number requested or zero
655 */
658  u32 ring_size, u32 n_buffers,
659  u8 buffer_pool_index)
660 {
661  u32 n_alloc;
662 
663  ASSERT (n_buffers <= ring_size);
664 
665  if (PREDICT_TRUE (start + n_buffers <= ring_size))
666  return vlib_buffer_alloc_from_pool (vm, ring + start, n_buffers,
667  buffer_pool_index);
668 
669  n_alloc = vlib_buffer_alloc_from_pool (vm, ring + start, ring_size - start,
670  buffer_pool_index);
671 
672  if (PREDICT_TRUE (n_alloc == ring_size - start))
673  n_alloc += vlib_buffer_alloc_from_pool (vm, ring, n_buffers - n_alloc,
674  buffer_pool_index);
675 
676  return n_alloc;
677 }
678 
680 vlib_buffer_pool_put (vlib_main_t * vm, u8 buffer_pool_index,
681  u32 * buffers, u32 n_buffers)
682 {
683  vlib_buffer_pool_t *bp = vlib_get_buffer_pool (vm, buffer_pool_index);
686 
687  if (CLIB_DEBUG > 0)
688  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
690 
691  vec_add_aligned (bpt->cached_buffers, buffers, n_buffers,
693 
694  if (vec_len (bpt->cached_buffers) > 4 * VLIB_FRAME_SIZE)
695  {
696  clib_spinlock_lock (&bp->lock);
697  /* keep last stored buffers, as they are more likely hot in the cache */
701  bpt->n_alloc -= VLIB_FRAME_SIZE;
702  clib_spinlock_unlock (&bp->lock);
703  }
704 }
705 
707 vlib_buffer_free_inline (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
708  int maybe_next)
709 {
710  const int queue_size = 128;
711  vlib_buffer_pool_t *bp = 0;
712  u8 buffer_pool_index = ~0;
713  u32 n_queue = 0, queue[queue_size + 4];
714  vlib_buffer_t bt = { };
715 #if defined(CLIB_HAVE_VEC128)
716  vlib_buffer_t bpi_mask = {.buffer_pool_index = ~0 };
717  vlib_buffer_t bpi_vec = {.buffer_pool_index = ~0 };
718  vlib_buffer_t flags_refs_mask = {
719  .flags = VLIB_BUFFER_NEXT_PRESENT,
720  .ref_count = ~1
721  };
722 #endif
723 
724  while (n_buffers)
725  {
726  vlib_buffer_t *b[8];
727  u32 bi, sum = 0, flags, next;
728 
729  if (n_buffers < 12)
730  goto one_by_one;
731 
732  vlib_get_buffers (vm, buffers, b, 4);
733  vlib_get_buffers (vm, buffers + 8, b + 4, 4);
734 
735  vlib_prefetch_buffer_header (b[4], LOAD);
736  vlib_prefetch_buffer_header (b[5], LOAD);
737  vlib_prefetch_buffer_header (b[6], LOAD);
738  vlib_prefetch_buffer_header (b[7], LOAD);
739 
740 #if defined(CLIB_HAVE_VEC128)
741  u8x16 p0, p1, p2, p3, r;
742  p0 = u8x16_load_unaligned (b[0]);
743  p1 = u8x16_load_unaligned (b[1]);
744  p2 = u8x16_load_unaligned (b[2]);
745  p3 = u8x16_load_unaligned (b[3]);
746 
747  r = p0 ^ bpi_vec.as_u8x16[0];
748  r |= p1 ^ bpi_vec.as_u8x16[0];
749  r |= p2 ^ bpi_vec.as_u8x16[0];
750  r |= p3 ^ bpi_vec.as_u8x16[0];
751  r &= bpi_mask.as_u8x16[0];
752  r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0];
753 
754  sum = !u8x16_is_all_zero (r);
755 #else
756  sum |= b[0]->flags;
757  sum |= b[1]->flags;
758  sum |= b[2]->flags;
759  sum |= b[3]->flags;
760  sum &= VLIB_BUFFER_NEXT_PRESENT;
761  sum += b[0]->ref_count - 1;
762  sum += b[1]->ref_count - 1;
763  sum += b[2]->ref_count - 1;
764  sum += b[3]->ref_count - 1;
765  sum |= b[0]->buffer_pool_index ^ buffer_pool_index;
766  sum |= b[1]->buffer_pool_index ^ buffer_pool_index;
767  sum |= b[2]->buffer_pool_index ^ buffer_pool_index;
768  sum |= b[3]->buffer_pool_index ^ buffer_pool_index;
769 #endif
770 
771  if (sum)
772  goto one_by_one;
773 
774  vlib_buffer_copy_indices (queue + n_queue, buffers, 4);
775  vlib_buffer_copy_template (b[0], &bt);
776  vlib_buffer_copy_template (b[1], &bt);
777  vlib_buffer_copy_template (b[2], &bt);
778  vlib_buffer_copy_template (b[3], &bt);
779  n_queue += 4;
780 
781  vlib_buffer_validate (vm, b[0]);
782  vlib_buffer_validate (vm, b[1]);
783  vlib_buffer_validate (vm, b[2]);
784  vlib_buffer_validate (vm, b[3]);
785 
790 
791  if (n_queue >= queue_size)
792  {
793  vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
794  n_queue = 0;
795  }
796  buffers += 4;
797  n_buffers -= 4;
798  continue;
799 
800  one_by_one:
801  bi = buffers[0];
802 
803  next_in_chain:
804  b[0] = vlib_get_buffer (vm, bi);
805  flags = b[0]->flags;
806  next = b[0]->next_buffer;
807 
808  if (PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index))
809  {
810 
811  if (n_queue)
812  {
813  vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
814  n_queue = 0;
815  }
816 
817  buffer_pool_index = b[0]->buffer_pool_index;
818 #if defined(CLIB_HAVE_VEC128)
819  bpi_vec.buffer_pool_index = buffer_pool_index;
820 #endif
821  bp = vlib_get_buffer_pool (vm, buffer_pool_index);
823  }
824 
825  vlib_buffer_validate (vm, b[0]);
826 
828 
829  if (clib_atomic_sub_fetch (&b[0]->ref_count, 1) == 0)
830  {
831  vlib_buffer_copy_template (b[0], &bt);
832  queue[n_queue++] = bi;
833  }
834 
835  if (n_queue == queue_size)
836  {
837  vlib_buffer_pool_put (vm, buffer_pool_index, queue, queue_size);
838  n_queue = 0;
839  }
840 
841  if (flags & VLIB_BUFFER_NEXT_PRESENT)
842  {
843  bi = next;
844  goto next_in_chain;
845  }
846 
847  buffers++;
848  n_buffers--;
849  }
850 
851  if (n_queue)
852  vlib_buffer_pool_put (vm, buffer_pool_index, queue, n_queue);
853 }
854 
855 
856 /** \brief Free buffers
857  Frees the entire buffer chain for each buffer
858 
859  @param vm - (vlib_main_t *) vlib main data structure pointer
860  @param buffers - (u32 * ) buffer index array
861  @param n_buffers - (u32) number of buffers to free
862 
863 */
864 always_inline void
866  /* pointer to first buffer */
867  u32 * buffers,
868  /* number of buffers to free */
869  u32 n_buffers)
870 {
871  vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 1);
872 }
873 
874 /** \brief Free buffers, does not free the buffer chain for each buffer
875 
876  @param vm - (vlib_main_t *) vlib main data structure pointer
877  @param buffers - (u32 * ) buffer index array
878  @param n_buffers - (u32) number of buffers to free
879 
880 */
881 always_inline void
883  /* pointer to first buffer */
884  u32 * buffers,
885  /* number of buffers to free */
886  u32 n_buffers)
887 {
888  vlib_buffer_free_inline (vm, buffers, n_buffers, /* maybe next */ 0);
889 }
890 
891 /** \brief Free one buffer
892  Shorthand to free a single buffer chain.
893 
894  @param vm - (vlib_main_t *) vlib main data structure pointer
895  @param buffer_index - (u32) buffer index to free
896 */
897 always_inline void
899 {
900  vlib_buffer_free_inline (vm, &buffer_index, 1, /* maybe next */ 1);
901 }
902 
903 /** \brief Free buffers from ring
904 
905  @param vm - (vlib_main_t *) vlib main data structure pointer
906  @param buffers - (u32 * ) buffer index ring
907  @param start - (u32) first slot in the ring
908  @param ring_size - (u32) ring size
909  @param n_buffers - (u32) number of buffers
910 */
911 always_inline void
913  u32 ring_size, u32 n_buffers)
914 {
915  ASSERT (n_buffers <= ring_size);
916 
917  if (PREDICT_TRUE (start + n_buffers <= ring_size))
918  {
919  vlib_buffer_free (vm, ring + start, n_buffers);
920  }
921  else
922  {
923  vlib_buffer_free (vm, ring + start, ring_size - start);
924  vlib_buffer_free (vm, ring, n_buffers - (ring_size - start));
925  }
926 }
927 
928 /** \brief Free buffers from ring without freeing tail buffers
929 
930  @param vm - (vlib_main_t *) vlib main data structure pointer
931  @param buffers - (u32 * ) buffer index ring
932  @param start - (u32) first slot in the ring
933  @param ring_size - (u32) ring size
934  @param n_buffers - (u32) number of buffers
935 */
936 always_inline void
938  u32 ring_size, u32 n_buffers)
939 {
940  ASSERT (n_buffers <= ring_size);
941 
942  if (PREDICT_TRUE (start + n_buffers <= ring_size))
943  {
944  vlib_buffer_free_no_next (vm, ring + start, n_buffers);
945  }
946  else
947  {
948  vlib_buffer_free_no_next (vm, ring + start, ring_size - start);
949  vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start));
950  }
951 }
952 
953 /* Append given data to end of buffer, possibly allocating new buffers. */
954 int vlib_buffer_add_data (vlib_main_t * vm, u32 * buffer_index, void *data,
955  u32 n_data_bytes);
956 
957 /* duplicate all buffers in chain */
960 {
961  vlib_buffer_t *s, *d, *fd;
962  uword n_alloc, n_buffers = 1;
963  u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
964  int i;
965 
966  s = b;
967  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
968  {
969  n_buffers++;
970  s = vlib_get_buffer (vm, s->next_buffer);
971  }
972  u32 new_buffers[n_buffers];
973 
974  n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
975 
976  /* No guarantee that we'll get all the buffers we asked for */
977  if (PREDICT_FALSE (n_alloc < n_buffers))
978  {
979  if (n_alloc > 0)
980  vlib_buffer_free (vm, new_buffers, n_alloc);
981  return 0;
982  }
983 
984  /* 1st segment */
985  s = b;
986  fd = d = vlib_get_buffer (vm, new_buffers[0]);
987  d->current_data = s->current_data;
989  d->flags = s->flags & flag_mask;
992  clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
993  clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
996 
997  /* next segments */
998  for (i = 1; i < n_buffers; i++)
999  {
1000  /* previous */
1001  d->next_buffer = new_buffers[i];
1002  /* current */
1003  s = vlib_get_buffer (vm, s->next_buffer);
1004  d = vlib_get_buffer (vm, new_buffers[i]);
1005  d->current_data = s->current_data;
1009  d->flags = s->flags & flag_mask;
1010  }
1011 
1012  return fd;
1013 }
1014 
1015 /* duplicate first buffer in chain */
1018 {
1019  vlib_buffer_t *d;
1020 
1021  if ((vlib_buffer_alloc (vm, di, 1)) != 1)
1022  return 0;
1023 
1024  d = vlib_get_buffer (vm, *di);
1025  /* 1st segment */
1026  d->current_data = b->current_data;
1028  clib_memcpy_fast (d->opaque, b->opaque, sizeof (b->opaque));
1029  clib_memcpy_fast (d->opaque2, b->opaque2, sizeof (b->opaque2));
1032 
1033  return d;
1034 }
1035 
1036 /* \brief Move packet from current position to offset position in buffer.
1037  Only work for small packet using one buffer with room to fit the move
1038  @param vm - (vlib_main_t *) vlib main data structure pointer
1039  @param b - (vlib_buffer_t *) pointer to buffer
1040  @param offset - (i16) position to move the packet in buffer
1041  */
1042 always_inline void
1044 {
1045  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1046  ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
1047  ASSERT (offset + b->current_length <
1049 
1050  u8 *source = vlib_buffer_get_current (b);
1051  b->current_data = offset;
1052  u8 *destination = vlib_buffer_get_current (b);
1053  u16 length = b->current_length;
1054 
1055  if (source + length <= destination) /* no overlap */
1056  clib_memcpy_fast (destination, source, length);
1057  else
1058  memmove (destination, source, length);
1059 }
1060 
1061 /** \brief Create a maximum of 256 clones of buffer and store them
1062  in the supplied array
1063 
1064  @param vm - (vlib_main_t *) vlib main data structure pointer
1065  @param src_buffer - (u32) source buffer index
1066  @param buffers - (u32 * ) buffer index array
1067  @param n_buffers - (u16) number of buffer clones requested (<=256)
1068  @param head_end_offset - (u16) offset relative to current position
1069  where packet head ends
1070  @param offset - (i16) copy packet head at current position if 0,
1071  else at offset position to change headroom space as specified
1072  @return - (u16) number of buffers actually cloned, may be
1073  less than the number requested or zero
1074 */
1076 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1077  u16 n_buffers, u16 head_end_offset, i16 offset)
1078 {
1079  u16 i;
1080  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
1081 
1082  ASSERT (s->ref_count == 1);
1083  ASSERT (n_buffers);
1084  ASSERT (n_buffers <= 256);
1085  ASSERT (offset + VLIB_BUFFER_PRE_DATA_SIZE >= 0);
1086  ASSERT ((offset + head_end_offset) <
1088 
1089  if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
1090  {
1091  buffers[0] = src_buffer;
1092  if (offset)
1093  vlib_buffer_move (vm, s, offset);
1094 
1095  for (i = 1; i < n_buffers; i++)
1096  {
1097  vlib_buffer_t *d;
1098  d = vlib_buffer_copy (vm, s);
1099  if (d == 0)
1100  return i;
1101  buffers[i] = vlib_get_buffer_index (vm, d);
1102 
1103  }
1104  return n_buffers;
1105  }
1106 
1107  if (PREDICT_FALSE ((n_buffers == 1) && (offset == 0)))
1108  {
1109  buffers[0] = src_buffer;
1110  return 1;
1111  }
1112 
1113  n_buffers = vlib_buffer_alloc_from_pool (vm, buffers, n_buffers,
1114  s->buffer_pool_index);
1115 
1116  for (i = 0; i < n_buffers; i++)
1117  {
1118  vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
1119  if (offset)
1120  d->current_data = offset;
1121  else
1122  d->current_data = s->current_data;
1123 
1124  d->current_length = head_end_offset;
1126 
1128  head_end_offset;
1129  if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
1130  {
1133  }
1134  d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
1135  d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
1136  clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
1137  clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
1139  vlib_buffer_get_current (s), head_end_offset);
1140  d->next_buffer = src_buffer;
1141  }
1142  vlib_buffer_advance (s, head_end_offset);
1143  s->ref_count = n_buffers;
1144  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
1145  {
1146  s = vlib_get_buffer (vm, s->next_buffer);
1147  s->ref_count = n_buffers;
1148  }
1149 
1150  return n_buffers;
1151 }
1152 
1153 /** \brief Create multiple clones of buffer and store them
1154  in the supplied array
1155 
1156  @param vm - (vlib_main_t *) vlib main data structure pointer
1157  @param src_buffer - (u32) source buffer index
1158  @param buffers - (u32 * ) buffer index array
1159  @param n_buffers - (u16) number of buffer clones requested (<=256)
1160  @param head_end_offset - (u16) offset relative to current position
1161  where packet head ends
1162  @param offset - (i16) copy packet head at current position if 0,
1163  else at offset position to change headroom space as specified
1164  @return - (u16) number of buffers actually cloned, may be
1165  less than the number requested or zero
1166 */
1168 vlib_buffer_clone_at_offset (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1169  u16 n_buffers, u16 head_end_offset, i16 offset)
1170 {
1171  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
1172  u16 n_cloned = 0;
1173 
1174  while (n_buffers > 256)
1175  {
1176  vlib_buffer_t *copy;
1177  copy = vlib_buffer_copy (vm, s);
1178  n_cloned += vlib_buffer_clone_256 (vm,
1179  vlib_get_buffer_index (vm, copy),
1180  (buffers + n_cloned),
1181  256, head_end_offset, offset);
1182  n_buffers -= 256;
1183  }
1184  n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
1185  buffers + n_cloned,
1186  n_buffers, head_end_offset, offset);
1187 
1188  return n_cloned;
1189 }
1190 
1191 /** \brief Create multiple clones of buffer and store them
1192  in the supplied array
1193 
1194  @param vm - (vlib_main_t *) vlib main data structure pointer
1195  @param src_buffer - (u32) source buffer index
1196  @param buffers - (u32 * ) buffer index array
1197  @param n_buffers - (u16) number of buffer clones requested (<=256)
1198  @param head_end_offset - (u16) offset relative to current position
1199  where packet head ends
1200  @return - (u16) number of buffers actually cloned, may be
1201  less than the number requested or zero
1202 */
1204 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
1205  u16 n_buffers, u16 head_end_offset)
1206 {
1207  return vlib_buffer_clone_at_offset (vm, src_buffer, buffers, n_buffers,
1208  head_end_offset, 0);
1209 }
1210 
1211 /** \brief Attach cloned tail to the buffer
1212 
1213  @param vm - (vlib_main_t *) vlib main data structure pointer
1214  @param head - (vlib_buffer_t *) head buffer
1215  @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
1216 */
1217 
1218 always_inline void
1220  vlib_buffer_t * tail)
1221 {
1222  ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1223  ASSERT (head->buffer_pool_index == tail->buffer_pool_index);
1224 
1225  head->flags |= VLIB_BUFFER_NEXT_PRESENT;
1226  head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1227  head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
1228  head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
1229  head->next_buffer = vlib_get_buffer_index (vm, tail);
1232 
1233 next_segment:
1234  clib_atomic_add_fetch (&tail->ref_count, 1);
1235 
1236  if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
1237  {
1238  tail = vlib_get_buffer (vm, tail->next_buffer);
1239  goto next_segment;
1240  }
1241 }
1242 
1243 /* Initializes the buffer as an empty packet with no chained buffers. */
1244 always_inline void
1246 {
1248  first->current_length = 0;
1249  first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1250  first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1251 }
1252 
1253 /* The provided next_bi buffer index is appended to the end of the packet. */
1256 {
1257  vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
1258  last->next_buffer = next_bi;
1259  last->flags |= VLIB_BUFFER_NEXT_PRESENT;
1260  next_buffer->current_length = 0;
1261  next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1262  return next_buffer;
1263 }
1264 
1265 /* Increases or decreases the packet length.
1266  * It does not allocate or deallocate new buffers.
1267  * Therefore, the added length must be compatible
1268  * with the last buffer. */
1269 always_inline void
1271  vlib_buffer_t * last, i32 len)
1272 {
1273  last->current_length += len;
1274  if (first != last)
1276 }
1277 
1278 /* Copy data to the end of the packet and increases its length.
1279  * It does not allocate new buffers.
1280  * Returns the number of copied bytes. */
1283  vlib_buffer_t * first,
1284  vlib_buffer_t * last, void *data, u16 data_len)
1285 {
1286  u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm);
1287  ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
1288  u16 len = clib_min (data_len,
1289  n_buffer_bytes - last->current_length -
1290  last->current_data);
1292  data, len);
1293  vlib_buffer_chain_increase_length (first, last, len);
1294  return len;
1295 }
1296 
1297 /* Copy data to the end of the packet and increases its length.
1298  * Allocates additional buffers from the free list if necessary.
1299  * Returns the number of copied bytes.
1300  * 'last' value is modified whenever new buffers are allocated and
1301  * chained and points to the last buffer in the chain. */
1302 u16
1304  vlib_buffer_t * first,
1305  vlib_buffer_t ** last, void *data,
1306  u16 data_len);
1308 
1311 
1312 typedef struct
1313 {
1314  /* Vector of packet data. */
1316 
1317  /* Number of buffers to allocate in each call to allocator. */
1319 
1322 
1325  void *packet_data,
1326  uword n_packet_data_bytes,
1327  uword min_n_buffers_each_alloc,
1328  char *fmt, ...);
1329 
1332  u32 * bi_result);
1333 
1334 always_inline void
1336 {
1337  vec_free (t->packet_data);
1338 }
1339 
1342 {
1343  return b->data + vlib_buffer_get_default_data_size (vm) -
1344  ((u8 *) vlib_buffer_get_current (b) + b->current_length);
1345 }
1346 
1349 {
1350  vlib_buffer_t *db = b, *sb, *first = b;
1351  int is_cloned = 0;
1352  u32 bytes_left = 0, data_size;
1353  u16 src_left, dst_left, n_buffers = 1;
1354  u8 *dp, *sp;
1355  u32 to_free = 0;
1356 
1357  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
1358  return 1;
1359 
1360  data_size = vlib_buffer_get_default_data_size (vm);
1361 
1362  dst_left = vlib_buffer_space_left_at_end (vm, b);
1363 
1364  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1365  {
1366  b = vlib_get_buffer (vm, b->next_buffer);
1367  if (b->ref_count > 1)
1368  is_cloned = 1;
1369  bytes_left += b->current_length;
1370  n_buffers++;
1371  }
1372 
1373  /* if buffer is cloned, create completely new chain - unless everything fits
1374  * into one buffer */
1375  if (is_cloned && bytes_left >= dst_left)
1376  {
1377  u32 len = 0;
1378  u32 space_needed = bytes_left - dst_left;
1379  u32 tail;
1380 
1381  if (vlib_buffer_alloc (vm, &tail, 1) == 0)
1382  return 0;
1383 
1384  ++n_buffers;
1385  len += data_size;
1386  b = vlib_get_buffer (vm, tail);
1387 
1388  while (len < space_needed)
1389  {
1390  u32 bi;
1391  if (vlib_buffer_alloc (vm, &bi, 1) == 0)
1392  {
1393  vlib_buffer_free_one (vm, tail);
1394  return 0;
1395  }
1396  b->flags = VLIB_BUFFER_NEXT_PRESENT;
1397  b->next_buffer = bi;
1398  b = vlib_get_buffer (vm, bi);
1399  len += data_size;
1400  n_buffers++;
1401  }
1402  sb = vlib_get_buffer (vm, first->next_buffer);
1403  to_free = first->next_buffer;
1404  first->next_buffer = tail;
1405  }
1406  else
1407  sb = vlib_get_buffer (vm, first->next_buffer);
1408 
1409  src_left = sb->current_length;
1410  sp = vlib_buffer_get_current (sb);
1411  dp = vlib_buffer_get_tail (db);
1412 
1413  while (bytes_left)
1414  {
1415  u16 bytes_to_copy;
1416 
1417  if (dst_left == 0)
1418  {
1419  if (db != first)
1420  db->current_data = 0;
1421  db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1422  ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT);
1423  db = vlib_get_buffer (vm, db->next_buffer);
1424  dst_left = data_size;
1425  dp = db->data;
1426  }
1427 
1428  while (src_left == 0)
1429  {
1430  ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT);
1431  sb = vlib_get_buffer (vm, sb->next_buffer);
1432  src_left = sb->current_length;
1433  sp = vlib_buffer_get_current (sb);
1434  }
1435 
1436  bytes_to_copy = clib_min (dst_left, src_left);
1437 
1438  if (dp != sp)
1439  {
1440  if (sb == db)
1441  bytes_to_copy = clib_min (bytes_to_copy, sp - dp);
1442 
1443  clib_memcpy_fast (dp, sp, bytes_to_copy);
1444  }
1445 
1446  src_left -= bytes_to_copy;
1447  dst_left -= bytes_to_copy;
1448  dp += bytes_to_copy;
1449  sp += bytes_to_copy;
1450  bytes_left -= bytes_to_copy;
1451  }
1452  if (db != first)
1453  db->current_data = 0;
1454  db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1455 
1456  if (is_cloned && to_free)
1457  vlib_buffer_free_one (vm, to_free);
1458  else
1459  {
1460  if (db->flags & VLIB_BUFFER_NEXT_PRESENT)
1462  db->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1463  b = first;
1464  n_buffers = 1;
1465  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1466  {
1467  b = vlib_get_buffer (vm, b->next_buffer);
1468  ++n_buffers;
1469  }
1470  }
1471 
1472  first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1473 
1474  return n_buffers;
1475 }
1476 
1477 #endif /* included_vlib_buffer_funcs_h */
1478 
1479 /*
1480  * fd.io coding-style-patch-verification: ON
1481  *
1482  * Local Variables:
1483  * eval: (c-set-style "gnu")
1484  * End:
1485  */
u32 opaque2[14]
Definition: buffer.h:170
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:427
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:310
u32 flags
Definition: vhost_user.h:115
#define clib_min(x, y)
Definition: clib.h:295
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
Definition: buffer.c:78
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
#define clib_atomic_add_fetch(a, b)
Definition: atomics.h:30
format_function_t format_vlib_buffer_contents
static uword vlib_buffer_get_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:421
#define PREDICT_TRUE(x)
Definition: clib.h:112
static_always_inline u64x2 u32x4_extend_to_u64x2_high(u32x4 v)
Definition: vector_neon.h:117
format_function_t format_vlib_buffer
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
static_always_inline vlib_buffer_pool_t * vlib_get_buffer_pool(vlib_main_t *vm, u8 buffer_pool_index)
Definition: buffer_funcs.h:478
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
Definition: buffer.c:465
static void vlib_buffer_attach_clone(vlib_main_t *vm, vlib_buffer_t *head, vlib_buffer_t *tail)
Attach cloned tail to the buffer.
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
static void vlib_buffer_move(vlib_main_t *vm, vlib_buffer_t *b, i16 offset)
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
int vlib_buffer_add_data(vlib_main_t *vm, u32 *buffer_index, void *data, u32 n_data_bytes)
Definition: buffer.c:410
u32 thread_index
Definition: main.h:197
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:50
u8 data[0]
Packet data.
Definition: buffer.h:181
int i
vlib_buffer_main_t * buffer_main
Definition: main.h:129
foreach_avx2_vec256i static foreach_avx2_vec256u u32x8 u32x8_permute(u32x8 v, u32x8 idx)
Definition: vector_avx2.h:73
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
Definition: buffer_funcs.h:178
u8 data[128]
Definition: ipsec.api:248
u32 numa_node
Definition: main.h:199
vl_api_ip4_address_t dst
Definition: ipsec_gre.api:39
u8 *( format_function_t)(u8 *s, va_list *args)
Definition: format.h:48
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
static vlib_buffer_known_state_t vlib_buffer_is_known(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:462
static u32 vlib_buffer_alloc_to_ring_from_pool(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers into ring from specific buffer pool.
Definition: buffer_funcs.h:657
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:366
static u32 vlib_buffer_alloc_on_numa(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 numa_node)
Allocate buffers from specific numa node into supplied array.
Definition: buffer_funcs.h:595
unsigned char u8
Definition: types.h:56
static vlib_buffer_t * vlib_buffer_copy(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:959
u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES]
Definition: buffer.h:421
STATIC_ASSERT_OFFSET_OF(vlib_buffer_t, template_end, 64)
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
static_always_inline void vlib_get_buffer_indices(vlib_main_t *vm, vlib_buffer_t **b, u32 *bi, uword count)
Translate array of buffer pointers into buffer indices.
Definition: buffer_funcs.h:337
vlib_buffer_pool_thread_t * threads
Definition: buffer.h:398
#define static_always_inline
Definition: clib.h:99
#define always_inline
Definition: clib.h:98
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
void di(unformat_input_t *i)
Definition: unformat.c:163
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:376
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:257
static u16 vlib_buffer_chain_append_data(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, void *data, u16 data_len)
epu8_epi32 epu16_epi32 u64x2
Definition: vector_sse42.h:665
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:912
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
static u32 vlib_buffer_chain_linearize(vlib_main_t *vm, vlib_buffer_t *b)
#define hash_get(h, key)
Definition: hash.h:249
static vlib_buffer_t * vlib_buffer_copy_no_chain(vlib_main_t *vm, vlib_buffer_t *b, u32 *di)
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 buffer_index, uword follow_chain)
Definition: buffer.c:243
static_always_inline void vlib_buffer_copy_template(vlib_buffer_t *b, vlib_buffer_t *bt)
Definition: buffer_funcs.h:145
format_function_t format_vlib_buffer_and_data
unsigned short u16
Definition: types.h:57
static void vlib_buffer_validate(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:55
vlib_buffer_t buffer_template
Definition: buffer.h:401
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
static_always_inline uword vlib_buffer_pool_get(vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers, u32 n_buffers)
Definition: buffer_funcs.h:485
static_always_inline u32x4 u32x4_shuffle(u32x4 v, const int a, const int b, const int c, const int d)
Definition: vector_sse42.h:642
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:413
#define PREDICT_FALSE(x)
Definition: clib.h:111
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
Definition: buffer_funcs.h:276
vl_api_ip4_address_t src
Definition: ipsec_gre.api:38
static uword vlib_buffer_contents(vlib_main_t *vm, u32 buffer_index, u8 *contents)
Copy buffer contents to memory.
Definition: buffer_funcs.h:400
u8 len
Definition: ip_types.api:49
static void vlib_buffer_chain_init(vlib_buffer_t *first)
#define clib_atomic_sub_fetch(a, b)
Definition: atomics.h:31
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:96
#define vec_add_aligned(V, E, N, A)
Add N elements to end of vector V (no header, specified alignment)
Definition: vec.h:608
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:420
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
Definition: buffer_funcs.h:882
static u16 vlib_buffer_clone(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset)
Create multiple clones of buffer and store them in the supplied array.
static_always_inline void vlib_buffer_pool_put(vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers, u32 n_buffers)
Definition: buffer_funcs.h:680
void vlib_buffer_chain_validate(vlib_main_t *vm, vlib_buffer_t *first)
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
#define VLIB_BUFFER_MAX_NUMA_NODES
Definition: buffer.h:404
clib_spinlock_t lock
Definition: buffer.h:395
static vlib_buffer_t * vlib_get_next_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Get next buffer in buffer linklist, or zero for end of list.
Definition: buffer_funcs.h:350
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
Definition: buffer.c:327
static u32 vlib_buffer_space_left_at_end(vlib_main_t *vm, vlib_buffer_t *b)
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:241
vlib_buffer_known_state_t
Definition: buffer_funcs.h:446
static_always_inline u64x2 u32x4_extend_to_u64x2(u32x4 v)
Definition: vector_neon.h:111
signed int i32
Definition: types.h:77
static u32 vlib_buffer_alloc_from_pool(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers from specific pool into supplied array.
Definition: buffer_funcs.h:523
#define uword_to_pointer(u, type)
Definition: types.h:136
vlib buffer structure definition and a few select access methods.
#define ASSERT(truth)
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:784
static u16 vlib_buffer_clone_at_offset(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset, i16 offset)
Create multiple clones of buffer and store them in the supplied array.
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
size_t count
Definition: vapi.c:47
static uword vlib_buffer_index_length_in_chain(vlib_main_t *vm, u32 bi)
Get length in bytes of the buffer index buffer chain.
Definition: buffer_funcs.h:386
static void * vlib_buffer_ptr_from_index(uword buffer_mem_start, u32 buffer_index, uword offset)
Definition: buffer_funcs.h:71
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, int maybe_next)
Definition: buffer_funcs.h:707
static uword pointer_to_uword(const void *p)
Definition: types.h:131
template key/value backing page structure
Definition: bihash_doc.h:44
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
static u32 vlib_buffer_alloc_to_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Allocate buffers into ring.
Definition: buffer_funcs.h:628
static_always_inline void vlib_buffer_copy_indices(u32 *dst, u32 *src, u32 n_indices)
Definition: buffer_funcs.h:102
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *last, u32 next_bi)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:451
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
uword buffer_mem_start
Definition: buffer.h:411
uword * buffer_known_hash
Definition: buffer.h:419
u32 default_data_size
Definition: buffer.h:426
struct clib_bihash_value offset
template key/value backing page structure
u64x4
Definition: vector_avx2.h:121
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
Definition: buffer.c:389
static u16 vlib_buffer_clone_256(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset, i16 offset)
Create a maximum of 256 clones of buffer and store them in the supplied array.
unsigned long long u32x4
Definition: ixge.c:28
static void vlib_packet_template_free(vlib_main_t *vm, vlib_packet_template_t *t)
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:898
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
Definition: buffer.c:367
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:612
static u8 vlib_buffer_pool_get_default_for_numa(vlib_main_t *vm, u32 numa_node)
Definition: buffer_funcs.h:163
static void vlib_buffer_free_from_ring_no_next(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring without freeing tail buffers.
Definition: buffer_funcs.h:937
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:153
signed short i16
Definition: types.h:46