FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
buffer_funcs.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer_funcs.h: VLIB buffer related functions/inlines
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_funcs_h
41 #define included_vlib_buffer_funcs_h
42 
43 #include <vppinfra/hash.h>
44 #include <vppinfra/fifo.h>
45 #include <vlib/buffer.h>
46 #include <vlib/physmem_funcs.h>
47 #include <vlib/main.h>
48 #include <vlib/node.h>
49 
50 /** \file
51  vlib buffer access methods.
52 */
53 
54 
55 /** \brief Translate buffer index into buffer pointer
56 
57  @param vm - (vlib_main_t *) vlib main data structure pointer
58  @param buffer_index - (u32) buffer index
59  @return - (vlib_buffer_t *) buffer pointer
60 */
62 vlib_get_buffer (vlib_main_t * vm, u32 buffer_index)
63 {
65  uword offset = ((uword) buffer_index) << CLIB_LOG2_CACHE_LINE_BYTES;
66  ASSERT (offset < bm->buffer_mem_size);
67 
68  return uword_to_pointer (bm->buffer_mem_start + offset, void *);
69 }
70 
71 /** \brief Translate array of buffer indices into buffer pointers with offset
72 
73  @param vm - (vlib_main_t *) vlib main data structure pointer
74  @param bi - (u32 *) array of buffer indices
75  @param b - (void **) array to store buffer pointers
76  @param count - (uword) number of elements
77  @param offset - (i32) offset applied to each pointer
78 */
81  i32 offset)
82 {
83 #ifdef CLIB_HAVE_VEC256
84  u64x4 off = u64x4_splat (buffer_main.buffer_mem_start + offset);
85  /* if count is not const, compiler will not unroll while loop
86  se we maintain two-in-parallel variant */
87  while (count >= 8)
88  {
89  u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
90  u64x4 b1 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi + 4));
91  /* shift and add to get vlib_buffer_t pointer */
92  u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
93  u64x4_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 4);
94  b += 8;
95  bi += 8;
96  count -= 8;
97  }
98 #endif
99  while (count >= 4)
100  {
101 #ifdef CLIB_HAVE_VEC256
102  u64x4 b0 = u32x4_extend_to_u64x4 (u32x4_load_unaligned (bi));
103  /* shift and add to get vlib_buffer_t pointer */
104  u64x4_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
105 #elif defined (CLIB_HAVE_VEC128)
106  u64x2 off = u64x2_splat (buffer_main.buffer_mem_start + offset);
107  u32x4 bi4 = u32x4_load_unaligned (bi);
108  u64x2 b0 = u32x4_extend_to_u64x2 ((u32x4) bi4);
109 #if defined (__aarch64__)
111 #else
112  bi4 = u32x4_shuffle (bi4, 2, 3, 0, 1);
113  u64x2 b1 = u32x4_extend_to_u64x2 ((u32x4) bi4);
114 #endif
115  u64x2_store_unaligned ((b0 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b);
116  u64x2_store_unaligned ((b1 << CLIB_LOG2_CACHE_LINE_BYTES) + off, b + 2);
117 #else
118  b[0] = ((u8 *) vlib_get_buffer (vm, bi[0])) + offset;
119  b[1] = ((u8 *) vlib_get_buffer (vm, bi[1])) + offset;
120  b[2] = ((u8 *) vlib_get_buffer (vm, bi[2])) + offset;
121  b[3] = ((u8 *) vlib_get_buffer (vm, bi[3])) + offset;
122 #endif
123  b += 4;
124  bi += 4;
125  count -= 4;
126  }
127  while (count)
128  {
129  b[0] = ((u8 *) vlib_get_buffer (vm, bi[0])) + offset;
130  b += 1;
131  bi += 1;
132  count -= 1;
133  }
134 }
135 
136 /** \brief Translate array of buffer indices into buffer pointers
137 
138  @param vm - (vlib_main_t *) vlib main data structure pointer
139  @param bi - (u32 *) array of buffer indices
140  @param b - (vlib_buffer_t **) array to store buffer pointers
141  @param count - (uword) number of elements
142 */
143 
146 {
147  vlib_get_buffers_with_offset (vm, bi, (void **) b, count, 0);
148 }
149 
150 /** \brief Translate buffer pointer into buffer index
151 
152  @param vm - (vlib_main_t *) vlib main data structure pointer
153  @param p - (void *) buffer pointer
154  @return - (u32) buffer index
155 */
156 
159 {
163  ASSERT (offset < bm->buffer_mem_size);
164  ASSERT ((offset % (1 << CLIB_LOG2_CACHE_LINE_BYTES)) == 0);
165  return offset >> CLIB_LOG2_CACHE_LINE_BYTES;
166 }
167 
168 /** \brief Translate array of buffer pointers into buffer indices with offset
169 
170  @param vm - (vlib_main_t *) vlib main data structure pointer
171  @param b - (void **) array of buffer pointers
172  @param bi - (u32 *) array to store buffer indices
173  @param count - (uword) number of elements
174  @param offset - (i32) offset applied to each pointer
175 */
179 {
180 #ifdef CLIB_HAVE_VEC256
181  u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
182  u64x4 off4 = u64x4_splat (buffer_main.buffer_mem_start - offset);
183 
184  while (count >= 8)
185  {
186  /* load 4 pointers into 256-bit register */
187  u64x4 v0 = u64x4_load_unaligned (b);
188  u64x4 v1 = u64x4_load_unaligned (b + 4);
189  u32x8 v2, v3;
190 
191  v0 -= off4;
192  v1 -= off4;
193 
196 
197  /* permute 256-bit register so lower u32s of each buffer index are
198  * placed into lower 128-bits */
199  v2 = u32x8_permute ((u32x8) v0, mask);
200  v3 = u32x8_permute ((u32x8) v1, mask);
201 
202  /* extract lower 128-bits and save them to the array of buffer indices */
203  u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
204  u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
205  bi += 8;
206  b += 8;
207  count -= 8;
208  }
209 #endif
210  while (count >= 4)
211  {
212  /* equivalent non-nector implementation */
213  bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
214  bi[1] = vlib_get_buffer_index (vm, ((u8 *) b[1]) + offset);
215  bi[2] = vlib_get_buffer_index (vm, ((u8 *) b[2]) + offset);
216  bi[3] = vlib_get_buffer_index (vm, ((u8 *) b[3]) + offset);
217  bi += 4;
218  b += 4;
219  count -= 4;
220  }
221  while (count)
222  {
223  bi[0] = vlib_get_buffer_index (vm, ((u8 *) b[0]) + offset);
224  bi += 1;
225  b += 1;
226  count -= 1;
227  }
228 }
229 
230 /** \brief Translate array of buffer pointers into buffer indices
231 
232  @param vm - (vlib_main_t *) vlib main data structure pointer
233  @param b - (vlib_buffer_t **) array of buffer pointers
234  @param bi - (u32 *) array to store buffer indices
235  @param count - (uword) number of elements
236 */
239  uword count)
240 {
241  vlib_get_buffer_indices_with_offset (vm, (void **) b, bi, count, 0);
242 }
243 
244 /** \brief Get next buffer in buffer linklist, or zero for end of list.
245 
246  @param vm - (vlib_main_t *) vlib main data structure pointer
247  @param b - (void *) buffer pointer
248  @return - (vlib_buffer_t *) next buffer, or NULL
249 */
252 {
253  return (b->flags & VLIB_BUFFER_NEXT_PRESENT
254  ? vlib_get_buffer (vm, b->next_buffer) : 0);
255 }
256 
258  vlib_buffer_t * b_first);
259 
260 /** \brief Get length in bytes of the buffer chain
261 
262  @param vm - (vlib_main_t *) vlib main data structure pointer
263  @param b - (void *) buffer pointer
264  @return - (uword) length of buffer chain
265 */
268 {
269  uword len = b->current_length;
270 
271  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
272  return len;
273 
274  if (PREDICT_TRUE (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID))
276 
278 }
279 
280 /** \brief Get length in bytes of the buffer index buffer chain
281 
282  @param vm - (vlib_main_t *) vlib main data structure pointer
283  @param bi - (u32) buffer index
284  @return - (uword) length of buffer chain
285 */
288 {
289  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
290  return vlib_buffer_length_in_chain (vm, b);
291 }
292 
293 /** \brief Copy buffer contents to memory
294 
295  @param vm - (vlib_main_t *) vlib main data structure pointer
296  @param buffer_index - (u32) buffer index
297  @param contents - (u8 *) memory, <strong>must be large enough</strong>
298  @return - (uword) length of buffer chain
299 */
301 vlib_buffer_contents (vlib_main_t * vm, u32 buffer_index, u8 * contents)
302 {
303  uword content_len = 0;
304  uword l;
305  vlib_buffer_t *b;
306 
307  while (1)
308  {
309  b = vlib_get_buffer (vm, buffer_index);
310  l = b->current_length;
311  clib_memcpy_fast (contents + content_len, b->data + b->current_data, l);
312  content_len += l;
313  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
314  break;
315  buffer_index = b->next_buffer;
316  }
317 
318  return content_len;
319 }
320 
323 {
324  return vlib_physmem_get_pa (vm, b->data);
325 }
326 
329 {
330  return vlib_buffer_get_pa (vm, b) + b->current_data;
331 }
332 
333 /** \brief Prefetch buffer metadata by buffer index
334  The first 64 bytes of buffer contains most header information
335 
336  @param vm - (vlib_main_t *) vlib main data structure pointer
337  @param bi - (u32) buffer index
338  @param type - LOAD, STORE. In most cases, STORE is the right answer
339 */
340 /* Prefetch buffer header given index. */
341 #define vlib_prefetch_buffer_with_index(vm,bi,type) \
342  do { \
343  vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \
344  vlib_prefetch_buffer_header (_b, type); \
345  } while (0)
346 
347 typedef enum
348 {
349  /* Index is unknown. */
351 
352  /* Index is known and free/allocated. */
356 
357 void vlib_buffer_validate_alloc_free (vlib_main_t * vm, u32 * buffers,
358  uword n_buffers,
360  expected_state);
361 
363 vlib_buffer_is_known (u32 buffer_index)
364 {
366 
368  uword *p = hash_get (bm->buffer_known_hash, buffer_index);
370  return p ? p[0] : VLIB_BUFFER_UNKNOWN;
371 }
372 
373 always_inline void
376 {
378 
380  hash_set (bm->buffer_known_hash, buffer_index, state);
382 }
383 
384 /* Validates sanity of a single buffer.
385  Returns format'ed vector with error message if any. */
386 u8 *vlib_validate_buffer (vlib_main_t * vm, u32 buffer_index,
387  uword follow_chain);
388 
391 {
392  return round_pow2 (size, sizeof (vlib_buffer_t));
393 }
394 
397 {
398  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NON_DEFAULT_FREELIST))
399  return b->free_list_index;
400 
401  return 0;
402 }
403 
404 always_inline void
407 {
408  if (PREDICT_FALSE (index))
409  {
410  b->flags |= VLIB_BUFFER_NON_DEFAULT_FREELIST;
411  b->free_list_index = index;
412  }
413  else
414  b->flags &= ~VLIB_BUFFER_NON_DEFAULT_FREELIST;
415 }
416 
417 /** \brief Allocate buffers from specific freelist into supplied array
418 
419  @param vm - (vlib_main_t *) vlib main data structure pointer
420  @param buffers - (u32 * ) buffer index array
421  @param n_buffers - (u32) number of buffers requested
422  @return - (u32) number of buffers actually allocated, may be
423  less than the number requested or zero
424 */
427  u32 * buffers,
428  u32 n_buffers,
430 {
433  u32 *src;
434  uword len;
435 
437 
438  fl = pool_elt_at_index (vm->buffer_free_list_pool, index);
439 
440  len = vec_len (fl->buffers);
441 
442  if (PREDICT_FALSE (len < n_buffers))
443  {
444  bm->cb.vlib_buffer_fill_free_list_cb (vm, fl, n_buffers);
445  if (PREDICT_FALSE ((len = vec_len (fl->buffers)) == 0))
446  return 0;
447 
448  /* even if fill free list didn't manage to refill free list
449  we should give what we have */
450  n_buffers = clib_min (len, n_buffers);
451 
452  /* following code is intentionaly duplicated to allow compiler
453  to optimize fast path when n_buffers is constant value */
454  src = fl->buffers + len - n_buffers;
455  clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32));
456  _vec_len (fl->buffers) -= n_buffers;
457 
458  /* Verify that buffers are known free. */
459  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
461 
462  return n_buffers;
463  }
464 
465  src = fl->buffers + len - n_buffers;
466  clib_memcpy_fast (buffers, src, n_buffers * sizeof (u32));
467  _vec_len (fl->buffers) -= n_buffers;
468 
469  /* Verify that buffers are known free. */
470  vlib_buffer_validate_alloc_free (vm, buffers, n_buffers,
472 
473  return n_buffers;
474 }
475 
476 /** \brief Allocate buffers into supplied array
477 
478  @param vm - (vlib_main_t *) vlib main data structure pointer
479  @param buffers - (u32 * ) buffer index array
480  @param n_buffers - (u32) number of buffers requested
481  @return - (u32) number of buffers actually allocated, may be
482  less than the number requested or zero
483 */
485 vlib_buffer_alloc (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
486 {
487  return vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
489 }
490 
491 /** \brief Allocate buffers into ring
492 
493  @param vm - (vlib_main_t *) vlib main data structure pointer
494  @param buffers - (u32 * ) buffer index ring
495  @param start - (u32) first slot in the ring
496  @param ring_size - (u32) ring size
497  @param n_buffers - (u32) number of buffers requested
498  @return - (u32) number of buffers actually allocated, may be
499  less than the number requested or zero
500 */
503  u32 ring_size, u32 n_buffers)
504 {
505  u32 n_alloc;
506 
507  ASSERT (n_buffers <= ring_size);
508 
509  if (PREDICT_TRUE (start + n_buffers <= ring_size))
510  return vlib_buffer_alloc (vm, ring + start, n_buffers);
511 
512  n_alloc = vlib_buffer_alloc (vm, ring + start, ring_size - start);
513 
514  if (PREDICT_TRUE (n_alloc == ring_size - start))
515  n_alloc += vlib_buffer_alloc (vm, ring, n_buffers - n_alloc);
516 
517  return n_alloc;
518 }
519 
520 /** \brief Free buffers
521  Frees the entire buffer chain for each buffer
522 
523  @param vm - (vlib_main_t *) vlib main data structure pointer
524  @param buffers - (u32 * ) buffer index array
525  @param n_buffers - (u32) number of buffers to free
526 
527 */
528 always_inline void
530  /* pointer to first buffer */
531  u32 * buffers,
532  /* number of buffers to free */
533  u32 n_buffers)
534 {
536 
538 
539  return bm->cb.vlib_buffer_free_cb (vm, buffers, n_buffers);
540 }
541 
542 /** \brief Free buffers, does not free the buffer chain for each buffer
543 
544  @param vm - (vlib_main_t *) vlib main data structure pointer
545  @param buffers - (u32 * ) buffer index array
546  @param n_buffers - (u32) number of buffers to free
547 
548 */
549 always_inline void
551  /* pointer to first buffer */
552  u32 * buffers,
553  /* number of buffers to free */
554  u32 n_buffers)
555 {
557 
559 
560  return bm->cb.vlib_buffer_free_no_next_cb (vm, buffers, n_buffers);
561 }
562 
563 /** \brief Free one buffer
564  Shorthand to free a single buffer chain.
565 
566  @param vm - (vlib_main_t *) vlib main data structure pointer
567  @param buffer_index - (u32) buffer index to free
568 */
569 always_inline void
571 {
572  vlib_buffer_free (vm, &buffer_index, /* n_buffers */ 1);
573 }
574 
575 /** \brief Free buffers from ring
576 
577  @param vm - (vlib_main_t *) vlib main data structure pointer
578  @param buffers - (u32 * ) buffer index ring
579  @param start - (u32) first slot in the ring
580  @param ring_size - (u32) ring size
581  @param n_buffers - (u32) number of buffers
582 */
583 always_inline void
585  u32 ring_size, u32 n_buffers)
586 {
587  ASSERT (n_buffers <= ring_size);
588 
589  if (PREDICT_TRUE (start + n_buffers <= ring_size))
590  {
591  vlib_buffer_free (vm, ring + start, n_buffers);
592  }
593  else
594  {
595  vlib_buffer_free (vm, ring + start, ring_size - start);
596  vlib_buffer_free (vm, ring, n_buffers - (ring_size - start));
597  }
598 }
599 
600 /** \brief Free buffers from ring without freeing tail buffers
601 
602  @param vm - (vlib_main_t *) vlib main data structure pointer
603  @param buffers - (u32 * ) buffer index ring
604  @param start - (u32) first slot in the ring
605  @param ring_size - (u32) ring size
606  @param n_buffers - (u32) number of buffers
607 */
608 always_inline void
610  u32 ring_size, u32 n_buffers)
611 {
612  ASSERT (n_buffers <= ring_size);
613 
614  if (PREDICT_TRUE (start + n_buffers <= ring_size))
615  {
616  vlib_buffer_free_no_next (vm, ring + start, n_buffers);
617  }
618  else
619  {
620  vlib_buffer_free_no_next (vm, ring + start, ring_size - start);
621  vlib_buffer_free_no_next (vm, ring, n_buffers - (ring_size - start));
622  }
623 }
624 
625 /* Add/delete buffer free lists. */
627  u32 n_data_bytes,
628  char *fmt, ...);
629 always_inline void
631  vlib_buffer_free_list_index_t free_list_index)
632 {
634 
636 
637  bm->cb.vlib_buffer_delete_free_list_cb (vm, free_list_index);
638 }
639 
640 /* Make sure we have at least given number of unaligned buffers. */
643  free_list,
644  uword n_unaligned_buffers);
645 
649 {
651 
652  *index = i = vlib_buffer_get_free_list_index (b);
654 }
655 
658  vlib_buffer_free_list_index_t free_list_index)
659 {
661 
662  f = pool_elt_at_index (vm->buffer_free_list_pool, free_list_index);
663 
664  /* Sanity: indices must match. */
665  ASSERT (f->index == free_list_index);
666 
667  return f;
668 }
669 
673 {
675  return f->n_data_bytes;
676 }
677 
678 /* Append given data to end of buffer, possibly allocating new buffers. */
680  vlib_buffer_free_list_index_t free_list_index,
681  u32 * buffer_index, void *data, u32 n_data_bytes);
682 
683 /* duplicate all buffers in chain */
686 {
687  vlib_buffer_t *s, *d, *fd;
688  uword n_alloc, n_buffers = 1;
689  u32 flag_mask = VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID;
690  int i;
691 
692  s = b;
693  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
694  {
695  n_buffers++;
696  s = vlib_get_buffer (vm, s->next_buffer);
697  }
698  u32 new_buffers[n_buffers];
699 
700  n_alloc = vlib_buffer_alloc (vm, new_buffers, n_buffers);
701 
702  /* No guarantee that we'll get all the buffers we asked for */
703  if (PREDICT_FALSE (n_alloc < n_buffers))
704  {
705  if (n_alloc > 0)
706  vlib_buffer_free (vm, new_buffers, n_alloc);
707  return 0;
708  }
709 
710  /* 1st segment */
711  s = b;
712  fd = d = vlib_get_buffer (vm, new_buffers[0]);
713  d->current_data = s->current_data;
715  d->flags = s->flags & flag_mask;
718  clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
719  clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
722 
723  /* next segments */
724  for (i = 1; i < n_buffers; i++)
725  {
726  /* previous */
727  d->next_buffer = new_buffers[i];
728  /* current */
729  s = vlib_get_buffer (vm, s->next_buffer);
730  d = vlib_get_buffer (vm, new_buffers[i]);
731  d->current_data = s->current_data;
735  d->flags = s->flags & flag_mask;
736  }
737 
738  return fd;
739 }
740 
741 /** \brief Create a maximum of 256 clones of buffer and store them
742  in the supplied array
743 
744  @param vm - (vlib_main_t *) vlib main data structure pointer
745  @param src_buffer - (u32) source buffer index
746  @param buffers - (u32 * ) buffer index array
747  @param n_buffers - (u16) number of buffer clones requested (<=256)
748  @param head_end_offset - (u16) offset relative to current position
749  where packet head ends
750  @return - (u16) number of buffers actually cloned, may be
751  less than the number requested or zero
752 */
754 vlib_buffer_clone_256 (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
755  u16 n_buffers, u16 head_end_offset)
756 {
757  u16 i;
758  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
759 
760  ASSERT (s->n_add_refs == 0);
761  ASSERT (n_buffers);
762  ASSERT (n_buffers <= 256);
763 
764  if (s->current_length <= head_end_offset + CLIB_CACHE_LINE_BYTES * 2)
765  {
766  buffers[0] = src_buffer;
767  for (i = 1; i < n_buffers; i++)
768  {
769  vlib_buffer_t *d;
770  d = vlib_buffer_copy (vm, s);
771  if (d == 0)
772  return i;
773  buffers[i] = vlib_get_buffer_index (vm, d);
774 
775  }
776  return n_buffers;
777  }
778 
779  if (PREDICT_FALSE (n_buffers == 1))
780  {
781  buffers[0] = src_buffer;
782  return 1;
783  }
784 
785  n_buffers = vlib_buffer_alloc_from_free_list (vm, buffers, n_buffers,
787  (s));
788 
789  for (i = 0; i < n_buffers; i++)
790  {
791  vlib_buffer_t *d = vlib_get_buffer (vm, buffers[i]);
792  d->current_data = s->current_data;
793  d->current_length = head_end_offset;
796 
798  head_end_offset;
799  if (PREDICT_FALSE (s->flags & VLIB_BUFFER_NEXT_PRESENT))
800  {
803  }
804  d->flags = s->flags | VLIB_BUFFER_NEXT_PRESENT;
805  d->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
806  clib_memcpy_fast (d->opaque, s->opaque, sizeof (s->opaque));
807  clib_memcpy_fast (d->opaque2, s->opaque2, sizeof (s->opaque2));
809  vlib_buffer_get_current (s), head_end_offset);
810  d->next_buffer = src_buffer;
811  }
812  vlib_buffer_advance (s, head_end_offset);
813  s->n_add_refs = n_buffers - 1;
814  while (s->flags & VLIB_BUFFER_NEXT_PRESENT)
815  {
816  s = vlib_get_buffer (vm, s->next_buffer);
817  s->n_add_refs = n_buffers - 1;
818  }
819 
820  return n_buffers;
821 }
822 
823 /** \brief Create multiple clones of buffer and store them
824  in the supplied array
825 
826  @param vm - (vlib_main_t *) vlib main data structure pointer
827  @param src_buffer - (u32) source buffer index
828  @param buffers - (u32 * ) buffer index array
829  @param n_buffers - (u16) number of buffer clones requested (<=256)
830  @param head_end_offset - (u16) offset relative to current position
831  where packet head ends
832  @return - (u16) number of buffers actually cloned, may be
833  less than the number requested or zero
834 */
836 vlib_buffer_clone (vlib_main_t * vm, u32 src_buffer, u32 * buffers,
837  u16 n_buffers, u16 head_end_offset)
838 {
839  vlib_buffer_t *s = vlib_get_buffer (vm, src_buffer);
840  u16 n_cloned = 0;
841 
842  while (n_buffers > 256)
843  {
844  vlib_buffer_t *copy;
845  copy = vlib_buffer_copy (vm, s);
846  n_cloned += vlib_buffer_clone_256 (vm,
847  vlib_get_buffer_index (vm, copy),
848  (buffers + n_cloned),
849  256, head_end_offset);
850  n_buffers -= 256;
851  }
852  n_cloned += vlib_buffer_clone_256 (vm, src_buffer,
853  buffers + n_cloned,
854  n_buffers, head_end_offset);
855 
856  return n_cloned;
857 }
858 
859 /** \brief Attach cloned tail to the buffer
860 
861  @param vm - (vlib_main_t *) vlib main data structure pointer
862  @param head - (vlib_buffer_t *) head buffer
863  @param tail - (Vlib buffer_t *) tail buffer to clone and attach to head
864 */
865 
866 always_inline void
868  vlib_buffer_t * tail)
869 {
870  ASSERT ((head->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
873 
874  head->flags |= VLIB_BUFFER_NEXT_PRESENT;
875  head->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
876  head->flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
877  head->flags |= (tail->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
878  head->next_buffer = vlib_get_buffer_index (vm, tail);
881 
882 next_segment:
883  clib_atomic_add_fetch (&tail->n_add_refs, 1);
884 
885  if (tail->flags & VLIB_BUFFER_NEXT_PRESENT)
886  {
887  tail = vlib_get_buffer (vm, tail->next_buffer);
888  goto next_segment;
889  }
890 }
891 
892 /* Initializes the buffer as an empty packet with no chained buffers. */
893 always_inline void
895 {
897  first->current_length = 0;
898  first->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
899  first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
900 }
901 
902 /* The provided next_bi buffer index is appended to the end of the packet. */
905 {
906  vlib_buffer_t *next_buffer = vlib_get_buffer (vm, next_bi);
907  last->next_buffer = next_bi;
908  last->flags |= VLIB_BUFFER_NEXT_PRESENT;
909  next_buffer->current_length = 0;
910  next_buffer->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
911  return next_buffer;
912 }
913 
914 /* Increases or decreases the packet length.
915  * It does not allocate or deallocate new buffers.
916  * Therefore, the added length must be compatible
917  * with the last buffer. */
918 always_inline void
920  vlib_buffer_t * last, i32 len)
921 {
922  last->current_length += len;
923  if (first != last)
925 }
926 
927 /* Copy data to the end of the packet and increases its length.
928  * It does not allocate new buffers.
929  * Returns the number of copied bytes. */
932  vlib_buffer_free_list_index_t free_list_index,
933  vlib_buffer_t * first,
934  vlib_buffer_t * last, void *data, u16 data_len)
935 {
936  u32 n_buffer_bytes =
937  vlib_buffer_free_list_buffer_size (vm, free_list_index);
938  ASSERT (n_buffer_bytes >= last->current_length + last->current_data);
939  u16 len = clib_min (data_len,
940  n_buffer_bytes - last->current_length -
941  last->current_data);
943  data, len);
944  vlib_buffer_chain_increase_length (first, last, len);
945  return len;
946 }
947 
948 /* Copy data to the end of the packet and increases its length.
949  * Allocates additional buffers from the free list if necessary.
950  * Returns the number of copied bytes.
951  * 'last' value is modified whenever new buffers are allocated and
952  * chained and points to the last buffer in the chain. */
953 u16
956  free_list_index,
957  vlib_buffer_t * first,
958  vlib_buffer_t ** last, void *data,
959  u16 data_len);
961 
964 
965 typedef struct
966 {
967  /* Vector of packet data. */
969 
970  /* Number of buffers to allocate in each call to allocator. */
972 
973  /* Buffer free list for this template. */
975 
977 
980 
983 
986  void *packet_data,
987  uword n_packet_data_bytes,
988  uword min_n_buffers_each_alloc,
989  char *fmt, ...);
990 
993  u32 * bi_result);
994 
995 always_inline void
997 {
998  vec_free (t->packet_data);
999 }
1000 
1001 /* Set a buffer quickly into "uninitialized" state. We want this to
1002  be extremely cheap and arrange for all fields that need to be
1003  initialized to be in the first 128 bits of the buffer. */
1004 always_inline void
1007 {
1009 
1010  /* Make sure vlib_buffer_t is cacheline aligned and sized */
1011  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline0) == 0);
1012  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline1) ==
1014  ASSERT (STRUCT_OFFSET_OF (vlib_buffer_t, cacheline2) ==
1015  CLIB_CACHE_LINE_BYTES * 2);
1016 
1017  /* Make sure buffer template is sane. */
1019 
1020  clib_memcpy_fast (STRUCT_MARK_PTR (dst, template_start),
1021  STRUCT_MARK_PTR (src, template_start),
1022  STRUCT_OFFSET_OF (vlib_buffer_t, template_end) -
1023  STRUCT_OFFSET_OF (vlib_buffer_t, template_start));
1024 
1025  /* Not in the first 16 octets. */
1026  dst->n_add_refs = src->n_add_refs;
1028 
1029  /* Make sure it really worked. */
1030 #define _(f) ASSERT (dst->f == src->f);
1031  _(current_data);
1032  _(current_length);
1033  _(flags);
1034 #undef _
1035  /* ASSERT (dst->total_length_not_including_first_buffer == 0); */
1036  /* total_length_not_including_first_buffer is not in the template anymore
1037  * so it may actually not zeroed for some buffers. One option is to
1038  * uncomment the line lower (comes at a cost), the other, is to just not
1039  * care */
1040  /* dst->total_length_not_including_first_buffer = 0; */
1041  ASSERT (dst->n_add_refs == 0);
1042 }
1043 
1044 always_inline void
1047  u32 buffer_index, u8 do_init)
1048 {
1050  vlib_buffer_t *b;
1051  b = vlib_get_buffer (vm, buffer_index);
1052  if (PREDICT_TRUE (do_init))
1054  vec_add1_aligned (f->buffers, buffer_index, CLIB_CACHE_LINE_BYTES);
1055 
1056  if (vec_len (f->buffers) > 4 * VLIB_FRAME_SIZE)
1057  {
1058  clib_spinlock_lock (&bp->lock);
1059  /* keep last stored buffers, as they are more likely hot in the cache */
1063  f->n_alloc -= VLIB_FRAME_SIZE;
1064  clib_spinlock_unlock (&bp->lock);
1065  }
1066 }
1067 
1068 #if CLIB_DEBUG > 0
1071 extern void *vlib_buffer_state_heap;
1072 #endif
1073 
1074 static inline void
1076 {
1077 #if CLIB_DEBUG > 0
1078  uword *p;
1079  void *oldheap;
1080 
1082 
1084  ;
1085 
1087 
1088  /* If we don't know about b, declare it to be in the expected state */
1089  if (!p)
1090  {
1092  goto out;
1093  }
1094 
1095  if (p[0] != expected)
1096  {
1097  void cj_stop (void);
1098  u32 bi;
1100 
1101  cj_stop ();
1102 
1103  bi = vlib_get_buffer_index (vm, b);
1104 
1105  clib_mem_set_heap (oldheap);
1106  clib_warning ("%.6f buffer %llx (%d): %s, not %s",
1107  vlib_time_now (vm), bi,
1108  p[0] ? "busy" : "free", expected ? "busy" : "free");
1109  os_panic ();
1110  }
1111 out:
1114  clib_mem_set_heap (oldheap);
1115 #endif
1116 }
1117 
1118 static inline void
1120 {
1121 #if CLIB_DEBUG > 0
1122  void *oldheap;
1123 
1125 
1127  ;
1128 
1130 
1133  clib_mem_set_heap (oldheap);
1134 #endif
1135 }
1136 
1139 {
1140  return b->data + VLIB_BUFFER_DATA_SIZE -
1141  ((u8 *) vlib_buffer_get_current (b) + b->current_length);
1142 }
1143 
1146 {
1147  vlib_buffer_t *db = b, *sb, *first = b;
1148  int is_cloned = 0;
1149  u32 bytes_left = 0, data_size;
1150  u16 src_left, dst_left, n_buffers = 1;
1151  u8 *dp, *sp;
1152  u32 to_free = 0;
1153 
1154  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
1155  return 1;
1156 
1157  data_size = VLIB_BUFFER_DATA_SIZE;
1158 
1159  dst_left = vlib_buffer_space_left_at_end (vm, b);
1160 
1161  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1162  {
1163  b = vlib_get_buffer (vm, b->next_buffer);
1164  if (b->n_add_refs > 0)
1165  is_cloned = 1;
1166  bytes_left += b->current_length;
1167  n_buffers++;
1168  }
1169 
1170  /* if buffer is cloned, create completely new chain - unless everything fits
1171  * into one buffer */
1172  if (is_cloned && bytes_left >= dst_left)
1173  {
1174  u32 len = 0;
1175  u32 space_needed = bytes_left - dst_left;
1176  u32 tail;
1177 
1178  if (vlib_buffer_alloc (vm, &tail, 1) == 0)
1179  return 0;
1180 
1181  ++n_buffers;
1182  len += data_size;
1183  b = vlib_get_buffer (vm, tail);
1184 
1185  while (len < space_needed)
1186  {
1187  u32 bi;
1188  if (vlib_buffer_alloc (vm, &bi, 1) == 0)
1189  {
1190  vlib_buffer_free_one (vm, tail);
1191  return 0;
1192  }
1193  b->flags = VLIB_BUFFER_NEXT_PRESENT;
1194  b->next_buffer = bi;
1195  b = vlib_get_buffer (vm, bi);
1196  len += data_size;
1197  n_buffers++;
1198  }
1199  sb = vlib_get_buffer (vm, first->next_buffer);
1200  to_free = first->next_buffer;
1201  first->next_buffer = tail;
1202  }
1203  else
1204  sb = vlib_get_buffer (vm, first->next_buffer);
1205 
1206  src_left = sb->current_length;
1207  sp = vlib_buffer_get_current (sb);
1208  dp = vlib_buffer_get_tail (db);
1209 
1210  while (bytes_left)
1211  {
1212  u16 bytes_to_copy;
1213 
1214  if (dst_left == 0)
1215  {
1216  if (db != first)
1217  db->current_data = 0;
1218  db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1219  ASSERT (db->flags & VLIB_BUFFER_NEXT_PRESENT);
1220  db = vlib_get_buffer (vm, db->next_buffer);
1221  dst_left = data_size;
1222  dp = db->data;
1223  }
1224 
1225  while (src_left == 0)
1226  {
1227  ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT);
1228  sb = vlib_get_buffer (vm, sb->next_buffer);
1229  src_left = sb->current_length;
1230  sp = vlib_buffer_get_current (sb);
1231  }
1232 
1233  bytes_to_copy = clib_min (dst_left, src_left);
1234 
1235  if (dp != sp)
1236  {
1237  if (sb == db)
1238  bytes_to_copy = clib_min (bytes_to_copy, sp - dp);
1239 
1240  clib_memcpy_fast (dp, sp, bytes_to_copy);
1241  }
1242 
1243  src_left -= bytes_to_copy;
1244  dst_left -= bytes_to_copy;
1245  dp += bytes_to_copy;
1246  sp += bytes_to_copy;
1247  bytes_left -= bytes_to_copy;
1248  }
1249  if (db != first)
1250  db->current_data = 0;
1251  db->current_length = dp - (u8 *) vlib_buffer_get_current (db);
1252 
1253  if (is_cloned && to_free)
1254  vlib_buffer_free_one (vm, to_free);
1255  else
1256  {
1257  if (db->flags & VLIB_BUFFER_NEXT_PRESENT)
1259  db->flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1260  b = first;
1261  n_buffers = 1;
1262  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
1263  {
1264  b = vlib_get_buffer (vm, b->next_buffer);
1265  ++n_buffers;
1266  }
1267  }
1268 
1269  first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1270 
1271  return n_buffers;
1272 }
1273 
1274 #endif /* included_vlib_buffer_funcs_h */
1275 
1276 /*
1277  * fd.io coding-style-patch-verification: ON
1278  *
1279  * Local Variables:
1280  * eval: (c-set-style "gnu")
1281  * End:
1282  */
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:328
vlib_main_t vlib_global_main
Definition: main.c:1850
#define hash_set(h, key, value)
Definition: hash.h:255
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:295
u32 flags
Definition: vhost_user.h:115
vl_api_address_t src
Definition: vxlan_gbp.api:32
#define clib_min(x, y)
Definition: clib.h:295
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
Definition: buffer.c:62
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:529
#define clib_atomic_add_fetch(a, b)
Definition: atomics.h:30
#define STRUCT_MARK_PTR(v, f)
Definition: clib.h:71
format_function_t format_vlib_buffer_contents
Definition: buffer_funcs.h:962
static uword vlib_buffer_get_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:322
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:146
#define PREDICT_TRUE(x)
Definition: clib.h:112
static_always_inline u64x2 u32x4_extend_to_u64x2_high(u32x4 v)
Definition: vector_neon.h:110
format_function_t format_vlib_buffer
Definition: buffer_funcs.h:962
vlib_buffer_callbacks_t cb
Definition: buffer.h:457
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static void vlib_buffer_attach_clone(vlib_main_t *vm, vlib_buffer_t *head, vlib_buffer_t *tail)
Attach cloned tail to the buffer.
Definition: buffer_funcs.h:867
static void vlib_validate_buffer_in_use(vlib_buffer_t *b, u32 expected)
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:232
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
Definition: buffer_funcs.h:919
vlib_buffer_t buffer_init_template
Definition: buffer.h:365
void os_panic(void)
Definition: unix-misc.c:174
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:50
int i
foreach_avx2_vec256i static foreach_avx2_vec256u u32x8 u32x8_permute(u32x8 v, u32x8 idx)
Definition: vector_avx2.h:73
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:65
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
Definition: buffer_funcs.h:80
static void vlib_buffer_delete_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:630
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
Definition: buffer.c:768
uword * vlib_buffer_state_validation_hash
Definition: buffer.c:508
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:267
vlib_buffer_free_list_index_t index
Definition: buffer.h:368
unsigned char u8
Definition: types.h:56
static vlib_buffer_t * vlib_buffer_copy(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:685
vlib_buffer_free_list_index_t free_list_index
Definition: buffer_funcs.h:974
static_always_inline void vlib_get_buffer_indices(vlib_main_t *vm, vlib_buffer_t **b, u32 *bi, uword count)
Translate array of buffer pointers into buffer indices.
Definition: buffer_funcs.h:238
void cj_stop(void)
Definition: cj.c:58
u8 *() format_function_t(u8 *s, va_list *args)
Definition: format.h:48
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
Definition: vec.h:535
static u16 vlib_buffer_clone_256(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset)
Create a maximum of 256 clones of buffer and store them in the supplied array.
Definition: buffer_funcs.h:754
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
#define static_always_inline
Definition: clib.h:99
#define always_inline
Definition: clib.h:98
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
Definition: buffer.h:409
u32 * vlib_buffer_state_validation_lock
Definition: buffer.c:507
vhost_vring_state_t state
Definition: vhost_user.h:120
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:401
#define clib_atomic_test_and_set(a)
Definition: atomics.h:40
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:158
#define fl(x, y)
epu8_epi32 epu16_epi32 u64x2
Definition: vector_sse42.h:640
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:584
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
static u32 vlib_buffer_chain_linearize(vlib_main_t *vm, vlib_buffer_t *b)
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
uword size
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:114
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 buffer_index, uword follow_chain)
Definition: buffer.c:232
format_function_t format_vlib_buffer_and_data
Definition: buffer_funcs.h:962
unsigned short u16
Definition: types.h:57
void(* vlib_buffer_delete_free_list_cb)(struct vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer.h:410
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:214
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:442
static_always_inline u32x4 u32x4_shuffle(u32x4 v, const int a, const int b, const int c, const int d)
Definition: vector_sse42.h:617
#define PREDICT_FALSE(x)
Definition: clib.h:111
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: main.h:123
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
Definition: buffer_funcs.h:177
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
static uword vlib_buffer_contents(vlib_main_t *vm, u32 buffer_index, u8 *contents)
Copy buffer contents to memory.
Definition: buffer_funcs.h:301
u8 len
Definition: ip_types.api:49
static void vlib_buffer_set_known_state(u32 buffer_index, vlib_buffer_known_state_t state)
Definition: buffer_funcs.h:374
static void vlib_buffer_chain_init(vlib_buffer_t *first)
Definition: buffer_funcs.h:894
#define vec_add_aligned(V, E, N, A)
Add N elements to end of vector V (no header, specified alignment)
Definition: vec.h:612
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:454
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
Definition: buffer.h:407
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
Definition: buffer_funcs.h:550
static u16 vlib_buffer_clone(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset)
Create multiple clones of buffer and store them in the supplied array.
Definition: buffer_funcs.h:836
vl_api_address_t dst
Definition: vxlan_gbp.api:33
void vlib_buffer_chain_validate(vlib_main_t *vm, vlib_buffer_t *first)
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:261
#define clib_warning(format, args...)
Definition: error.h:59
clib_spinlock_t lock
Definition: buffer.h:426
static vlib_buffer_t * vlib_get_next_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Get next buffer in buffer linklist, or zero for end of list.
Definition: buffer_funcs.h:251
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
Definition: buffer.c:316
static u32 vlib_buffer_space_left_at_end(vlib_main_t *vm, vlib_buffer_t *b)
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:241
vlib_buffer_known_state_t
Definition: buffer_funcs.h:347
#define VLIB_BUFFER_DATA_SIZE
Definition: buffer.h:51
static void vlib_validate_buffer_set_in_use(vlib_buffer_t *b, u32 expected)
int vlib_buffer_add_data(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, u32 *buffer_index, void *data, u32 n_data_bytes)
Definition: buffer.c:709
static_always_inline u64x2 u32x4_extend_to_u64x2(u32x4 v)
Definition: vector_neon.h:104
signed int i32
Definition: types.h:77
#define uword_to_pointer(u, type)
Definition: types.h:136
vlib buffer structure definition and a few select access methods.
#define ASSERT(truth)
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:788
vlib_buffer_free_cb_t * vlib_buffer_free_cb
Definition: buffer.h:408
static vlib_buffer_free_list_index_t vlib_buffer_get_free_list_index(vlib_buffer_t *b)
Definition: buffer_funcs.h:396
void vlib_buffer_free_list_fill_unaligned(vlib_main_t *vm, vlib_buffer_free_list_t *free_list, uword n_unaligned_buffers)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:130
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, vlib_buffer_free_list_index_t *index)
Definition: buffer_funcs.h:647
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:233
size_t count
Definition: vapi.c:47
vlib_buffer_free_list_index_t vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:426
static uword vlib_buffer_index_length_in_chain(vlib_main_t *vm, u32 bi)
Get length in bytes of the buffer index buffer chain.
Definition: buffer_funcs.h:287
static uword pointer_to_uword(const void *p)
Definition: types.h:131
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:142
void * vlib_buffer_state_heap
Definition: buffer.c:509
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, vlib_buffer_free_list_index_t index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:426
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:156
template key/value backing page structure
Definition: bihash_doc.h:44
static u16 vlib_buffer_chain_append_data(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t *first, vlib_buffer_t *last, void *data, u16 data_len)
Definition: buffer_funcs.h:931
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
static u32 vlib_buffer_alloc_to_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Allocate buffers into ring.
Definition: buffer_funcs.h:502
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *last, u32 next_bi)
Definition: buffer_funcs.h:904
u32 opaque2[12]
More opaque data, see ../vnet/vnet/buffer.h.
Definition: buffer.h:164
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u64 uword
Definition: types.h:112
uword buffer_mem_start
Definition: buffer.h:434
uword * buffer_known_hash
Definition: buffer.h:453
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:671
u64x4
Definition: vector_avx2.h:121
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
Definition: buffer.c:688
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
static vlib_buffer_known_state_t vlib_buffer_is_known(u32 buffer_index)
Definition: buffer_funcs.h:363
vlib_buffer_main_t buffer_main
Definition: buffer.c:56
u8 data[0]
Packet data.
Definition: buffer.h:176
static void vlib_buffer_set_free_list_index(vlib_buffer_t *b, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:405
unsigned long long u32x4
Definition: ixge.c:28
static_always_inline vlib_buffer_pool_t * vlib_buffer_pool_get(u8 buffer_pool_index)
Definition: buffer.h:464
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:115
u8 vlib_buffer_free_list_index_t
Definition: buffer.h:62
static void vlib_packet_template_free(vlib_main_t *vm, vlib_packet_template_t *t)
Definition: buffer_funcs.h:996
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:570
void vlib_packet_template_get_packet_helper(vlib_main_t *vm, vlib_packet_template_t *t)
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:657
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:145
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
Definition: buffer.c:665
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:117
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:485
static void vlib_buffer_free_from_ring_no_next(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring without freeing tail buffers.
Definition: buffer_funcs.h:609
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:62
static u32 vlib_buffer_round_size(u32 size)
Definition: buffer_funcs.h:390
vlib_buffer_free_list_index_t free_list_index
Definition: buffer.h:160