FD.io VPP  v17.04-9-g99c0734
Vector Packet Processing
buffer.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.c: allocate/free network buffers.
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 /**
41  * @file
42  *
43  * Allocate/free network buffers.
44  */
45 
46 #include <vlib/vlib.h>
47 #include <vlib/unix/unix.h>
48 
49 uword
51  vlib_buffer_t * b_first)
52 {
53  vlib_buffer_t *b = b_first;
54  uword l_first = b_first->current_length;
55  uword l = 0;
56  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
57  {
58  b = vlib_get_buffer (vm, b->next_buffer);
59  l += b->current_length;
60  }
63  return l + l_first;
64 }
65 
66 u8 *
67 format_vlib_buffer (u8 * s, va_list * args)
68 {
69  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
70  uword indent = format_get_indent (s);
71 
72  s = format (s, "current data %d, length %d, free-list %d, clone-count %u",
74  b->n_add_refs);
75 
77  s = format (s, ", totlen-nifb %d",
79 
81  s = format (s, ", trace 0x%x", b->trace_index);
82 
83  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
84  {
86  u32 next_buffer = b->next_buffer;
87  b = vlib_get_buffer (vm, next_buffer);
88 
89  s =
90  format (s, "\n%Unext-buffer 0x%x, segment length %d, clone-count %u",
91  format_white_space, indent, next_buffer, b->current_length,
92  b->n_add_refs);
93  }
94 
95  return s;
96 }
97 
98 u8 *
99 format_vlib_buffer_and_data (u8 * s, va_list * args)
100 {
101  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
102 
103  s = format (s, "%U, %U",
106 
107  return s;
108 }
109 
110 static u8 *
111 format_vlib_buffer_known_state (u8 * s, va_list * args)
112 {
114  char *t;
115 
116  switch (state)
117  {
118  case VLIB_BUFFER_UNKNOWN:
119  t = "unknown";
120  break;
121 
123  t = "known-allocated";
124  break;
125 
127  t = "known-free";
128  break;
129 
130  default:
131  t = "invalid";
132  break;
133  }
134 
135  return format (s, "%s", t);
136 }
137 
138 u8 *
139 format_vlib_buffer_contents (u8 * s, va_list * va)
140 {
141  vlib_main_t *vm = va_arg (*va, vlib_main_t *);
142  vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *);
143 
144  while (1)
145  {
147  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
148  break;
149  b = vlib_get_buffer (vm, b->next_buffer);
150  }
151 
152  return s;
153 }
154 
155 static u8 *
157  u32 bi,
158  uword follow_buffer_next, uword ** unique_hash)
159 {
160  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
163 
165  return format (0, "unknown free list 0x%x", b->free_list_index);
166 
168 
169  if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE)
170  return format (0, "current data %d before pre-data", b->current_data);
171 
172  if (b->current_data + b->current_length > fl->n_data_bytes)
173  return format (0, "%d-%d beyond end of buffer %d",
175 
176  if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
177  {
179  u8 *msg, *result;
180 
181  k = vlib_buffer_is_known (vm, b->next_buffer);
183  return format (0, "next 0x%x: %U",
185 
186  if (unique_hash)
187  {
188  if (hash_get (*unique_hash, b->next_buffer))
189  return format (0, "duplicate buffer 0x%x", b->next_buffer);
190 
191  hash_set1 (*unique_hash, b->next_buffer);
192  }
193 
194  msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next);
195  if (msg)
196  {
197  result = format (0, "next 0x%x: %v", b->next_buffer, msg);
198  vec_free (msg);
199  return result;
200  }
201  }
202 
203  return 0;
204 }
205 
206 u8 *
207 vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next)
208 {
209  return vlib_validate_buffer_helper (vm, bi, follow_buffer_next,
210  /* unique_hash */ 0);
211 }
212 
213 u8 *
215  u32 * buffers,
216  uword next_buffer_stride,
217  uword n_buffers,
218  vlib_buffer_known_state_t known_state,
219  uword follow_buffer_next)
220 {
221  uword i, *hash;
222  u32 bi, *b = buffers;
224  u8 *msg = 0, *result = 0;
225 
226  hash = hash_create (0, 0);
227  for (i = 0; i < n_buffers; i++)
228  {
229  bi = b[0];
230  b += next_buffer_stride;
231 
232  /* Buffer is not unique. */
233  if (hash_get (hash, bi))
234  {
235  msg = format (0, "not unique");
236  goto done;
237  }
238 
239  k = vlib_buffer_is_known (vm, bi);
240  if (k != known_state)
241  {
242  msg = format (0, "is %U; expected %U",
244  format_vlib_buffer_known_state, known_state);
245  goto done;
246  }
247 
248  msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash);
249  if (msg)
250  goto done;
251 
252  hash_set1 (hash, bi);
253  }
254 
255 done:
256  if (msg)
257  {
258  result = format (0, "0x%x: %v", bi, msg);
259  vec_free (msg);
260  }
261  hash_free (hash);
262  return result;
263 }
264 
265 /*
266  * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1
267  * and vlib_mains[0] = &vlib_global_main from the beginning of time.
268  *
269  * The only place which should ever expand vlib_mains is start_workers()
270  * in threads.c. It knows about the bootstrap vector.
271  */
272 /* *INDENT-OFF* */
273 static struct
274 {
277 } __attribute__ ((packed)) __bootstrap_vlib_main_vector
278  __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) =
279 {
280  .h.len = 1,
281  .vm = &vlib_global_main,
282 };
283 /* *INDENT-ON* */
284 
285 vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
286 
287 
288 /* When dubugging validate that given buffers are either known allocated
289  or known free. */
290 static void
292  u32 * buffers,
293  uword n_buffers,
294  vlib_buffer_known_state_t expected_state)
295 {
296  u32 *b;
297  uword i, bi, is_free;
298 
299  if (CLIB_DEBUG == 0)
300  return;
301 
302  ASSERT (os_get_cpu_number () == 0);
303 
304  /* smp disaster check */
305  if (vec_len (vlib_mains) > 1)
306  ASSERT (vm == vlib_mains[0]);
307 
308  is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
309  b = buffers;
310  for (i = 0; i < n_buffers; i++)
311  {
313 
314  bi = b[0];
315  b += 1;
316  known = vlib_buffer_is_known (vm, bi);
317  if (known != expected_state)
318  {
319  ASSERT (0);
321  (vm, "%s %U buffer 0x%x",
322  is_free ? "freeing" : "allocating",
323  format_vlib_buffer_known_state, known, bi);
324  }
325 
327  (vm, bi,
329  }
330 }
331 
332 void
335 {
336  uword l;
337  u32 *d;
338 
339  l = vec_len (src->buffers);
340  if (l > 0)
341  {
343  clib_memcpy (d, src->buffers, l * sizeof (d[0]));
344  vec_free (src->buffers);
345  }
346 }
347 
348 /* Add buffer free list. */
349 static u32
351  u32 n_data_bytes,
352  u32 is_public, u32 is_default, u8 * name)
353 {
356  int i;
357 
358  ASSERT (os_get_cpu_number () == 0);
359 
360  if (!is_default && pool_elts (bm->buffer_free_list_pool) == 0)
361  {
362  u32 default_free_free_list_index;
363 
364  /* *INDENT-OFF* */
365  default_free_free_list_index =
367  (vm,
368  /* default buffer size */ VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
369  /* is_public */ 1,
370  /* is_default */ 1,
371  (u8 *) "default");
372  /* *INDENT-ON* */
373  ASSERT (default_free_free_list_index ==
375 
376  if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public)
377  return default_free_free_list_index;
378  }
379 
381 
382  memset (f, 0, sizeof (f[0]));
383  f->index = f - bm->buffer_free_list_pool;
384  f->n_data_bytes = vlib_buffer_round_size (n_data_bytes);
386  f->name = clib_mem_is_heap_object (name) ? name : format (0, "%s", name);
387 
388  /* Setup free buffer template. */
391 
392  if (is_public)
393  {
395  if (!p)
397  }
398 
399  for (i = 1; i < vec_len (vlib_mains); i++)
400  {
401  vlib_buffer_main_t *wbm = vlib_mains[i]->buffer_main;
405  ASSERT (f - bm->buffer_free_list_pool ==
406  wf - wbm->buffer_free_list_pool);
407  wf[0] = f[0];
408  wf->buffers = 0;
409  wf->n_alloc = 0;
410  }
411 
412  return f->index;
413 }
414 
415 u32
417  char *fmt, ...)
418 {
419  va_list va;
420  u8 *name;
421 
422  va_start (va, fmt);
423  name = va_format (0, fmt, &va);
424  va_end (va);
425 
426  return vlib_buffer_create_free_list_helper (vm, n_data_bytes,
427  /* is_public */ 0,
428  /* is_default */ 0,
429  name);
430 }
431 
432 u32
434  char *fmt, ...)
435 {
436  u32 i = vlib_buffer_get_free_list_with_size (vm, n_data_bytes);
437 
438  if (i == ~0)
439  {
440  va_list va;
441  u8 *name;
442 
443  va_start (va, fmt);
444  name = va_format (0, fmt, &va);
445  va_end (va);
446 
447  i = vlib_buffer_create_free_list_helper (vm, n_data_bytes,
448  /* is_public */ 1,
449  /* is_default */ 0,
450  name);
451  }
452 
453  return i;
454 }
455 
456 static void
458 {
459  u32 i;
460 
461  for (i = 0; i < vec_len (f->buffer_memory_allocated); i++)
463  vec_free (f->name);
465  vec_free (f->buffers);
466 }
467 
468 /* Add buffer free list. */
469 void
471 {
474  u32 merge_index;
475  int i;
476 
477  ASSERT (os_get_cpu_number () == 0);
478 
479  f = vlib_buffer_get_free_list (vm, free_list_index);
480 
481  ASSERT (vec_len (f->buffers) == f->n_alloc);
482  merge_index = vlib_buffer_get_free_list_with_size (vm, f->n_data_bytes);
483  if (merge_index != ~0 && merge_index != free_list_index)
484  {
486  (bm->buffer_free_list_pool, merge_index),
487  f);
488  }
489 
490  del_free_list (vm, f);
491 
492  /* Poison it. */
493  memset (f, 0xab, sizeof (f[0]));
494 
496 
497  for (i = 1; i < vec_len (vlib_mains); i++)
498  {
499  bm = vlib_mains[i]->buffer_main;
500  f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);;
501  memset (f, 0xab, sizeof (f[0]));
503  }
504 }
505 
506 /* Make sure free list has at least given number of free buffers. */
507 static uword
509  vlib_buffer_free_list_t * fl, uword min_free_buffers)
510 {
511  vlib_buffer_t *buffers, *b;
512  int n, n_bytes, i;
513  u32 *bi;
514  u32 n_remaining, n_alloc, n_this_chunk;
515 
516  /* Already have enough free buffers on free list? */
517  n = min_free_buffers - vec_len (fl->buffers);
518  if (n <= 0)
519  return min_free_buffers;
520 
521  /* Always allocate round number of buffers. */
522  n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
523 
524  /* Always allocate new buffers in reasonably large sized chunks. */
526 
527  n_remaining = n;
528  n_alloc = 0;
529  while (n_remaining > 0)
530  {
531  n_this_chunk = clib_min (n_remaining, 16);
532 
533  n_bytes = n_this_chunk * (sizeof (b[0]) + fl->n_data_bytes);
534 
535  /* drb: removed power-of-2 ASSERT */
536  buffers = vm->os_physmem_alloc_aligned (&vm->physmem_main,
537  n_bytes,
538  sizeof (vlib_buffer_t));
539  if (!buffers)
540  return n_alloc;
541 
542  /* Record chunk as being allocated so we can free it later. */
543  vec_add1 (fl->buffer_memory_allocated, buffers);
544 
545  fl->n_alloc += n_this_chunk;
546  n_alloc += n_this_chunk;
547  n_remaining -= n_this_chunk;
548 
549  b = buffers;
550  vec_add2_aligned (fl->buffers, bi, n_this_chunk, CLIB_CACHE_LINE_BYTES);
551  for (i = 0; i < n_this_chunk; i++)
552  {
553  bi[i] = vlib_get_buffer_index (vm, b);
554 
555  if (CLIB_DEBUG > 0)
558  }
559 
560  memset (buffers, 0, n_bytes);
561 
562  /* Initialize all new buffers. */
563  b = buffers;
564  for (i = 0; i < n_this_chunk; i++)
565  {
568  }
569 
570  if (fl->buffer_init_function)
571  fl->buffer_init_function (vm, fl, bi, n_this_chunk);
572  }
573  return n_alloc;
574 }
575 
576 static u32
578  vlib_buffer_free_list_t * free_list,
579  u32 * alloc_buffers, u32 n_alloc_buffers)
580 {
581  u32 *dst, *src;
582  uword len;
583  uword n_filled;
584 
585  dst = alloc_buffers;
586 
587  /* wait with buffer memory allocation as long as possible
588  in case external buffer manager takes over */
590  unix_physmem_init (vm, 0 /* fail_if_physical_memory_not_present */ );
591 
592  n_filled = fill_free_list (vm, free_list, n_alloc_buffers);
593  if (n_filled == 0)
594  return 0;
595 
596  len = vec_len (free_list->buffers);
597  ASSERT (len >= n_alloc_buffers);
598 
599  src = free_list->buffers + len - n_alloc_buffers;
600  clib_memcpy (dst, src, n_alloc_buffers * sizeof (u32));
601 
602  _vec_len (free_list->buffers) -= n_alloc_buffers;
603 
604  /* Verify that buffers are known free. */
605  vlib_buffer_validate_alloc_free (vm, alloc_buffers,
606  n_alloc_buffers, VLIB_BUFFER_KNOWN_FREE);
607 
608  return n_alloc_buffers;
609 }
610 
611 
612 /* Allocate a given number of buffers into given array.
613  Returns number actually allocated which will be either zero or
614  number requested. */
615 static u32
617 {
619 
620  return alloc_from_free_list
621  (vm,
624  buffers, n_buffers);
625 }
626 
627 static u32
629  u32 * buffers,
630  u32 n_buffers, u32 free_list_index)
631 {
634  f = pool_elt_at_index (bm->buffer_free_list_pool, free_list_index);
635  return alloc_from_free_list (vm, f, buffers, n_buffers);
636 }
637 
638 void *
640 {
642  void *rv = bm->buffer_free_callback;
643 
644  bm->buffer_free_callback = fp;
645  return rv;
646 }
647 
650  u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
651 {
654  u32 fi;
655  int i;
656  u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
657  u32 follow_buffer_next);
658 
659  cb = bm->buffer_free_callback;
660 
661  if (PREDICT_FALSE (cb != 0))
662  n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
663 
664  if (!n_buffers)
665  return;
666 
667  for (i = 0; i < n_buffers; i++)
668  {
669  vlib_buffer_t *b;
670  u32 bi = buffers[i];
671 
672  b = vlib_get_buffer (vm, bi);
673 
674  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
675 
676  /* The only current use of this callback: multicast recycle */
678  {
679  int j;
680 
682  (vm, fl, buffers[i], (b->flags & VLIB_BUFFER_RECYCLE) == 0);
683 
684  for (j = 0; j < vec_len (bm->announce_list); j++)
685  {
686  if (fl == bm->announce_list[j])
687  goto already_announced;
688  }
689  vec_add1 (bm->announce_list, fl);
690  already_announced:
691  ;
692  }
693  else
694  {
695  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
696  {
697  u32 flags, next;
698 
699  do
700  {
701  vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
702  flags = nb->flags;
703  next = nb->next_buffer;
704  if (nb->n_add_refs)
705  nb->n_add_refs--;
706  else
707  {
710  vlib_buffer_add_to_free_list (vm, fl, bi, 1);
711  }
712  bi = next;
713  }
714  while (follow_buffer_next
715  && (flags & VLIB_BUFFER_NEXT_PRESENT));
716 
717  }
718  }
719  }
720  if (vec_len (bm->announce_list))
721  {
723  for (i = 0; i < vec_len (bm->announce_list); i++)
724  {
725  fl = bm->announce_list[i];
727  }
728  _vec_len (bm->announce_list) = 0;
729  }
730 }
731 
732 static void
733 vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
734 {
735  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
736  1);
737 }
738 
739 static void
741  u32 n_buffers)
742 {
743  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
744  0);
745 }
746 
747 /* Copy template packet data into buffers as they are allocated. */
748 static void __attribute__ ((unused))
751  u32 * buffers, u32 n_buffers)
752 {
754  uword_to_pointer (fl->buffer_init_function_opaque,
756  uword i;
757 
758  for (i = 0; i < n_buffers; i++)
759  {
760  vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]);
763  b->current_length);
764  }
765 }
766 
767 void
770  void *packet_data,
771  uword n_packet_data_bytes,
772  uword min_n_buffers_each_physmem_alloc,
773  char *fmt, ...)
774 {
776  va_list va;
777  __attribute__ ((unused)) u8 *name;
779 
780  va_start (va, fmt);
781  name = va_format (0, fmt, &va);
782  va_end (va);
783 
785  bm->cb.vlib_packet_template_init_cb (vm, (void *) t, packet_data,
786  n_packet_data_bytes,
787  min_n_buffers_each_physmem_alloc,
788  name);
789 
791 
792  memset (t, 0, sizeof (t[0]));
793 
794  vec_add (t->packet_data, packet_data, n_packet_data_bytes);
795  t->min_n_buffers_each_physmem_alloc = min_n_buffers_each_physmem_alloc;
796 
798  (vm, n_packet_data_bytes,
799  /* is_public */ 1,
800  /* is_default */ 0,
801  name);
802 
803  ASSERT (t->free_list_index != 0);
806 
809 
811  fl->buffer_init_template.current_length = n_packet_data_bytes;
812  fl->buffer_init_template.flags = 0;
815 }
816 
817 void *
819  vlib_packet_template_t * t, u32 * bi_result)
820 {
821  u32 bi;
822  vlib_buffer_t *b;
823 
824  if (vlib_buffer_alloc (vm, &bi, 1) != 1)
825  return 0;
826 
827  *bi_result = bi;
828 
829  b = vlib_get_buffer (vm, bi);
831  t->packet_data, vec_len (t->packet_data));
833 
834  return b->data;
835 }
836 
837 void
840 {
842  word l = vec_len (t->packet_data);
843  word n_alloc;
844 
845  ASSERT (l > 0);
846  ASSERT (vec_len (t->free_buffers) == 0);
847 
848  vec_validate (t->free_buffers, n - 1);
850  n, t->free_list_index);
851  _vec_len (t->free_buffers) = n_alloc;
852 }
853 
854 /* Append given data to end of buffer, possibly allocating new buffers. */
855 u32
857  u32 free_list_index,
858  u32 buffer_index, void *data, u32 n_data_bytes)
859 {
860  u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
861  vlib_buffer_t *b;
862  void *d;
863 
864  bi = buffer_index;
865  if (bi == 0
866  && 1 != vlib_buffer_alloc_from_free_list (vm, &bi, 1, free_list_index))
867  goto out_of_buffers;
868 
869  d = data;
870  n_left = n_data_bytes;
871  n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index);
872 
873  b = vlib_get_buffer (vm, bi);
875 
876  /* Get to the end of the chain before we try to append data... */
877  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
878  b = vlib_get_buffer (vm, b->next_buffer);
879 
880  while (1)
881  {
882  u32 n;
883 
884  ASSERT (n_buffer_bytes >= b->current_length);
885  n_left_this_buffer =
886  n_buffer_bytes - (b->current_data + b->current_length);
887  n = clib_min (n_left_this_buffer, n_left);
889  b->current_length += n;
890  n_left -= n;
891  if (n_left == 0)
892  break;
893 
894  d += n;
895  if (1 !=
897  free_list_index))
898  goto out_of_buffers;
899 
901 
902  b = vlib_get_buffer (vm, b->next_buffer);
903  }
904 
905  return bi;
906 
907 out_of_buffers:
908  clib_error ("out of buffers");
909  return bi;
910 }
911 
912 u16
914  u32 free_list_index,
916  vlib_buffer_t ** last,
917  void *data, u16 data_len)
918 {
919  vlib_buffer_t *l = *last;
920  u32 n_buffer_bytes =
921  vlib_buffer_free_list_buffer_size (vm, free_list_index);
922  u16 copied = 0;
923  ASSERT (n_buffer_bytes >= l->current_length + l->current_data);
924  while (data_len)
925  {
926  u16 max = n_buffer_bytes - l->current_length - l->current_data;
927  if (max == 0)
928  {
929  if (1 !=
931  free_list_index))
932  return copied;
933  *last = l = vlib_buffer_chain_buffer (vm, first, l, l->next_buffer);
934  max = n_buffer_bytes - l->current_length - l->current_data;
935  }
936 
937  u16 len = (data_len > max) ? max : data_len;
939  data + copied, len);
940  vlib_buffer_chain_increase_length (first, l, len);
941  data_len -= len;
942  copied += len;
943  }
944  return copied;
945 }
946 
947 
948 static u8 *
949 format_vlib_buffer_free_list (u8 * s, va_list * va)
950 {
952  u32 threadnum = va_arg (*va, u32);
953  uword bytes_alloc, bytes_free, n_free, size;
954 
955  if (!f)
956  return format (s, "%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s",
957  "Thread", "Name", "Index", "Size", "Alloc", "Free",
958  "#Alloc", "#Free");
959 
960  size = sizeof (vlib_buffer_t) + f->n_data_bytes;
961  n_free = vec_len (f->buffers);
962  bytes_alloc = size * f->n_alloc;
963  bytes_free = size * n_free;
964 
965  s = format (s, "%7d%30s%12d%12d%=12U%=12U%=12d%=12d", threadnum,
966  f->name, f->index, f->n_data_bytes,
967  format_memory_size, bytes_alloc,
968  format_memory_size, bytes_free, f->n_alloc, n_free);
969 
970  return s;
971 }
972 
973 static clib_error_t *
975  unformat_input_t * input, vlib_cli_command_t * cmd)
976 {
977  vlib_buffer_main_t *bm;
979  vlib_main_t *curr_vm;
980  u32 vm_index = 0;
981 
983 
984  do
985  {
986  curr_vm = vlib_mains[vm_index];
987  bm = curr_vm->buffer_main;
988 
989  /* *INDENT-OFF* */
991  vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f, vm_index);
992  }));
993  /* *INDENT-ON* */
994 
995  vm_index++;
996  }
997  while (vm_index < vec_len (vlib_mains));
998 
999  return 0;
1000 }
1001 
1002 /* *INDENT-OFF* */
1003 VLIB_CLI_COMMAND (show_buffers_command, static) = {
1004  .path = "show buffers",
1005  .short_help = "Show packet buffer allocation",
1006  .function = show_buffers,
1007 };
1008 /* *INDENT-ON* */
1009 
1010 void
1012 {
1013  vlib_buffer_main_t *bm = vm->buffer_main;
1021  bm->extern_buffer_mgmt = 0;
1022 }
1023 
1024 int
1026 {
1027  vlib_buffer_main_t *bm = vm->buffer_main;
1028  if (bm->extern_buffer_mgmt)
1029  return -1;
1030 
1031 #define _(x) bm->cb.x = cb->x
1032  _(vlib_buffer_alloc_cb);
1033  _(vlib_buffer_alloc_from_free_list_cb);
1034  _(vlib_buffer_free_cb);
1035  _(vlib_buffer_free_no_next_cb);
1036  _(vlib_buffer_delete_free_list_cb);
1037 #undef _
1038  bm->extern_buffer_mgmt = 1;
1039  return 0;
1040 }
1041 
1042 /** @endcond */
1043 /*
1044  * fd.io coding-style-patch-verification: ON
1045  *
1046  * Local Variables:
1047  * eval: (c-set-style "gnu")
1048  * End:
1049  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:436
#define hash_set(h, key, value)
Definition: hash.h:254
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
void vlib_buffer_merge_free_lists(vlib_buffer_free_list_t *dst, vlib_buffer_free_list_t *src)
Definition: buffer.c:333
#define clib_min(x, y)
Definition: clib.h:332
#define VLIB_BUFFER_RECYCLE
Definition: buffer.h:91
static void vlib_buffer_set_known_state(vlib_main_t *vm, u32 buffer_index, vlib_buffer_known_state_t state)
Definition: buffer_funcs.h:219
int vlib_buffer_cb_register(struct vlib_main_t *vm, vlib_buffer_callbacks_t *cb)
Definition: buffer.c:1025
u32 free_list_index
Buffer free list that this buffer was allocated from and will be freed to.
Definition: buffer.h:99
#define PREDICT_TRUE(x)
Definition: clib.h:98
vlib_physmem_main_t physmem_main
Definition: main.h:106
#define clib_error(format, args...)
Definition: error.h:62
vlib_buffer_callbacks_t cb
Definition: buffer.h:407
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, u32 next_bi)
Definition: buffer_funcs.h:646
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:573
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
Definition: buffer_funcs.h:663
vlib_buffer_t buffer_init_template
Definition: buffer.h:318
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static u32 alloc_from_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *free_list, u32 *alloc_buffers, u32 n_alloc_buffers)
Definition: buffer.c:577
static u32 vlib_buffer_alloc_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:616
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
void ** buffer_memory_allocated
Definition: buffer.h:339
vlib_buffer_main_t * buffer_main
Definition: main.h:104
static void vlib_buffer_free_no_next_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:740
static uword fill_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
Definition: buffer.c:508
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:418
u8 * va_format(u8 *s, const char *fmt, va_list *va)
Definition: format.c:386
u32 min_n_buffers_each_physmem_alloc
Definition: buffer.h:328
static clib_error_t * show_buffers(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: buffer.c:974
static vlib_buffer_known_state_t vlib_buffer_is_known(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:209
vlib_main_t ** vlib_mains
Definition: buffer.c:285
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
Definition: buffer.c:50
void vlib_packet_template_get_packet_helper(vlib_main_t *vm, vlib_packet_template_t *t)
Definition: buffer.c:838
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:599
void vlib_buffer_delete_free_list_internal(vlib_main_t *vm, u32 free_list_index)
Definition: buffer.c:470
static u8 * format_vlib_buffer_free_list(u8 *s, va_list *va)
Definition: buffer.c:949
#define VLIB_BUFFER_NEXT_PRESENT
Definition: buffer.h:87
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:67
#define static_always_inline
Definition: clib.h:85
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
Definition: buffer.c:818
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:376
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.h:346
static uword format_get_indent(u8 *s)
Definition: format.h:72
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
#define VLIB_BUFFER_TOTAL_LENGTH_VALID
Definition: buffer.h:89
u8 * format_memory_size(u8 *s, va_list *va)
Definition: std-formats.c:193
u8 * format_hex_bytes(u8 *s, va_list *va)
Definition: std-formats.c:84
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:391
static vlib_buffer_t * vlib_buffer_next_contiguous(vlib_buffer_t *b, u32 buffer_bytes)
Definition: buffer.h:167
u8 * format_vlib_buffer(u8 *s, va_list *args)
Definition: buffer.c:67
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:70
vlib_buffer_free_list_t ** announce_list
Definition: buffer.h:404
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
#define hash_get(h, key)
Definition: hash.h:248
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:397
void * vlib_set_buffer_free_callback(vlib_main_t *vm, void *fp)
Definition: buffer.c:639
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:71
vlib_main_t vlib_global_main
Definition: main.c:1617
static u32 vlib_buffer_get_free_list_with_size(vlib_main_t *vm, u32 size)
Definition: buffer_funcs.h:364
static u32 vlib_buffer_alloc_from_free_list_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 free_list_index)
Definition: buffer.c:628
static u32 vlib_buffer_create_free_list_helper(vlib_main_t *vm, u32 n_data_bytes, u32 is_public, u32 is_default, u8 *name)
Definition: buffer.c:350
struct _unformat_input_t unformat_input_t
uword os_get_cpu_number(void)
Definition: unix-misc.c:224
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:188
#define hash_free(h)
Definition: hash.h:286
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:390
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:241
void(* vlib_buffer_delete_free_list_cb)(struct vlib_main_t *vm, u32 free_list_index)
Definition: buffer.h:375
static u8 * vlib_validate_buffer_helper(vlib_main_t *vm, u32 bi, uword follow_buffer_next, uword **unique_hash)
Definition: buffer.c:156
#define PREDICT_FALSE(x)
Definition: clib.h:97
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:399
#define VLIB_FRAME_SIZE
Definition: node.h:328
static void vlib_packet_template_buffer_init(vlib_main_t *vm, vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.c:749
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
Definition: buffer_funcs.h:812
void(* vlib_buffer_free_cb)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:365
void(* vlib_buffer_free_no_next_cb)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:367
#define uword_to_pointer(u, type)
Definition: types.h:136
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, u32 free_list_index, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
Definition: buffer.c:913
#define pool_get_aligned(P, E, A)
Allocate an object E from a pool P (general version).
Definition: pool.h:169
u32 vlib_buffer_add_data(vlib_main_t *vm, u32 free_list_index, u32 buffer_index, void *data, u32 n_data_bytes)
Definition: buffer.c:856
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.c:649
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.h:382
vlib_main_t * vm
Definition: buffer.c:276
vec_header_t h
Definition: buffer.c:275
u32 vlib_buffer_get_or_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:433
void *(* os_physmem_alloc_aligned)(vlib_physmem_main_t *pm, uword n_bytes, uword alignment)
Definition: main.h:110
u8 * format_vlib_buffer_contents(u8 *s, va_list *va)
Definition: buffer.c:139
void vlib_buffer_cb_init(struct vlib_main_t *vm)
Definition: buffer.c:1011
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:340
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 bi, uword follow_buffer_next)
Definition: buffer.c:207
#define VLIB_BUFFER_IS_TRACED
Definition: buffer.h:85
#define clib_memcpy(a, b, c)
Definition: string.h:69
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 free_list_index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:269
void(* buffers_added_to_freelist_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl)
Definition: buffer.h:353
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
Definition: pool.h:238
void vlib_worker_thread_barrier_sync(vlib_main_t *vm)
Definition: threads.c:1199
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:278
vlib_buffer_known_state_t
Definition: buffer_funcs.h:198
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
#define hash_set1(h, key)
Definition: hash.h:257
#define hash_create(elts, value_bytes)
Definition: hash.h:658
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
u32(* vlib_buffer_alloc_cb)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:360
u32(* vlib_buffer_alloc_from_free_list_cb)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 free_list_index)
Definition: buffer.h:362
static u8 * format_vlib_buffer_known_state(u8 *s, va_list *args)
Definition: buffer.c:111
void(* vlib_packet_template_init_cb)(struct vlib_main_t *vm, void *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, u8 *name)
Definition: buffer.h:369
vhost_vring_state_t state
Definition: vhost-user.h:83
u64 size
Definition: vhost-user.h:77
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, u32 *index)
Definition: buffer_funcs.h:374
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:109
u8 * format_vlib_buffer_and_data(u8 *s, va_list *args)
Definition: buffer.c:99
vector header structure
Definition: vec_bootstrap.h:55
static uword clib_mem_is_heap_object(void *p)
Definition: mem.h:159
static void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
Definition: buffer.c:291
u32 vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:416
void(* os_physmem_free)(void *x)
Definition: main.h:112
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:124
#define clib_max(x, y)
Definition: clib.h:325
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
u64 uword
Definition: types.h:112
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:103
unsigned short u16
Definition: types.h:57
i64 word
Definition: types.h:111
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: buffer.h:389
clib_error_t * unix_physmem_init(vlib_main_t *vm, int physical_memory_required)
Definition: physmem.c:209
u8 * vlib_validate_buffers(vlib_main_t *vm, u32 *buffers, uword next_buffer_stride, uword n_buffers, vlib_buffer_known_state_t known_state, uword follow_buffer_next)
Definition: buffer.c:214
int extern_buffer_mgmt
Definition: buffer.h:408
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, char *fmt,...)
Definition: buffer.c:768
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
Definition: buffer_funcs.h:777
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1231
u8 data[0]
Packet data.
Definition: buffer.h:152
uword buffer_init_function_opaque
Definition: buffer.h:355
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, u32 free_list_index)
Definition: buffer_funcs.h:385
u32 flags
Definition: vhost-user.h:78
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:74
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:245
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
Definition: buffer.c:457
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:577
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:136
uword * free_list_by_size
Definition: buffer.h:395
#define vlib_panic_with_msg(vm, args...)
Definition: main.h:236
static void vlib_buffer_free_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:733
static u32 vlib_buffer_round_size(u32 size)
Definition: buffer_funcs.h:255
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:109