FD.io VPP  v19.01.2-3-gf61a1a8
Vector Packet Processing
buffer.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.c: allocate/free network buffers.
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 /**
41  * @file
42  *
43  * Allocate/free network buffers.
44  */
45 
46 #include <vlib/vlib.h>
47 #include <vlib/unix/unix.h>
48 
49 vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0;
50 
51 /* when running unpriviledged we are limited by RLIMIT_MEMLOCK which is
52  typically set to 16MB so setting default size for buffer memory to 14MB
53  */
54 static u32 vlib_buffer_physmem_sz = 14 << 20;
55 
57 
58 /* logging */
60 
61 uword
63  vlib_buffer_t * b_first)
64 {
65  vlib_buffer_t *b = b_first;
66  uword l_first = b_first->current_length;
67  uword l = 0;
68  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
69  {
70  b = vlib_get_buffer (vm, b->next_buffer);
71  l += b->current_length;
72  }
74  b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
75  return l + l_first;
76 }
77 
78 u8 *
79 format_vlib_buffer (u8 * s, va_list * args)
80 {
81  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
82  u32 indent = format_get_indent (s);
83  u8 *a = 0;
84 
85 #define _(bit, name, v) \
86  if (v && (b->flags & VLIB_BUFFER_##name)) \
87  a = format (a, "%s ", v);
89 #undef _
90  s = format (s, "current data %d, length %d, free-list %d, clone-count %u",
93 
94  if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
95  s = format (s, ", totlen-nifb %d",
97 
98  if (b->flags & VLIB_BUFFER_IS_TRACED)
99  s = format (s, ", trace 0x%x", b->trace_index);
100 
101  if (a)
102  s = format (s, "\n%U%v", format_white_space, indent, a);
103  vec_free (a);
104 
105  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
106  {
108  u32 next_buffer = b->next_buffer;
109  b = vlib_get_buffer (vm, next_buffer);
110 
111  s =
112  format (s, "\n%Unext-buffer 0x%x, segment length %d, clone-count %u",
113  format_white_space, indent, next_buffer, b->current_length,
114  b->n_add_refs);
115  }
116 
117  return s;
118 }
119 
120 u8 *
121 format_vlib_buffer_and_data (u8 * s, va_list * args)
122 {
123  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
124 
125  s = format (s, "%U, %U",
128 
129  return s;
130 }
131 
132 static u8 *
133 format_vlib_buffer_known_state (u8 * s, va_list * args)
134 {
136  char *t;
137 
138  switch (state)
139  {
140  case VLIB_BUFFER_UNKNOWN:
141  t = "unknown";
142  break;
143 
145  t = "known-allocated";
146  break;
147 
149  t = "known-free";
150  break;
151 
152  default:
153  t = "invalid";
154  break;
155  }
156 
157  return format (s, "%s", t);
158 }
159 
160 u8 *
161 format_vlib_buffer_contents (u8 * s, va_list * va)
162 {
163  vlib_main_t *vm = va_arg (*va, vlib_main_t *);
164  vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *);
165 
166  while (1)
167  {
169  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
170  break;
171  b = vlib_get_buffer (vm, b->next_buffer);
172  }
173 
174  return s;
175 }
176 
177 static u8 *
179  u32 bi,
180  uword follow_buffer_next, uword ** unique_hash)
181 {
182  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
184 
187  return format (0, "unknown free list 0x%x",
189 
190  fl =
193 
194  if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE)
195  return format (0, "current data %d before pre-data", b->current_data);
196 
197  if (b->current_data + b->current_length > fl->n_data_bytes)
198  return format (0, "%d-%d beyond end of buffer %d",
200 
201  if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
202  {
204  u8 *msg, *result;
205 
208  return format (0, "next 0x%x: %U",
210 
211  if (unique_hash)
212  {
213  if (hash_get (*unique_hash, b->next_buffer))
214  return format (0, "duplicate buffer 0x%x", b->next_buffer);
215 
216  hash_set1 (*unique_hash, b->next_buffer);
217  }
218 
219  msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next);
220  if (msg)
221  {
222  result = format (0, "next 0x%x: %v", b->next_buffer, msg);
223  vec_free (msg);
224  return result;
225  }
226  }
227 
228  return 0;
229 }
230 
231 u8 *
232 vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next)
233 {
234  return vlib_validate_buffer_helper (vm, bi, follow_buffer_next,
235  /* unique_hash */ 0);
236 }
237 
238 u8 *
240  u32 * buffers,
241  uword next_buffer_stride,
242  uword n_buffers,
243  vlib_buffer_known_state_t known_state,
244  uword follow_buffer_next)
245 {
246  uword i, *hash;
247  u32 bi, *b = buffers;
249  u8 *msg = 0, *result = 0;
250 
251  hash = hash_create (0, 0);
252  for (i = 0; i < n_buffers; i++)
253  {
254  bi = b[0];
255  b += next_buffer_stride;
256 
257  /* Buffer is not unique. */
258  if (hash_get (hash, bi))
259  {
260  msg = format (0, "not unique");
261  goto done;
262  }
263 
264  k = vlib_buffer_is_known (bi);
265  if (k != known_state)
266  {
267  msg = format (0, "is %U; expected %U",
269  format_vlib_buffer_known_state, known_state);
270  goto done;
271  }
272 
273  msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash);
274  if (msg)
275  goto done;
276 
277  hash_set1 (hash, bi);
278  }
279 
280 done:
281  if (msg)
282  {
283  result = format (0, "0x%x: %v", bi, msg);
284  vec_free (msg);
285  }
286  hash_free (hash);
287  return result;
288 }
289 
290 /*
291  * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1
292  * and vlib_mains[0] = &vlib_global_main from the beginning of time.
293  *
294  * The only place which should ever expand vlib_mains is start_workers()
295  * in threads.c. It knows about the bootstrap vector.
296  */
297 /* *INDENT-OFF* */
298 static struct
299 {
302 } __attribute__ ((packed)) __bootstrap_vlib_main_vector
303  __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) =
304 {
305  .h.len = 1,
306  .vm = &vlib_global_main,
307 };
308 /* *INDENT-ON* */
309 
310 vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
311 
312 
313 /* When dubugging validate that given buffers are either known allocated
314  or known free. */
315 void
317  u32 * buffers,
318  uword n_buffers,
319  vlib_buffer_known_state_t expected_state)
320 {
321  u32 *b;
322  uword i, bi, is_free;
323 
324  if (CLIB_DEBUG == 0)
325  return;
326 
327  if (vlib_buffer_callbacks)
328  return;
329 
330  is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
331  b = buffers;
332  for (i = 0; i < n_buffers; i++)
333  {
335 
336  bi = b[0];
337  b += 1;
338  known = vlib_buffer_is_known (bi);
339  if (known != expected_state)
340  {
341  ASSERT (0);
343  (vm, "%s %U buffer 0x%x",
344  is_free ? "freeing" : "allocating",
345  format_vlib_buffer_known_state, known, bi);
346  }
347 
350  }
351 }
352 
353 /* Add buffer free list. */
356  u32 n_data_bytes,
357  u32 is_public, u32 is_default, u8 * name)
358 {
361  int i;
362 
363  ASSERT (vlib_get_thread_index () == 0);
364 
365  if (!is_default && pool_elts (vm->buffer_free_list_pool) == 0)
366  {
367  vlib_buffer_free_list_index_t default_free_free_list_index;
368 
369  /* *INDENT-OFF* */
370  default_free_free_list_index =
372  (vm,
373  /* default buffer size */ VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
374  /* is_public */ 1,
375  /* is_default */ 1,
376  (u8 *) "default");
377  /* *INDENT-ON* */
378  ASSERT (default_free_free_list_index ==
380 
381  if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public)
382  return default_free_free_list_index;
383  }
384 
386 
387  clib_memset (f, 0, sizeof (f[0]));
388  f->index = f - vm->buffer_free_list_pool;
389  vec_validate (f->buffers, 0);
391  f->n_data_bytes = vlib_buffer_round_size (n_data_bytes);
393  f->buffer_pool_index = 0;
394  f->name = clib_mem_is_vec (name) ? name : format (0, "%s", name);
395 
396  /* Setup free buffer template. */
399 
400  if (is_public)
401  {
403  if (!p)
405  }
406 
407  for (i = 1; i < vec_len (vlib_mains); i++)
408  {
409  vlib_main_t *wvm = vlib_mains[i];
413  ASSERT (f - vm->buffer_free_list_pool ==
414  wf - wvm->buffer_free_list_pool);
415  wf[0] = f[0];
416  wf->buffers = 0;
417  vec_validate (wf->buffers, 0);
419  wf->n_alloc = 0;
420  }
421 
422  return f->index;
423 }
424 
427  char *fmt, ...)
428 {
429  va_list va;
430  u8 *name;
431 
432  va_start (va, fmt);
433  name = va_format (0, fmt, &va);
434  va_end (va);
435 
436  return vlib_buffer_create_free_list_helper (vm, n_data_bytes,
437  /* is_public */ 0,
438  /* is_default */ 0,
439  name);
440 }
441 
442 static void
444 {
446 
449  vec_free (f->name);
450  vec_free (f->buffers);
451 
452  /* Poison it. */
453  clib_memset (f, 0xab, sizeof (f[0]));
454 }
455 
456 /* Add buffer free list. */
457 void
460 {
462  int i;
463 
464  ASSERT (vlib_get_thread_index () == 0);
465 
466  f = vlib_buffer_get_free_list (vm, index);
467 
468  ASSERT (vec_len (f->buffers) == f->n_alloc);
469 
470  del_free_list (vm, f);
471 
473 
474  for (i = 1; i < vec_len (vlib_mains); i++)
475  {
476  vlib_main_t *wvm = vlib_mains[i];
477  f = vlib_buffer_get_free_list (vlib_mains[i], index);
478  del_free_list (wvm, f);
480  }
481 }
482 
485 {
488 }
489 
490 /* Make sure free list has at least given number of free buffers. */
491 static uword
494  uword min_free_buffers)
495 {
496  vlib_buffer_t *b;
498  int n;
499  u32 *bi;
500  u32 n_alloc = 0;
501 
502  /* Already have enough free buffers on free list? */
503  n = min_free_buffers - vec_len (fl->buffers);
504  if (n <= 0)
505  return min_free_buffers;
506 
507  if (vec_len (bp->buffers) > 0)
508  {
509  int n_copy, n_left;
510  clib_spinlock_lock (&bp->lock);
511  n_copy = clib_min (vec_len (bp->buffers), n);
512  n_left = vec_len (bp->buffers) - n_copy;
513  vec_add_aligned (fl->buffers, bp->buffers + n_left, n_copy,
515  _vec_len (bp->buffers) = n_left;
516  clib_spinlock_unlock (&bp->lock);
517  n = min_free_buffers - vec_len (fl->buffers);
518  if (n <= 0)
519  return min_free_buffers;
520  }
521 
522  /* Always allocate round number of buffers. */
523  n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
524 
525  /* Always allocate new buffers in reasonably large sized chunks. */
526  n = clib_max (n, fl->min_n_buffers_each_alloc);
527 
528  clib_spinlock_lock (&bp->lock);
529  while (n_alloc < n)
530  {
531  if ((b = vlib_buffer_pool_get_buffer (vm, bp)) == 0)
532  goto done;
533 
534  n_alloc += 1;
535 
537  bi[0] = vlib_get_buffer_index (vm, b);
538 
539  if (CLIB_DEBUG > 0)
541 
542  clib_memset (b, 0, sizeof (vlib_buffer_t));
544 
545  if (fl->buffer_init_function)
546  fl->buffer_init_function (vm, fl, bi, 1);
547  }
548 
549 done:
550  clib_spinlock_unlock (&bp->lock);
551  fl->n_alloc += n_alloc;
552  return n_alloc;
553 }
554 
555 void *
557 {
559  void *rv = bm->buffer_free_callback;
560 
561  bm->buffer_free_callback = fp;
562  return rv;
563 }
564 
567  vlib_buffer_t * b, u32 follow_buffer_next)
568 {
571  u32 flags, next;
572 
573  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
574 
575  do
576  {
577  vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
578  flags = nb->flags;
579  next = nb->next_buffer;
580  if (nb->n_add_refs)
581  nb->n_add_refs--;
582  else
583  {
586  vlib_buffer_add_to_free_list (vm, fl, bi, 1);
587  }
588  bi = next;
589  }
590  while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
591 }
592 
595  u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
596 {
598  vlib_buffer_t *p, *b0, *b1, *b2, *b3;
599  int i = 0;
600  u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
601  u32 follow_buffer_next);
602 
603  cb = bm->buffer_free_callback;
604 
605  if (PREDICT_FALSE (cb != 0))
606  n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
607 
608  if (!n_buffers)
609  return;
610 
611  while (i + 11 < n_buffers)
612  {
613  p = vlib_get_buffer (vm, buffers[i + 8]);
614  vlib_prefetch_buffer_header (p, LOAD);
615  p = vlib_get_buffer (vm, buffers[i + 9]);
616  vlib_prefetch_buffer_header (p, LOAD);
617  p = vlib_get_buffer (vm, buffers[i + 10]);
618  vlib_prefetch_buffer_header (p, LOAD);
619  p = vlib_get_buffer (vm, buffers[i + 11]);
620  vlib_prefetch_buffer_header (p, LOAD);
621 
622  b0 = vlib_get_buffer (vm, buffers[i]);
623  b1 = vlib_get_buffer (vm, buffers[i + 1]);
624  b2 = vlib_get_buffer (vm, buffers[i + 2]);
625  b3 = vlib_get_buffer (vm, buffers[i + 3]);
626 
631 
632  recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
633  recycle_or_free (vm, bm, buffers[i + 1], b1, follow_buffer_next);
634  recycle_or_free (vm, bm, buffers[i + 2], b2, follow_buffer_next);
635  recycle_or_free (vm, bm, buffers[i + 3], b3, follow_buffer_next);
636 
637  i += 4;
638  }
639 
640  while (i < n_buffers)
641  {
642  b0 = vlib_get_buffer (vm, buffers[i]);
644  recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
645  i++;
646  }
647 }
648 
649 static void
650 vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
651 {
652  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
653  1);
654 }
655 
656 static void
658  u32 n_buffers)
659 {
660  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
661  0);
662 }
663 
664 void
667  void *packet_data,
668  uword n_packet_data_bytes,
669  uword min_n_buffers_each_alloc, char *fmt, ...)
670 {
671  va_list va;
672 
673  va_start (va, fmt);
674  t->name = va_format (0, fmt, &va);
675  va_end (va);
676 
678 
679  clib_memset (t, 0, sizeof (t[0]));
680 
681  vec_add (t->packet_data, packet_data, n_packet_data_bytes);
682  t->min_n_buffers_each_alloc = min_n_buffers_each_alloc;
683 
685 }
686 
687 void *
689  vlib_packet_template_t * t, u32 * bi_result)
690 {
691  u32 bi;
692  vlib_buffer_t *b;
693 
694  if (vlib_buffer_alloc (vm, &bi, 1) != 1)
695  return 0;
696 
697  *bi_result = bi;
698 
699  b = vlib_get_buffer (vm, bi);
701  t->packet_data, vec_len (t->packet_data));
703 
704  return b->data;
705 }
706 
707 /* Append given data to end of buffer, possibly allocating new buffers. */
708 int
710  vlib_buffer_free_list_index_t free_list_index,
711  u32 * buffer_index, void *data, u32 n_data_bytes)
712 {
713  u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
714  vlib_buffer_t *b;
715  void *d;
716 
717  bi = *buffer_index;
718  if (bi == ~0
719  && 1 != vlib_buffer_alloc_from_free_list (vm, &bi, 1, free_list_index))
720  goto out_of_buffers;
721 
722  d = data;
723  n_left = n_data_bytes;
724  n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index);
725 
726  b = vlib_get_buffer (vm, bi);
727  b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
728 
729  /* Get to the end of the chain before we try to append data... */
730  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
731  b = vlib_get_buffer (vm, b->next_buffer);
732 
733  while (1)
734  {
735  u32 n;
736 
737  ASSERT (n_buffer_bytes >= b->current_length);
738  n_left_this_buffer =
739  n_buffer_bytes - (b->current_data + b->current_length);
740  n = clib_min (n_left_this_buffer, n_left);
742  n);
743  b->current_length += n;
744  n_left -= n;
745  if (n_left == 0)
746  break;
747 
748  d += n;
749  if (1 !=
751  free_list_index))
752  goto out_of_buffers;
753 
754  b->flags |= VLIB_BUFFER_NEXT_PRESENT;
755 
756  b = vlib_get_buffer (vm, b->next_buffer);
757  }
758 
759  *buffer_index = bi;
760  return 0;
761 
762 out_of_buffers:
763  clib_warning ("out of buffers");
764  return 1;
765 }
766 
767 u16
770  free_list_index,
772  vlib_buffer_t ** last, void *data,
773  u16 data_len)
774 {
775  vlib_buffer_t *l = *last;
776  u32 n_buffer_bytes =
777  vlib_buffer_free_list_buffer_size (vm, free_list_index);
778  u16 copied = 0;
779  ASSERT (n_buffer_bytes >= l->current_length + l->current_data);
780  while (data_len)
781  {
782  u16 max = n_buffer_bytes - l->current_length - l->current_data;
783  if (max == 0)
784  {
785  if (1 !=
787  free_list_index))
788  return copied;
789  *last = l = vlib_buffer_chain_buffer (vm, l, l->next_buffer);
790  max = n_buffer_bytes - l->current_length - l->current_data;
791  }
792 
793  u16 len = (data_len > max) ? max : data_len;
795  data + copied, len);
796  vlib_buffer_chain_increase_length (first, l, len);
797  data_len -= len;
798  copied += len;
799  }
800  return copied;
801 }
802 
803 u8
805 {
808  vlib_physmem_map_t *m = vlib_physmem_get_map (vm, physmem_map_index);
809  uword start = pointer_to_uword (m->base);
810  uword size = (uword) m->n_pages << m->log2_page_size;
811 
812  if (bm->buffer_mem_size == 0)
813  {
814  bm->buffer_mem_start = start;
815  bm->buffer_mem_size = size;
816  }
817  else if (start < bm->buffer_mem_start)
818  {
819  bm->buffer_mem_size += bm->buffer_mem_start - start;
820  bm->buffer_mem_start = start;
821  if (size > bm->buffer_mem_size)
822  bm->buffer_mem_size = size;
823  }
824  else if (start > bm->buffer_mem_start)
825  {
826  uword new_size = start - bm->buffer_mem_start + size;
827  if (new_size > bm->buffer_mem_size)
828  bm->buffer_mem_size = new_size;
829  }
830 
831  if ((u64) bm->buffer_mem_size >
832  ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
833  {
834  clib_panic ("buffer memory size out of range!");
835  }
836 
837  vec_add2 (bm->buffer_pools, p, 1);
838  p->start = start;
839  p->size = size;
840  p->physmem_map_index = physmem_map_index;
841 
842  ASSERT (p - bm->buffer_pools < 256);
843  return p - bm->buffer_pools;
844 }
845 
846 static u8 *
847 format_vlib_buffer_free_list (u8 * s, va_list * va)
848 {
850  u32 threadnum = va_arg (*va, u32);
851  uword bytes_alloc, bytes_free, n_free, size;
852 
853  if (!f)
854  return format (s, "%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s",
855  "Thread", "Name", "Index", "Size", "Alloc", "Free",
856  "#Alloc", "#Free");
857 
858  size = sizeof (vlib_buffer_t) + f->n_data_bytes;
859  n_free = vec_len (f->buffers);
860  bytes_alloc = size * f->n_alloc;
861  bytes_free = size * n_free;
862 
863  s = format (s, "%7d%30v%12d%12d%=12U%=12U%=12d%=12d", threadnum,
864  f->name, f->index, f->n_data_bytes,
865  format_memory_size, bytes_alloc,
866  format_memory_size, bytes_free, f->n_alloc, n_free);
867 
868  return s;
869 }
870 
871 static clib_error_t *
873  unformat_input_t * input, vlib_cli_command_t * cmd)
874 {
876  vlib_main_t *curr_vm;
877  u32 vm_index = 0;
878 
880 
881  do
882  {
883  curr_vm = vlib_mains[vm_index];
884 
885  /* *INDENT-OFF* */
886  pool_foreach (f, curr_vm->buffer_free_list_pool, ({
887  vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f, vm_index);
888  }));
889  /* *INDENT-ON* */
890 
891  vm_index++;
892  }
893  while (vm_index < vec_len (vlib_mains));
894 
895  return 0;
896 }
897 
898 /* *INDENT-OFF* */
899 VLIB_CLI_COMMAND (show_buffers_command, static) = {
900  .path = "show buffers",
901  .short_help = "Show packet buffer allocation",
902  .function = show_buffers,
903 };
904 /* *INDENT-ON* */
905 
906 clib_error_t *
908 {
910  clib_error_t *error;
911  u32 physmem_map_index;
912  u8 pool_index;
913  int log2_page_size = 0;
914 
915  buffer_log_default = vlib_log_register_class ("buffer", 0);
916 
917  if (vlib_buffer_callbacks)
918  {
919  /* external plugin has registered own buffer callbacks
920  so we just copy them and quit */
921  clib_memcpy_fast (&bm->cb, vlib_buffer_callbacks,
922  sizeof (vlib_buffer_callbacks_t));
923  bm->callbacks_registered = 1;
924  return 0;
925  }
926 
933 
934 retry:
935  error = vlib_physmem_shared_map_create (vm, "buffers",
936  vlib_buffer_physmem_sz,
937  log2_page_size,
939  &physmem_map_index);
940 
941  if (error && log2_page_size == 0)
942  {
943  vlib_log_warn (buffer_log_default, "%U", format_clib_error, error);
944  clib_error_free (error);
945  vlib_log_warn (buffer_log_default, "falling back to non-hugepage "
946  "backed buffer pool");
947  log2_page_size = min_log2 (clib_mem_get_page_size ());
948  goto retry;
949  }
950 
951  if (error)
952  return error;
953 
954  pool_index = vlib_buffer_register_physmem_map (vm, physmem_map_index);
955  vlib_buffer_pool_t *bp = vlib_buffer_pool_get (pool_index);
956  clib_spinlock_init (&bp->lock);
958  sizeof (vlib_buffer_t);
959 
960  return 0;
961 }
962 
963 static clib_error_t *
965 {
966  u32 size_in_mb;
967 
969  {
970  if (unformat (input, "memory-size-in-mb %d", &size_in_mb))
971  vlib_buffer_physmem_sz = size_in_mb << 20;
972  else
973  return unformat_parse_error (input);
974  }
975 
976  unformat_free (input);
977  return 0;
978 }
979 
981 
982 
983 /** @endcond */
984 /*
985  * fd.io coding-style-patch-verification: ON
986  *
987  * Local Variables:
988  * eval: (c-set-style "gnu")
989  * End:
990  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:227
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
vlib_main_t vlib_global_main
Definition: main.c:1857
u8 * format_clib_error(u8 *s, va_list *va)
Definition: error.c:191
u32 vlib_log_class_t
Definition: log.h:21
#define hash_set(h, key, value)
Definition: hash.h:255
u32 flags
Definition: vhost_user.h:115
#define vlib_log_warn(...)
Definition: log.h:51
#define clib_min(x, y)
Definition: clib.h:295
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
a
Definition: bitmap.h:538
static clib_error_t * vlib_buffers_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: buffer.c:964
static vlib_buffer_free_list_index_t vlib_buffer_create_free_list_helper(vlib_main_t *vm, u32 n_data_bytes, u32 is_public, u32 is_default, u8 *name)
Definition: buffer.c:355
u8 vlib_buffer_register_physmem_map(vlib_main_t *vm, u32 physmem_map_index)
Definition: buffer.c:804
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
Definition: buffer.c:768
unsigned long u64
Definition: types.h:89
vlib_buffer_callbacks_t cb
Definition: buffer.h:457
#define CLIB_PMALLOC_NUMA_LOCAL
Definition: pmalloc.h:24
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:576
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
Definition: buffer_funcs.h:919
vlib_buffer_t buffer_init_template
Definition: buffer.h:365
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
u32 physmem_map_index
Definition: buffer.h:423
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:50
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:564
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
static void vlib_buffer_free_no_next_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:657
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
u8 * va_format(u8 *s, const char *fmt, va_list *va)
Definition: format.c:387
vlib_physmem_map_t * vlib_physmem_get_map(vlib_main_t *vm, u32 index)
Definition: physmem.c:87
static clib_error_t * show_buffers(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: buffer.c:872
vlib_buffer_free_list_index_t index
Definition: buffer.h:368
unsigned char u8
Definition: types.h:56
static uword min_log2(uword x)
Definition: clib.h:144
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
Definition: buffer.c:62
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
static vlib_log_class_t buffer_log_default
Definition: buffer.c:59
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:602
static u8 * format_vlib_buffer_free_list(u8 *s, va_list *va)
Definition: buffer.c:847
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
#define static_always_inline
Definition: clib.h:99
static u32 vlib_buffer_physmem_sz
Definition: buffer.c:54
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
Definition: buffer.c:688
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:490
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.h:390
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
static_always_inline void * vlib_buffer_pool_get_buffer(vlib_main_t *vm, vlib_buffer_pool_t *bp)
Definition: buffer.c:484
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:188
uword buffer_mem_size
Definition: buffer.h:435
u8 * format_memory_size(u8 *s, va_list *va)
Definition: std-formats.c:193
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
Definition: buffer.h:409
u8 * format_hex_bytes(u8 *s, va_list *va)
Definition: std-formats.c:84
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:443
vhost_vring_state_t state
Definition: vhost_user.h:120
unsigned int u32
Definition: types.h:88
u8 * format_vlib_buffer(u8 *s, va_list *args)
Definition: buffer.c:79
#define VLIB_FRAME_SIZE
Definition: node.h:401
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:158
#define fl(x, y)
int callbacks_registered
Definition: buffer.h:458
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
uword size
void * vlib_set_buffer_free_callback(vlib_main_t *vm, void *fp)
Definition: buffer.c:556
clib_error_t * vlib_physmem_shared_map_create(vlib_main_t *vm, char *name, uword size, u32 log2_page_sz, u32 numa_node, u32 *map_index)
Definition: physmem.c:42
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
Definition: buffer.c:665
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:114
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
uword clib_mem_get_page_size(void)
Definition: mem.c:51
void(* vlib_buffer_delete_free_list_cb)(struct vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer.h:410
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:214
#define hash_free(h)
Definition: hash.h:310
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:442
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:436
static u8 * vlib_validate_buffer_helper(vlib_main_t *vm, u32 bi, uword follow_buffer_next, uword **unique_hash)
Definition: buffer.c:178
#define PREDICT_FALSE(x)
Definition: clib.h:111
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: main.h:123
u8 name[64]
Definition: memclnt.api:152
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
u8 len
Definition: ip_types.api:49
static void vlib_buffer_set_known_state(u32 buffer_index, vlib_buffer_known_state_t state)
Definition: buffer_funcs.h:374
#define pool_get_aligned(P, E, A)
Allocate an object E from a pool P with alignment A.
Definition: pool.h:230
#define vec_add_aligned(V, E, N, A)
Add N elements to end of vector V (no header, specified alignment)
Definition: vec.h:612
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:454
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:216
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
Definition: buffer.h:407
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.c:594
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:212
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.h:439
vlib_main_t * vm
Definition: buffer.c:301
vec_header_t h
Definition: buffer.c:300
u8 * format_vlib_buffer_contents(u8 *s, va_list *va)
Definition: buffer.c:161
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 bi, uword follow_buffer_next)
Definition: buffer.c:232
#define clib_warning(format, args...)
Definition: error.h:59
clib_spinlock_t lock
Definition: buffer.h:426
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:907
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
Definition: pool.h:283
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:241
vlib_buffer_known_state_t
Definition: buffer_funcs.h:347
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:155
#define hash_set1(h, key)
Definition: hash.h:258
#define hash_create(elts, value_bytes)
Definition: hash.h:696
void vlib_buffer_delete_free_list_internal(vlib_main_t *vm, vlib_buffer_free_list_index_t index)
Definition: buffer.c:458
#define ASSERT(truth)
vlib_buffer_free_cb_t * vlib_buffer_free_cb
Definition: buffer.h:408
static vlib_buffer_free_list_index_t vlib_buffer_get_free_list_index(vlib_buffer_t *b)
Definition: buffer_funcs.h:396
static u8 * format_vlib_buffer_known_state(u8 *s, va_list *args)
Definition: buffer.c:133
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:130
u8 * format_vlib_buffer_and_data(u8 *s, va_list *args)
Definition: buffer.c:121
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, vlib_buffer_free_list_index_t *index)
Definition: buffer_funcs.h:647
vector header structure
Definition: vec_bootstrap.h:55
static uword pointer_to_uword(const void *p)
Definition: types.h:131
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:142
#define clib_max(x, y)
Definition: clib.h:288
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, vlib_buffer_free_list_index_t index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:426
int vlib_buffer_add_data(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, u32 *buffer_index, void *data, u32 n_data_bytes)
Definition: buffer.c:709
#define unformat_parse_error(input)
Definition: format.h:268
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:156
static_always_inline void recycle_or_free(vlib_main_t *vm, vlib_buffer_main_t *bm, u32 bi, vlib_buffer_t *b, u32 follow_buffer_next)
Definition: buffer.c:566
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *last, u32 next_bi)
Definition: buffer_funcs.h:904
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:495
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
uword buffer_mem_start
Definition: buffer.h:434
u8 * vlib_validate_buffers(vlib_main_t *vm, u32 *buffers, uword next_buffer_stride, uword n_buffers, vlib_buffer_known_state_t known_state, uword follow_buffer_next)
Definition: buffer.c:239
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:671
#define clib_error_free(e)
Definition: error.h:86
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1468
static vlib_buffer_known_state_t vlib_buffer_is_known(u32 buffer_index)
Definition: buffer_funcs.h:363
vlib_buffer_main_t buffer_main
Definition: buffer.c:56
static uword clib_mem_is_vec(void *v)
Predicate function, says whether the supplied vector is a clib heap object.
Definition: vec.h:206
u8 data[0]
Packet data.
Definition: buffer.h:176
static void vlib_buffer_set_free_list_index(vlib_buffer_t *b, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:405
static_always_inline vlib_buffer_pool_t * vlib_buffer_pool_get(u8 buffer_pool_index)
Definition: buffer.h:464
static void * vlib_physmem_alloc_from_map(vlib_main_t *vm, u32 physmem_map_index, uword n_bytes, uword alignment)
Definition: physmem_funcs.h:79
u8 vlib_buffer_free_list_index_t
Definition: buffer.h:62
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:657
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:117
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:485
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
Definition: buffer.c:443
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:725
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:62
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:151
uword * free_list_by_size
Definition: buffer.h:447
#define clib_panic(format, args...)
Definition: error.h:72
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
Definition: buffer.c:316
#define vlib_panic_with_msg(vm, args...)
Definition: main.h:283
vlib_buffer_free_list_index_t vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:426
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
static uword vlib_buffer_fill_free_list_internal(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
Definition: buffer.c:492
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
static void vlib_buffer_free_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:650
static u32 vlib_buffer_round_size(u32 size)
Definition: buffer_funcs.h:390
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128