FD.io VPP  v18.04-17-g3a0d853
Vector Packet Processing
buffer.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.c: allocate/free network buffers.
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 /**
41  * @file
42  *
43  * Allocate/free network buffers.
44  */
45 
46 #include <vlib/vlib.h>
47 #include <vlib/unix/unix.h>
48 
49 vlib_buffer_callbacks_t *vlib_buffer_callbacks = 0;
50 static u32 vlib_buffer_physmem_sz = 32 << 20;
51 
53 
54 uword
56  vlib_buffer_t * b_first)
57 {
58  vlib_buffer_t *b = b_first;
59  uword l_first = b_first->current_length;
60  uword l = 0;
61  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
62  {
63  b = vlib_get_buffer (vm, b->next_buffer);
64  l += b->current_length;
65  }
67  b_first->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
68  return l + l_first;
69 }
70 
71 u8 *
72 format_vlib_buffer (u8 * s, va_list * args)
73 {
74  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
75  u32 indent = format_get_indent (s);
76  u8 *a = 0;
77 
78 #define _(bit, name, v) \
79  if (v && (b->flags & VLIB_BUFFER_##name)) \
80  a = format (a, "%s ", v);
82 #undef _
83  s = format (s, "current data %d, length %d, free-list %d, clone-count %u",
86 
87  if (b->flags & VLIB_BUFFER_TOTAL_LENGTH_VALID)
88  s = format (s, ", totlen-nifb %d",
90 
91  if (b->flags & VLIB_BUFFER_IS_TRACED)
92  s = format (s, ", trace 0x%x", b->trace_index);
93 
94  if (a)
95  s = format (s, "\n%U%v", format_white_space, indent, a);
96  vec_free (a);
97 
98  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
99  {
101  u32 next_buffer = b->next_buffer;
102  b = vlib_get_buffer (vm, next_buffer);
103 
104  s =
105  format (s, "\n%Unext-buffer 0x%x, segment length %d, clone-count %u",
106  format_white_space, indent, next_buffer, b->current_length,
107  b->n_add_refs);
108  }
109 
110  return s;
111 }
112 
113 u8 *
114 format_vlib_buffer_and_data (u8 * s, va_list * args)
115 {
116  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
117 
118  s = format (s, "%U, %U",
121 
122  return s;
123 }
124 
125 static u8 *
126 format_vlib_buffer_known_state (u8 * s, va_list * args)
127 {
129  char *t;
130 
131  switch (state)
132  {
133  case VLIB_BUFFER_UNKNOWN:
134  t = "unknown";
135  break;
136 
138  t = "known-allocated";
139  break;
140 
142  t = "known-free";
143  break;
144 
145  default:
146  t = "invalid";
147  break;
148  }
149 
150  return format (s, "%s", t);
151 }
152 
153 u8 *
154 format_vlib_buffer_contents (u8 * s, va_list * va)
155 {
156  vlib_main_t *vm = va_arg (*va, vlib_main_t *);
157  vlib_buffer_t *b = va_arg (*va, vlib_buffer_t *);
158 
159  while (1)
160  {
162  if (!(b->flags & VLIB_BUFFER_NEXT_PRESENT))
163  break;
164  b = vlib_get_buffer (vm, b->next_buffer);
165  }
166 
167  return s;
168 }
169 
170 static u8 *
172  u32 bi,
173  uword follow_buffer_next, uword ** unique_hash)
174 {
175  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
177 
180  return format (0, "unknown free list 0x%x",
182 
183  fl =
186 
187  if ((signed) b->current_data < (signed) -VLIB_BUFFER_PRE_DATA_SIZE)
188  return format (0, "current data %d before pre-data", b->current_data);
189 
190  if (b->current_data + b->current_length > fl->n_data_bytes)
191  return format (0, "%d-%d beyond end of buffer %d",
193 
194  if (follow_buffer_next && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
195  {
197  u8 *msg, *result;
198 
201  return format (0, "next 0x%x: %U",
203 
204  if (unique_hash)
205  {
206  if (hash_get (*unique_hash, b->next_buffer))
207  return format (0, "duplicate buffer 0x%x", b->next_buffer);
208 
209  hash_set1 (*unique_hash, b->next_buffer);
210  }
211 
212  msg = vlib_validate_buffer (vm, b->next_buffer, follow_buffer_next);
213  if (msg)
214  {
215  result = format (0, "next 0x%x: %v", b->next_buffer, msg);
216  vec_free (msg);
217  return result;
218  }
219  }
220 
221  return 0;
222 }
223 
224 u8 *
225 vlib_validate_buffer (vlib_main_t * vm, u32 bi, uword follow_buffer_next)
226 {
227  return vlib_validate_buffer_helper (vm, bi, follow_buffer_next,
228  /* unique_hash */ 0);
229 }
230 
231 u8 *
233  u32 * buffers,
234  uword next_buffer_stride,
235  uword n_buffers,
236  vlib_buffer_known_state_t known_state,
237  uword follow_buffer_next)
238 {
239  uword i, *hash;
240  u32 bi, *b = buffers;
242  u8 *msg = 0, *result = 0;
243 
244  hash = hash_create (0, 0);
245  for (i = 0; i < n_buffers; i++)
246  {
247  bi = b[0];
248  b += next_buffer_stride;
249 
250  /* Buffer is not unique. */
251  if (hash_get (hash, bi))
252  {
253  msg = format (0, "not unique");
254  goto done;
255  }
256 
257  k = vlib_buffer_is_known (bi);
258  if (k != known_state)
259  {
260  msg = format (0, "is %U; expected %U",
262  format_vlib_buffer_known_state, known_state);
263  goto done;
264  }
265 
266  msg = vlib_validate_buffer_helper (vm, bi, follow_buffer_next, &hash);
267  if (msg)
268  goto done;
269 
270  hash_set1 (hash, bi);
271  }
272 
273 done:
274  if (msg)
275  {
276  result = format (0, "0x%x: %v", bi, msg);
277  vec_free (msg);
278  }
279  hash_free (hash);
280  return result;
281 }
282 
283 /*
284  * Hand-craft a static vector w/ length 1, so vec_len(vlib_mains) =1
285  * and vlib_mains[0] = &vlib_global_main from the beginning of time.
286  *
287  * The only place which should ever expand vlib_mains is start_workers()
288  * in threads.c. It knows about the bootstrap vector.
289  */
290 /* *INDENT-OFF* */
291 static struct
292 {
295 } __attribute__ ((packed)) __bootstrap_vlib_main_vector
296  __attribute__ ((aligned (CLIB_CACHE_LINE_BYTES))) =
297 {
298  .h.len = 1,
299  .vm = &vlib_global_main,
300 };
301 /* *INDENT-ON* */
302 
303 vlib_main_t **vlib_mains = &__bootstrap_vlib_main_vector.vm;
304 
305 
306 /* When dubugging validate that given buffers are either known allocated
307  or known free. */
308 void
310  u32 * buffers,
311  uword n_buffers,
312  vlib_buffer_known_state_t expected_state)
313 {
314  u32 *b;
315  uword i, bi, is_free;
316 
317  if (CLIB_DEBUG == 0)
318  return;
319 
320  if (vlib_buffer_callbacks)
321  return;
322 
323  is_free = expected_state == VLIB_BUFFER_KNOWN_ALLOCATED;
324  b = buffers;
325  for (i = 0; i < n_buffers; i++)
326  {
328 
329  bi = b[0];
330  b += 1;
331  known = vlib_buffer_is_known (bi);
332  if (known != expected_state)
333  {
334  ASSERT (0);
336  (vm, "%s %U buffer 0x%x",
337  is_free ? "freeing" : "allocating",
338  format_vlib_buffer_known_state, known, bi);
339  }
340 
343  }
344 }
345 
346 /* Add buffer free list. */
349  u32 n_data_bytes,
350  u32 is_public, u32 is_default, u8 * name)
351 {
354  int i;
355 
356  ASSERT (vlib_get_thread_index () == 0);
357 
358  if (!is_default && pool_elts (vm->buffer_free_list_pool) == 0)
359  {
360  vlib_buffer_free_list_index_t default_free_free_list_index;
361 
362  /* *INDENT-OFF* */
363  default_free_free_list_index =
365  (vm,
366  /* default buffer size */ VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES,
367  /* is_public */ 1,
368  /* is_default */ 1,
369  (u8 *) "default");
370  /* *INDENT-ON* */
371  ASSERT (default_free_free_list_index ==
373 
374  if (n_data_bytes == VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES && is_public)
375  return default_free_free_list_index;
376  }
377 
379 
380  memset (f, 0, sizeof (f[0]));
381  f->index = f - vm->buffer_free_list_pool;
382  f->n_data_bytes = vlib_buffer_round_size (n_data_bytes);
384  f->buffer_pool_index = 0;
385  f->name = clib_mem_is_vec (name) ? name : format (0, "%s", name);
386 
387  /* Setup free buffer template. */
390 
391  if (is_public)
392  {
394  if (!p)
396  }
397 
398  for (i = 1; i < vec_len (vlib_mains); i++)
399  {
400  vlib_main_t *wvm = vlib_mains[i];
404  ASSERT (f - vm->buffer_free_list_pool ==
405  wf - wvm->buffer_free_list_pool);
406  wf[0] = f[0];
407  wf->buffers = 0;
408  wf->n_alloc = 0;
409  }
410 
411  return f->index;
412 }
413 
416  char *fmt, ...)
417 {
418  va_list va;
419  u8 *name;
420 
421  va_start (va, fmt);
422  name = va_format (0, fmt, &va);
423  va_end (va);
424 
425  return vlib_buffer_create_free_list_helper (vm, n_data_bytes,
426  /* is_public */ 0,
427  /* is_default */ 0,
428  name);
429 }
430 
431 static void
433 {
435 
438  vec_free (f->name);
439  vec_free (f->buffers);
440 
441  /* Poison it. */
442  memset (f, 0xab, sizeof (f[0]));
443 }
444 
445 /* Add buffer free list. */
446 void
449 {
451  int i;
452 
453  ASSERT (vlib_get_thread_index () == 0);
454 
455  f = vlib_buffer_get_free_list (vm, index);
456 
457  ASSERT (vec_len (f->buffers) == f->n_alloc);
458 
459  del_free_list (vm, f);
460 
462 
463  for (i = 1; i < vec_len (vlib_mains); i++)
464  {
465  vlib_main_t *wvm = vlib_mains[i];
466  f = vlib_buffer_get_free_list (vlib_mains[i], index);
467  del_free_list (wvm, f);
469  }
470 }
471 
474 {
475  uword slot, page, addr;
476 
477  if (PREDICT_FALSE (bp->n_elts == bp->n_used))
478  {
479  clib_spinlock_unlock (&bp->lock);
480  return 0;
481  }
482  slot = bp->next_clear;
483  bp->bitmap = clib_bitmap_set (bp->bitmap, slot, 1);
484  bp->next_clear = clib_bitmap_next_clear (bp->bitmap, slot + 1);
485  bp->n_used++;
486 
487  page = slot / bp->buffers_per_page;
488  slot -= page * bp->buffers_per_page;
489 
490  addr = bp->start + (page << bp->log2_page_size) + slot * bp->buffer_size;
491 
492  return uword_to_pointer (addr, void *);
493 }
494 
495 /* Make sure free list has at least given number of free buffers. */
496 static uword
499  uword min_free_buffers)
500 {
501  vlib_buffer_t *b;
503  int n;
504  u32 *bi;
505  u32 n_alloc = 0;
506 
507  /* Already have enough free buffers on free list? */
508  n = min_free_buffers - vec_len (fl->buffers);
509  if (n <= 0)
510  return min_free_buffers;
511 
512  if (vec_len (bp->buffers) > 0)
513  {
514  int n_copy, n_left;
515  clib_spinlock_lock (&bp->lock);
516  n_copy = clib_min (vec_len (bp->buffers), n);
517  n_left = vec_len (bp->buffers) - n_copy;
518  vec_add_aligned (fl->buffers, bp->buffers + n_left, n_copy,
520  _vec_len (bp->buffers) = n_left;
521  clib_spinlock_unlock (&bp->lock);
522  n = min_free_buffers - vec_len (fl->buffers);
523  if (n <= 0)
524  return min_free_buffers;
525  }
526 
527  /* Always allocate round number of buffers. */
528  n = round_pow2 (n, CLIB_CACHE_LINE_BYTES / sizeof (u32));
529 
530  /* Always allocate new buffers in reasonably large sized chunks. */
531  n = clib_max (n, fl->min_n_buffers_each_alloc);
532 
533  clib_spinlock_lock (&bp->lock);
534  while (n_alloc < n)
535  {
536  if ((b = vlib_buffer_pool_get_buffer (bp)) == 0)
537  goto done;
538 
539  n_alloc += 1;
540 
542  bi[0] = vlib_get_buffer_index (vm, b);
543 
544  if (CLIB_DEBUG > 0)
546 
547  memset (b, 0, sizeof (vlib_buffer_t));
549 
550  if (fl->buffer_init_function)
551  fl->buffer_init_function (vm, fl, bi, 1);
552  }
553 
554 done:
555  clib_spinlock_unlock (&bp->lock);
556  fl->n_alloc += n_alloc;
557  return n_alloc;
558 }
559 
560 void *
562 {
564  void *rv = bm->buffer_free_callback;
565 
566  bm->buffer_free_callback = fp;
567  return rv;
568 }
569 
572  vlib_buffer_t * b, u32 follow_buffer_next)
573 {
576  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
577 
578  /* The only current use of this callback:
579  * multicast recycle */
581  {
582  int j;
583 
584  vlib_buffer_add_to_free_list (vm, fl, bi,
585  (b->flags & VLIB_BUFFER_RECYCLE) == 0);
586  for (j = 0; j < vec_len (vm->buffer_announce_list); j++)
587  {
588  if (fl == vm->buffer_announce_list[j])
589  goto already_announced;
590  }
591  vec_add1 (vm->buffer_announce_list, fl);
592  already_announced:
593  ;
594  }
595  else
596  {
597  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
598  {
599  u32 flags, next;
600 
601  do
602  {
603  vlib_buffer_t *nb = vlib_get_buffer (vm, bi);
604  flags = nb->flags;
605  next = nb->next_buffer;
606  if (nb->n_add_refs)
607  nb->n_add_refs--;
608  else
609  {
612  vlib_buffer_add_to_free_list (vm, fl, bi, 1);
613  }
614  bi = next;
615  }
616  while (follow_buffer_next && (flags & VLIB_BUFFER_NEXT_PRESENT));
617 
618  }
619  }
620 }
621 
624  u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
625 {
627  vlib_buffer_t *p, *b0, *b1, *b2, *b3;
628  int i = 0;
629  u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
630  u32 follow_buffer_next);
631 
632  cb = bm->buffer_free_callback;
633 
634  if (PREDICT_FALSE (cb != 0))
635  n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
636 
637  if (!n_buffers)
638  return;
639 
640  while (i + 11 < n_buffers)
641  {
642  p = vlib_get_buffer (vm, buffers[i + 8]);
643  vlib_prefetch_buffer_header (p, LOAD);
644  p = vlib_get_buffer (vm, buffers[i + 9]);
645  vlib_prefetch_buffer_header (p, LOAD);
646  p = vlib_get_buffer (vm, buffers[i + 10]);
647  vlib_prefetch_buffer_header (p, LOAD);
648  p = vlib_get_buffer (vm, buffers[i + 11]);
649  vlib_prefetch_buffer_header (p, LOAD);
650 
651  b0 = vlib_get_buffer (vm, buffers[i]);
652  b1 = vlib_get_buffer (vm, buffers[i + 1]);
653  b2 = vlib_get_buffer (vm, buffers[i + 2]);
654  b3 = vlib_get_buffer (vm, buffers[i + 3]);
655 
660 
661  recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
662  recycle_or_free (vm, bm, buffers[i + 1], b1, follow_buffer_next);
663  recycle_or_free (vm, bm, buffers[i + 2], b2, follow_buffer_next);
664  recycle_or_free (vm, bm, buffers[i + 3], b3, follow_buffer_next);
665 
666  i += 4;
667  }
668 
669  while (i < n_buffers)
670  {
671  b0 = vlib_get_buffer (vm, buffers[i]);
673  recycle_or_free (vm, bm, buffers[i], b0, follow_buffer_next);
674  i++;
675  }
676 
677  if (vec_len (vm->buffer_announce_list))
678  {
680  for (i = 0; i < vec_len (vm->buffer_announce_list); i++)
681  {
682  fl = vm->buffer_announce_list[i];
684  }
685  _vec_len (vm->buffer_announce_list) = 0;
686  }
687 }
688 
689 static void
690 vlib_buffer_free_internal (vlib_main_t * vm, u32 * buffers, u32 n_buffers)
691 {
692  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
693  1);
694 }
695 
696 static void
698  u32 n_buffers)
699 {
700  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
701  0);
702 }
703 
704 /* Copy template packet data into buffers as they are allocated. */
705 static void __attribute__ ((unused))
708  u32 * buffers, u32 n_buffers)
709 {
711  uword_to_pointer (fl->buffer_init_function_opaque,
713  uword i;
714 
715  for (i = 0; i < n_buffers; i++)
716  {
717  vlib_buffer_t *b = vlib_get_buffer (vm, buffers[i]);
720  b->current_length);
721  }
722 }
723 
724 void
727  void *packet_data,
728  uword n_packet_data_bytes,
729  uword min_n_buffers_each_alloc, char *fmt, ...)
730 {
732  va_list va;
733  u8 *name;
735 
736  va_start (va, fmt);
737  name = va_format (0, fmt, &va);
738  va_end (va);
739 
741  bm->cb.vlib_packet_template_init_cb (vm, (void *) t, packet_data,
742  n_packet_data_bytes,
743  min_n_buffers_each_alloc, name);
744 
746 
747  memset (t, 0, sizeof (t[0]));
748 
749  vec_add (t->packet_data, packet_data, n_packet_data_bytes);
750  t->min_n_buffers_each_alloc = min_n_buffers_each_alloc;
751 
753  (vm, n_packet_data_bytes,
754  /* is_public */ 1,
755  /* is_default */ 0,
756  name);
757 
758  ASSERT (t->free_list_index != 0);
761 
764 
766  fl->buffer_init_template.current_length = n_packet_data_bytes;
767  fl->buffer_init_template.flags = 0;
770 }
771 
772 void *
774  vlib_packet_template_t * t, u32 * bi_result)
775 {
776  u32 bi;
777  vlib_buffer_t *b;
778 
779  if (vlib_buffer_alloc (vm, &bi, 1) != 1)
780  return 0;
781 
782  *bi_result = bi;
783 
784  b = vlib_get_buffer (vm, bi);
786  t->packet_data, vec_len (t->packet_data));
788 
789  return b->data;
790 }
791 
792 void
795 {
797  word l = vec_len (t->packet_data);
798  word n_alloc;
799 
800  ASSERT (l > 0);
801  ASSERT (vec_len (t->free_buffers) == 0);
802 
803  vec_validate (t->free_buffers, n - 1);
805  n, t->free_list_index);
806  _vec_len (t->free_buffers) = n_alloc;
807 }
808 
809 /* Append given data to end of buffer, possibly allocating new buffers. */
810 u32
812  vlib_buffer_free_list_index_t free_list_index,
813  u32 buffer_index, void *data, u32 n_data_bytes)
814 {
815  u32 n_buffer_bytes, n_left, n_left_this_buffer, bi;
816  vlib_buffer_t *b;
817  void *d;
818 
819  bi = buffer_index;
820  if (bi == ~0
821  && 1 != vlib_buffer_alloc_from_free_list (vm, &bi, 1, free_list_index))
822  goto out_of_buffers;
823 
824  d = data;
825  n_left = n_data_bytes;
826  n_buffer_bytes = vlib_buffer_free_list_buffer_size (vm, free_list_index);
827 
828  b = vlib_get_buffer (vm, bi);
829  b->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
830 
831  /* Get to the end of the chain before we try to append data... */
832  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
833  b = vlib_get_buffer (vm, b->next_buffer);
834 
835  while (1)
836  {
837  u32 n;
838 
839  ASSERT (n_buffer_bytes >= b->current_length);
840  n_left_this_buffer =
841  n_buffer_bytes - (b->current_data + b->current_length);
842  n = clib_min (n_left_this_buffer, n_left);
844  b->current_length += n;
845  n_left -= n;
846  if (n_left == 0)
847  break;
848 
849  d += n;
850  if (1 !=
852  free_list_index))
853  goto out_of_buffers;
854 
855  b->flags |= VLIB_BUFFER_NEXT_PRESENT;
856 
857  b = vlib_get_buffer (vm, b->next_buffer);
858  }
859 
860  return bi;
861 
862 out_of_buffers:
863  clib_error ("out of buffers");
864  return bi;
865 }
866 
867 u16
870  free_list_index,
872  vlib_buffer_t ** last, void *data,
873  u16 data_len)
874 {
875  vlib_buffer_t *l = *last;
876  u32 n_buffer_bytes =
877  vlib_buffer_free_list_buffer_size (vm, free_list_index);
878  u16 copied = 0;
879  ASSERT (n_buffer_bytes >= l->current_length + l->current_data);
880  while (data_len)
881  {
882  u16 max = n_buffer_bytes - l->current_length - l->current_data;
883  if (max == 0)
884  {
885  if (1 !=
887  free_list_index))
888  return copied;
889  *last = l = vlib_buffer_chain_buffer (vm, first, l, l->next_buffer);
890  max = n_buffer_bytes - l->current_length - l->current_data;
891  }
892 
893  u16 len = (data_len > max) ? max : data_len;
895  data + copied, len);
896  vlib_buffer_chain_increase_length (first, l, len);
897  data_len -= len;
898  copied += len;
899  }
900  return copied;
901 }
902 
903 u8
905  u16 buffer_size)
906 {
910  uword start = pointer_to_uword (pr->mem);
911  uword size = pr->size;
912 
913  if (bm->buffer_mem_size == 0)
914  {
915  bm->buffer_mem_start = start;
916  bm->buffer_mem_size = size;
917  }
918  else if (start < bm->buffer_mem_start)
919  {
920  bm->buffer_mem_size += bm->buffer_mem_start - start;
921  bm->buffer_mem_start = start;
922  if (size > bm->buffer_mem_size)
923  bm->buffer_mem_size = size;
924  }
925  else if (start > bm->buffer_mem_start)
926  {
927  uword new_size = start - bm->buffer_mem_start + size;
928  if (new_size > bm->buffer_mem_size)
929  bm->buffer_mem_size = new_size;
930  }
931 
932  if ((u64) bm->buffer_mem_size >
933  ((u64) 1 << (32 + CLIB_LOG2_CACHE_LINE_BYTES)))
934  {
935  clib_panic ("buffer memory size out of range!");
936  }
937 
938  vec_add2 (bm->buffer_pools, p, 1);
939  p->start = start;
940  p->size = size;
941  p->physmem_region = pri;
942 
943  if (buffer_size == 0)
944  goto done;
945 
947  p->buffer_size = buffer_size;
948  p->buffers_per_page = (1 << pr->log2_page_size) / p->buffer_size;
949  p->n_elts = p->buffers_per_page * pr->n_pages;
950  p->n_used = 0;
951  clib_spinlock_init (&p->lock);
952 done:
953  ASSERT (p - bm->buffer_pools < 256);
954  return p - bm->buffer_pools;
955 }
956 
957 static u8 *
958 format_vlib_buffer_free_list (u8 * s, va_list * va)
959 {
961  u32 threadnum = va_arg (*va, u32);
962  uword bytes_alloc, bytes_free, n_free, size;
963 
964  if (!f)
965  return format (s, "%=7s%=30s%=12s%=12s%=12s%=12s%=12s%=12s",
966  "Thread", "Name", "Index", "Size", "Alloc", "Free",
967  "#Alloc", "#Free");
968 
969  size = sizeof (vlib_buffer_t) + f->n_data_bytes;
970  n_free = vec_len (f->buffers);
971  bytes_alloc = size * f->n_alloc;
972  bytes_free = size * n_free;
973 
974  s = format (s, "%7d%30v%12d%12d%=12U%=12U%=12d%=12d", threadnum,
975  f->name, f->index, f->n_data_bytes,
976  format_memory_size, bytes_alloc,
977  format_memory_size, bytes_free, f->n_alloc, n_free);
978 
979  return s;
980 }
981 
982 static clib_error_t *
984  unformat_input_t * input, vlib_cli_command_t * cmd)
985 {
987  vlib_main_t *curr_vm;
988  u32 vm_index = 0;
989 
991 
992  do
993  {
994  curr_vm = vlib_mains[vm_index];
995 
996  /* *INDENT-OFF* */
997  pool_foreach (f, curr_vm->buffer_free_list_pool, ({
998  vlib_cli_output (vm, "%U", format_vlib_buffer_free_list, f, vm_index);
999  }));
1000  /* *INDENT-ON* */
1001 
1002  vm_index++;
1003  }
1004  while (vm_index < vec_len (vlib_mains));
1005 
1006  return 0;
1007 }
1008 
1009 /* *INDENT-OFF* */
1010 VLIB_CLI_COMMAND (show_buffers_command, static) = {
1011  .path = "show buffers",
1012  .short_help = "Show packet buffer allocation",
1013  .function = show_buffers,
1014 };
1015 /* *INDENT-ON* */
1016 
1017 clib_error_t *
1019 {
1022  clib_error_t *error;
1023 
1024  if (vlib_buffer_callbacks)
1025  {
1026  /* external plugin has registered own buffer callbacks
1027  so we just copy them and quit */
1028  clib_memcpy (&bm->cb, vlib_buffer_callbacks,
1029  sizeof (vlib_buffer_callbacks_t));
1030  bm->callbacks_registered = 1;
1031  return 0;
1032  }
1033 
1040 
1041  /* allocate default region */
1042  error = vlib_physmem_region_alloc (vm, "buffers",
1045  VLIB_PHYSMEM_F_HUGETLB, &pri);
1046 
1047  if (error == 0)
1048  goto done;
1049 
1050  clib_error_free (error);
1051 
1052  error = vlib_physmem_region_alloc (vm, "buffers",
1054  VLIB_PHYSMEM_F_SHARED, &pri);
1055 done:
1056  if (error == 0)
1057  vlib_buffer_pool_create (vm, pri, sizeof (vlib_buffer_t) +
1059  return error;
1060 }
1061 
1062 static clib_error_t *
1064 {
1065  u32 size_in_mb;
1066 
1067  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1068  {
1069  if (unformat (input, "memory-size-in-mb %d", &size_in_mb))
1070  vlib_buffer_physmem_sz = size_in_mb << 20;
1071  else
1072  return unformat_parse_error (input);
1073  }
1074 
1075  unformat_free (input);
1076  return 0;
1077 }
1078 
1080 
1081 
1082 /** @endcond */
1083 /*
1084  * fd.io coding-style-patch-verification: ON
1085  *
1086  * Local Variables:
1087  * eval: (c-set-style "gnu")
1088  * End:
1089  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:434
vlib_physmem_region_index_t physmem_region
Definition: buffer.h:420
vlib_main_t vlib_global_main
Definition: main.c:1642
u32 vlib_buffer_add_data(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, u32 buffer_index, void *data, u32 n_data_bytes)
Definition: buffer.c:811
#define hash_set(h, key, value)
Definition: hash.h:254
#define clib_min(x, y)
Definition: clib.h:340
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
static uword clib_bitmap_next_clear(uword *ai, uword i)
Return the next clear bit in a bitmap starting at bit i.
Definition: bitmap.h:659
a
Definition: bitmap.h:516
#define VLIB_PHYSMEM_F_HUGETLB
Definition: physmem.h:58
static clib_error_t * vlib_buffers_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: buffer.c:1063
static vlib_buffer_free_list_index_t vlib_buffer_create_free_list_helper(vlib_main_t *vm, u32 n_data_bytes, u32 is_public, u32 is_default, u8 *name)
Definition: buffer.c:348
#define PREDICT_TRUE(x)
Definition: clib.h:106
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
Definition: buffer.c:868
#define clib_error(format, args...)
Definition: error.h:62
vlib_buffer_callbacks_t cb
Definition: buffer.h:461
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, u32 next_bi)
Definition: buffer_funcs.h:784
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:571
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
Definition: buffer_funcs.h:801
vlib_buffer_t buffer_init_template
Definition: buffer.h:351
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:50
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:520
static clib_error_t * vlib_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:559
int i
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
static u32 format_get_indent(u8 *s)
Definition: format.h:72
static void vlib_buffer_free_no_next_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:697
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
u8 * va_format(u8 *s, const char *fmt, va_list *va)
Definition: format.c:387
uword log2_page_size
Definition: buffer.h:419
static clib_error_t * show_buffers(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: buffer.c:983
vlib_buffer_free_list_index_t index
Definition: buffer.h:354
vlib_main_t ** vlib_mains
Definition: buffer.c:303
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
Definition: buffer.c:55
vlib_buffer_free_list_index_t free_list_index
Definition: buffer_funcs.h:856
void vlib_packet_template_get_packet_helper(vlib_main_t *vm, vlib_packet_template_t *t)
Definition: buffer.c:793
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:212
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:597
#define VLIB_PHYSMEM_F_SHARED
Definition: physmem.h:59
static u8 * format_vlib_buffer_free_list(u8 *s, va_list *va)
Definition: buffer.c:958
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:104
#define static_always_inline
Definition: clib.h:93
static u32 vlib_buffer_physmem_sz
Definition: buffer.c:50
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
Definition: buffer.c:773
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:440
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.h:376
u8 vlib_buffer_pool_create(vlib_main_t *vm, vlib_physmem_region_index_t pri, u16 buffer_size)
Definition: buffer.c:904
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:191
uword buffer_mem_size
Definition: buffer.h:439
u8 * format_memory_size(u8 *s, va_list *va)
Definition: std-formats.c:193
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
Definition: buffer.h:400
u8 * format_hex_bytes(u8 *s, va_list *va)
Definition: std-formats.c:84
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:447
unsigned long u64
Definition: types.h:89
u8 * format_vlib_buffer(u8 *s, va_list *args)
Definition: buffer.c:72
#define VLIB_FRAME_SIZE
Definition: node.h:328
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:74
#define fl(x, y)
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
Definition: physmem_funcs.h:44
int callbacks_registered
Definition: buffer.h:462
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
#define hash_get(h, key)
Definition: hash.h:248
uword next_clear
Definition: buffer.h:428
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:461
u32 size
void * vlib_set_buffer_free_callback(vlib_main_t *vm, void *fp)
Definition: buffer.c:561
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
Definition: buffer.c:725
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:108
struct _unformat_input_t unformat_input_t
void(* vlib_buffer_delete_free_list_cb)(struct vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer.h:407
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:209
#define hash_free(h)
Definition: hash.h:309
#define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX
Definition: buffer.h:446
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:273
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:440
static u8 * vlib_validate_buffer_helper(vlib_main_t *vm, u32 bi, uword follow_buffer_next, uword **unique_hash)
Definition: buffer.c:171
#define PREDICT_FALSE(x)
Definition: clib.h:105
static_always_inline void * vlib_buffer_pool_get_buffer(vlib_buffer_pool_t *bp)
Definition: buffer.c:473
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: main.h:108
static void vlib_packet_template_buffer_init(vlib_main_t *vm, vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.c:706
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
Definition: buffer_funcs.h:955
#define uword_to_pointer(u, type)
Definition: types.h:136
static void vlib_buffer_set_known_state(u32 buffer_index, vlib_buffer_known_state_t state)
Definition: buffer_funcs.h:236
#define pool_get_aligned(P, E, A)
Allocate an object E from a pool P (general version).
Definition: pool.h:188
#define vec_add_aligned(V, E, N, A)
Add N elements to end of vector V (no header, specified alignment)
Definition: vec.h:607
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:458
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:140
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
Definition: buffer.h:398
#define UNFORMAT_END_OF_INPUT
Definition: format.h:143
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.c:623
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:221
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.h:443
vlib_main_t * vm
Definition: buffer.c:294
vec_header_t h
Definition: buffer.c:293
u8 * format_vlib_buffer_contents(u8 *s, va_list *va)
Definition: buffer.c:154
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 bi, uword follow_buffer_next)
Definition: buffer.c:225
#define clib_memcpy(a, b, c)
Definition: string.h:75
clib_spinlock_t lock
Definition: buffer.h:430
void(* buffers_added_to_freelist_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl)
Definition: buffer.h:383
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:1018
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
Definition: pool.h:270
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:286
vlib_buffer_known_state_t
Definition: buffer_funcs.h:209
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
uword * bitmap
Definition: buffer.h:429
#define hash_set1(h, key)
Definition: hash.h:257
#define hash_create(elts, value_bytes)
Definition: hash.h:681
void vlib_buffer_delete_free_list_internal(vlib_main_t *vm, vlib_buffer_free_list_index_t index)
Definition: buffer.c:447
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
vlib_buffer_free_cb_t * vlib_buffer_free_cb
Definition: buffer.h:399
static vlib_buffer_free_list_index_t vlib_buffer_get_free_list_index(vlib_buffer_t *b)
Definition: buffer_funcs.h:258
static u8 * format_vlib_buffer_known_state(u8 *s, va_list *args)
Definition: buffer.c:126
void(* vlib_packet_template_init_cb)(struct vlib_main_t *vm, void *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, u8 *name)
Definition: buffer.h:401
uword buffers_per_page
Definition: buffer.h:425
vhost_vring_state_t state
Definition: vhost-user.h:82
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:126
u8 * format_vlib_buffer_and_data(u8 *s, va_list *args)
Definition: buffer.c:114
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, vlib_buffer_free_list_index_t *index)
Definition: buffer_funcs.h:509
vector header structure
Definition: vec_bootstrap.h:55
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:141
#define clib_max(x, y)
Definition: clib.h:333
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static u32 vlib_buffer_alloc_from_free_list(vlib_main_t *vm, u32 *buffers, u32 n_buffers, vlib_buffer_free_list_index_t index)
Allocate buffers from specific freelist into supplied array.
Definition: buffer_funcs.h:288
u64 uword
Definition: types.h:112
#define unformat_parse_error(input)
Definition: format.h:267
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:159
unsigned short u16
Definition: types.h:57
i64 word
Definition: types.h:111
static_always_inline void recycle_or_free(vlib_main_t *vm, vlib_buffer_main_t *bm, u32 bi, vlib_buffer_t *b, u32 follow_buffer_next)
Definition: buffer.c:571
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:553
static void unformat_free(unformat_input_t *i)
Definition: format.h:161
uword buffer_mem_start
Definition: buffer.h:438
u8 * vlib_validate_buffers(vlib_main_t *vm, u32 *buffers, uword next_buffer_stride, uword n_buffers, vlib_buffer_known_state_t known_state, uword follow_buffer_next)
Definition: buffer.c:232
static u32 vlib_buffer_free_list_buffer_size(vlib_main_t *vm, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:533
#define clib_error_free(e)
Definition: error.h:86
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
Definition: buffer_funcs.h:915
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1497
static vlib_buffer_known_state_t vlib_buffer_is_known(u32 buffer_index)
Definition: buffer_funcs.h:225
vlib_buffer_main_t buffer_main
Definition: buffer.c:52
static uword clib_mem_is_vec(void *v)
Predicate function, says whether the supplied vector is a clib heap object.
Definition: vec.h:201
u8 data[0]
Packet data.
Definition: buffer.h:179
static void vlib_buffer_set_free_list_index(vlib_buffer_t *b, vlib_buffer_free_list_index_t index)
Definition: buffer_funcs.h:267
static_always_inline vlib_buffer_pool_t * vlib_buffer_pool_get(u8 buffer_pool_index)
Definition: buffer.h:468
u8 vlib_buffer_free_list_index_t
Definition: buffer.h:54
uword buffer_init_function_opaque
Definition: buffer.h:385
vhost_vring_addr_t addr
Definition: vhost-user.h:83
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:519
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:347
u8 vlib_physmem_region_index_t
Definition: physmem.h:43
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
Definition: buffer.c:432
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:680
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:154
uword * free_list_by_size
Definition: buffer.h:451
#define clib_panic(format, args...)
Definition: error.h:72
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
Definition: buffer.c:309
vlib_buffer_free_list_t ** buffer_announce_list
Definition: main.h:111
#define vlib_panic_with_msg(vm, args...)
Definition: main.h:278
vlib_buffer_free_list_index_t vlib_buffer_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:415
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
static uword vlib_buffer_fill_free_list_internal(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
Definition: buffer.c:497
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:169
static void vlib_buffer_free_internal(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:690
static u32 vlib_buffer_round_size(u32 size)
Definition: buffer_funcs.h:252
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128