FD.io VPP  v18.07-34-g55fbdb9
Vector Packet Processing
buffer.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.c: allocate/free network buffers.
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 /**
41  * @file
42  *
43  * Allocate/free network buffers.
44  */
45 
46 #include <unistd.h>
47 #include <linux/vfio.h>
48 #include <sys/ioctl.h>
49 
50 #include <rte_config.h>
51 
52 #include <rte_common.h>
53 #include <rte_log.h>
54 #include <rte_memory.h>
55 #include <rte_memzone.h>
56 #include <rte_tailq.h>
57 #include <rte_eal.h>
58 #include <rte_per_lcore.h>
59 #include <rte_launch.h>
60 #include <rte_atomic.h>
61 #include <rte_cycles.h>
62 #include <rte_prefetch.h>
63 #include <rte_lcore.h>
64 #include <rte_per_lcore.h>
65 #include <rte_branch_prediction.h>
66 #include <rte_interrupts.h>
67 #include <rte_pci.h>
68 #include <rte_random.h>
69 #include <rte_debug.h>
70 #include <rte_ether.h>
71 #include <rte_ethdev.h>
72 #include <rte_ring.h>
73 #include <rte_mempool.h>
74 #include <rte_mbuf.h>
75 #include <rte_version.h>
76 
77 #include <vlib/vlib.h>
78 #include <vlib/unix/unix.h>
79 #include <vlib/pci/pci.h>
80 #include <vlib/linux/vfio.h>
81 #include <vnet/vnet.h>
82 #include <dpdk/device/dpdk.h>
83 #include <dpdk/device/dpdk_priv.h>
84 
85 STATIC_ASSERT (VLIB_BUFFER_PRE_DATA_SIZE == RTE_PKTMBUF_HEADROOM,
86  "VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM");
87 
88 typedef struct
89 {
90  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
91  struct rte_mbuf **mbuf_alloc_list;
93 
94 typedef struct
95 {
99 
101 
104  int maybe_next)
105 {
106  struct rte_mbuf *mb;
107  u32 next, flags;
108 
109 next:
110  flags = b->flags;
111  next = b->next_buffer;
112  mb = rte_mbuf_from_vlib_buffer (b);
113 
114  if (PREDICT_FALSE (b->n_add_refs))
115  {
116  rte_mbuf_refcnt_update (mb, b->n_add_refs);
117  b->n_add_refs = 0;
118  }
119 
120  if ((mb = rte_pktmbuf_prefree_seg (mb)))
121  rte_mempool_put (mb->pool, mb);
122 
123  if (maybe_next && (flags & VLIB_BUFFER_NEXT_PRESENT))
124  {
125  b = vlib_get_buffer (vm, next);
126  goto next;
127  }
128 }
129 
130 #ifndef CLIB_MARCH_VARIANT
131 static void
133 {
134  u32 i;
135  vlib_buffer_t *b;
136  u32 thread_index = vlib_get_thread_index ();
137 
138  for (i = 0; i < vec_len (f->buffers); i++)
139  {
140  b = vlib_get_buffer (vm, f->buffers[i]);
141  dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
142  }
143 
144  vec_free (f->name);
145  vec_free (f->buffers);
146  /* Poison it. */
147  memset (f, 0xab, sizeof (f[0]));
148 }
149 
150 /* Add buffer free list. */
151 static void
153  vlib_buffer_free_list_index_t free_list_index)
154 {
156  int i;
157 
158  ASSERT (vlib_get_thread_index () == 0);
159 
160  f = vlib_buffer_get_free_list (vm, free_list_index);
161 
162  del_free_list (vm, f);
163 
165 
166  for (i = 1; i < vec_len (vlib_mains); i++)
167  {
168  vlib_main_t *wvm = vlib_mains[i];
169  f = vlib_buffer_get_free_list (vlib_mains[i], free_list_index);
170  del_free_list (wvm, f);
172  }
173 }
174 #endif
175 
176 /* Make sure free list has at least given number of free buffers. */
177 uword
180  uword min_free_buffers)
181 {
182  dpdk_main_t *dm = &dpdk_main;
184  struct rte_mbuf **mb;
185  uword n_left, first;
186  word n_alloc;
187  unsigned socket_id = rte_socket_id ();
188  u32 thread_index = vlib_get_thread_index ();
189  dpdk_buffer_per_thread_data *d = vec_elt_at_index (dbm->ptd, thread_index);
190  struct rte_mempool *rmp = dm->pktmbuf_pools[socket_id];
191  dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp);
192  vlib_buffer_t bt;
193  u32 *bi;
194 
195  /* Too early? */
196  if (PREDICT_FALSE (rmp == 0))
197  return 0;
198 
199  /* Already have enough free buffers on free list? */
200  n_alloc = min_free_buffers - vec_len (fl->buffers);
201  if (n_alloc <= 0)
202  return min_free_buffers;
203 
204  /* Always allocate round number of buffers. */
205  n_alloc = round_pow2 (n_alloc, CLIB_CACHE_LINE_BYTES / sizeof (u32));
206 
207  /* Always allocate new buffers in reasonably large sized chunks. */
208  n_alloc = clib_max (n_alloc, fl->min_n_buffers_each_alloc);
209 
210  vec_validate_aligned (d->mbuf_alloc_list, n_alloc - 1,
212 
213  if (rte_mempool_get_bulk (rmp, (void *) d->mbuf_alloc_list, n_alloc) < 0)
214  return 0;
215 
216  memset (&bt, 0, sizeof (vlib_buffer_t));
219 
220  _vec_len (d->mbuf_alloc_list) = n_alloc;
221 
222  first = vec_len (fl->buffers);
223  vec_resize_aligned (fl->buffers, n_alloc, CLIB_CACHE_LINE_BYTES);
224 
225  n_left = n_alloc;
226  mb = d->mbuf_alloc_list;
227  bi = fl->buffers + first;
228 
229  ASSERT (n_left % 8 == 0);
230 
231  while (n_left >= 8)
232  {
233  if (PREDICT_FALSE (n_left < 24))
234  goto no_prefetch;
235 
244 
245  no_prefetch:
246  vlib_get_buffer_indices_with_offset (vm, (void **) mb, bi, 8,
247  sizeof (struct rte_mbuf));
251  vlib_buffer_from_rte_mbuf (mb[3]), &bt);
255  vlib_buffer_from_rte_mbuf (mb[7]), &bt);
256 
257  n_left -= 8;
258  mb += 8;
259  bi += 8;
260  }
261 
262  if (fl->buffer_init_function)
263  fl->buffer_init_function (vm, fl, fl->buffers + first, n_alloc);
264 
265  fl->n_alloc += n_alloc;
266 
267  return n_alloc;
268 }
269 
272 {
273  struct rte_mbuf *mb;
274  mb = rte_mbuf_from_vlib_buffer (b);
275  CLIB_PREFETCH (mb, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
277 }
278 
281  vlib_buffer_t * b)
282 {
284  u32 thread_index = vlib_get_thread_index ();
286  fl = vlib_buffer_get_buffer_free_list (vm, b, &fi);
287 
288  /* The only current use of this callback: multicast recycle */
290  {
291  int j;
292 
293  vlib_buffer_add_to_free_list (vm, fl, bi,
294  (b->flags & VLIB_BUFFER_RECYCLE) == 0);
295 
296  for (j = 0; j < vec_len (vm->buffer_announce_list); j++)
297  {
298  if (fl == vm->buffer_announce_list[j])
299  goto already_announced;
300  }
301  vec_add1 (vm->buffer_announce_list, fl);
302  already_announced:
303  ;
304  }
305  else if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_RECYCLE) == 0))
306  dpdk_rte_pktmbuf_free (vm, thread_index, b, 1);
307 }
308 
311  u32 * buffers, u32 n_buffers, u32 follow_buffer_next)
312 {
314  vlib_buffer_t *bufp[n_buffers], **b = bufp;
315  u32 thread_index = vlib_get_thread_index ();
316  int i = 0;
317  u32 simple_mask = (VLIB_BUFFER_NON_DEFAULT_FREELIST | VLIB_BUFFER_RECYCLE |
318  VLIB_BUFFER_NEXT_PRESENT);
319  u32 n_left, *bi;
320  u32 (*cb) (vlib_main_t * vm, u32 * buffers, u32 n_buffers,
321  u32 follow_buffer_next);
322 
323  cb = bm->buffer_free_callback;
324 
325  if (PREDICT_FALSE (cb != 0))
326  n_buffers = (*cb) (vm, buffers, n_buffers, follow_buffer_next);
327 
328  if (!n_buffers)
329  return;
330 
331  n_left = n_buffers;
332  bi = buffers;
333  b = bufp;
334  vlib_get_buffers (vm, bi, b, n_buffers);
335 
336  while (n_left >= 4)
337  {
338  u32 or_flags;
339  vlib_buffer_t **p;
340 
341  if (n_left < 16)
342  goto no_prefetch;
343 
344  p = b + 12;
345  dpdk_prefetch_buffer (p[0]);
346  dpdk_prefetch_buffer (p[1]);
347  dpdk_prefetch_buffer (p[2]);
348  dpdk_prefetch_buffer (p[3]);
349  no_prefetch:
350 
351  for (i = 0; i < 4; i++)
353 
354  or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
355 
356  if (or_flags & simple_mask)
357  {
358  recycle_or_free (vm, bm, bi[0], b[0]);
359  recycle_or_free (vm, bm, bi[1], b[1]);
360  recycle_or_free (vm, bm, bi[2], b[2]);
361  recycle_or_free (vm, bm, bi[3], b[3]);
362  }
363  else
364  {
365  dpdk_rte_pktmbuf_free (vm, thread_index, b[0], 0);
366  dpdk_rte_pktmbuf_free (vm, thread_index, b[1], 0);
367  dpdk_rte_pktmbuf_free (vm, thread_index, b[2], 0);
368  dpdk_rte_pktmbuf_free (vm, thread_index, b[3], 0);
369  }
370  bi += 4;
371  b += 4;
372  n_left -= 4;
373  }
374  while (n_left)
375  {
377  recycle_or_free (vm, bm, bi[0], b[0]);
378  bi += 1;
379  b += 1;
380  n_left -= 1;
381  }
382  if (vec_len (vm->buffer_announce_list))
383  {
385  for (i = 0; i < vec_len (vm->buffer_announce_list); i++)
386  {
387  fl = vm->buffer_announce_list[i];
389  }
390  _vec_len (vm->buffer_announce_list) = 0;
391  }
392 }
393 
394 void
396  u32 n_buffers)
397 {
398  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
399  1);
400 }
401 
402 void
404  u32 n_buffers)
405 {
406  vlib_buffer_free_inline (vm, buffers, n_buffers, /* follow_buffer_next */
407  0);
408 }
409 
410 #ifndef CLIB_MARCH_VARIANT
411 static void
413  void *vt,
414  void *packet_data,
415  uword n_packet_data_bytes,
416  uword min_n_buffers_each_alloc, u8 * name)
417 {
419 
421  memset (t, 0, sizeof (t[0]));
422 
423  vec_add (t->packet_data, packet_data, n_packet_data_bytes);
424 
426 }
427 
428 static clib_error_t *
429 scan_vfio_fd (void *arg, u8 * path_name, u8 * file_name)
430 {
433  const char fn[] = "/dev/vfio/vfio";
434  char buff[sizeof (fn)] = { 0 };
435  int fd;
436  u8 *path = format (0, "%v%c", path_name, 0);
437 
438  if (readlink ((char *) path, buff, sizeof (fn)) + 1 != sizeof (fn))
439  goto done;
440 
441  if (strncmp (fn, buff, sizeof (fn)))
442  goto done;
443 
444  fd = atoi ((char *) file_name);
445  if (fd != lvm->container_fd)
446  dbm->vfio_container_fd = fd;
447 
448 done:
449  vec_free (path);
450  return 0;
451 }
452 
453 clib_error_t *
454 dpdk_pool_create (vlib_main_t * vm, u8 * pool_name, u32 elt_size,
455  u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa,
456  struct rte_mempool ** _mp,
458 {
460  struct rte_mempool *mp;
463  clib_error_t *error = 0;
464  u32 size, obj_size;
465  i32 ret;
466  uword i;
467 
468  obj_size = rte_mempool_calc_obj_size (elt_size, 0, 0);
469 
470 #if RTE_VERSION < RTE_VERSION_NUM(18, 5, 0, 0)
471  size = rte_mempool_xmem_size (num_elts, obj_size, 21, 0);
472 #else
473  size = rte_mempool_calc_mem_size_helper (num_elts, obj_size, 21);
474 #endif
475 
476  error = vlib_physmem_region_alloc (vm, (char *) pool_name, size, numa,
478  VLIB_PHYSMEM_F_SHARED, pri);
479  if (error)
480  return error;
481 
482  pr = vlib_physmem_get_region (vm, pri[0]);
483 
484  mp = rte_mempool_create_empty ((char *) pool_name, num_elts, elt_size,
485  512, pool_priv_size, numa, 0);
486  if (!mp)
487  return clib_error_return (0, "failed to create %s", pool_name);
488 
489  rte_mempool_set_ops_byname (mp, RTE_MBUF_DEFAULT_MEMPOOL_OPS, NULL);
490 
491  /* Call the mempool priv initializer */
492  priv.mbp_priv.mbuf_data_room_size = VLIB_BUFFER_PRE_DATA_SIZE +
494  priv.mbp_priv.mbuf_priv_size = VLIB_BUFFER_HDR_SIZE;
495  rte_pktmbuf_pool_init (mp, &priv);
496 
497  for (i = 0; i < pr->n_pages; i++)
498  {
499  size_t page_size = 1 << pr->log2_page_size;
500  ret = rte_mempool_populate_iova (mp, ((char *) pr->mem) + i * page_size,
501  pr->page_table[i], page_size, 0, 0);
502  if (ret < 0)
503  {
504  rte_mempool_free (mp);
505  return clib_error_return (0, "failed to populate %s", pool_name);
506  }
507  }
508 
509  _mp[0] = mp;
510 
511  /* DPDK currently doesn't provide API to map DMA memory for empty mempool
512  so we are using this hack, will be nice to have at least API to get
513  VFIO container FD */
514  if (dbm->vfio_container_fd == -1)
515  foreach_directory_file ("/proc/self/fd", scan_vfio_fd, 0, 0);
516 
517  if (dbm->vfio_container_fd != -1)
518  {
519  struct vfio_iommu_type1_dma_map dm = { 0 };
520  int i, rv = 0;
521  dm.argsz = sizeof (struct vfio_iommu_type1_dma_map);
522  dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
523 
524  /* *INDENT-OFF* */
526  {
527  dm.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size);
528  dm.size = 1 << pr->log2_page_size;
529  dm.iova = pr->page_table[i];
530  if ((rv = ioctl (dbm->vfio_container_fd, VFIO_IOMMU_MAP_DMA, &dm)))
531  break;
532  }
533  /* *INDENT-ON* */
534  if (rv != 0 && errno != EINVAL)
535  clib_unix_warning ("ioctl(VFIO_IOMMU_MAP_DMA) pool '%s'", pool_name);
536  }
537 
538  return 0;
539 }
540 
541 clib_error_t *
542 dpdk_buffer_pool_create (vlib_main_t * vm, unsigned num_mbufs,
543  unsigned socket_id)
544 {
545  dpdk_main_t *dm = &dpdk_main;
546  struct rte_mempool *rmp;
548  clib_error_t *error = 0;
549  u8 *pool_name;
550  u32 elt_size, i;
551 
553 
554  /* pool already exists, nothing to do */
555  if (dm->pktmbuf_pools[socket_id])
556  return 0;
557 
558  pool_name = format (0, "dpdk_mbuf_pool_socket%u%c", socket_id, 0);
559 
560  elt_size = sizeof (struct rte_mbuf) +
561  VLIB_BUFFER_HDR_SIZE /* priv size */ +
562  VLIB_BUFFER_PRE_DATA_SIZE + VLIB_BUFFER_DATA_SIZE; /*data room size */
563 
564  error =
565  dpdk_pool_create (vm, pool_name, elt_size, num_mbufs,
566  sizeof (dpdk_mempool_private_t), 512, socket_id,
567  &rmp, &pri);
568 
569  vec_free (pool_name);
570 
571  if (!error)
572  {
573  /* call the object initializers */
574  rte_mempool_obj_iter (rmp, rte_pktmbuf_init, 0);
575 
576  dpdk_mempool_private_t *privp = rte_mempool_get_priv (rmp);
577  privp->buffer_pool_index = vlib_buffer_pool_create (vm, pri, 0);
578 
579  dm->pktmbuf_pools[socket_id] = rmp;
580 
581  return 0;
582  }
583 
584  clib_error_report (error);
585 
586  /* no usable pool for this socket, try to use pool from another one */
587  for (i = 0; i < vec_len (dm->pktmbuf_pools); i++)
588  {
589  if (dm->pktmbuf_pools[i])
590  {
591  clib_warning ("WARNING: Failed to allocate mempool for CPU socket "
592  "%u. Threads running on socket %u will use socket %u "
593  "mempool.", socket_id, socket_id, i);
594  dm->pktmbuf_pools[socket_id] = dm->pktmbuf_pools[i];
595  return 0;
596  }
597  }
598 
599  return clib_error_return (0, "failed to allocate mempool on socket %u",
600  socket_id);
601 }
602 
603 #if CLIB_DEBUG > 0
604 
608 
609 static clib_error_t *
611 {
612  void *oldheap;
613 
614  vlib_buffer_state_heap = mheap_alloc (0, 10 << 20);
615 
617 
621  clib_mem_set_heap (oldheap);
622  return 0;
623 }
624 
626 #endif
627 
628 #if CLI_DEBUG
629 struct dpdk_validate_buf_result
630 {
631  u32 invalid;
632  u32 uninitialized;
633 };
634 
635 #define DPDK_TRAJECTORY_POISON 31
636 
637 static void
638 dpdk_buffer_validate_trajectory (struct rte_mempool *mp, void *opaque,
639  void *obj, unsigned obj_idx)
640 {
641  vlib_buffer_t *b;
642  struct dpdk_validate_buf_result *counter = opaque;
643  b = vlib_buffer_from_rte_mbuf ((struct rte_mbuf *) obj);
644  if (b->pre_data[0] != 0)
645  {
646  if (b->pre_data[0] == DPDK_TRAJECTORY_POISON)
647  counter->uninitialized++;
648  else
649  counter->invalid++;
650  }
651 }
652 
653 int
654 dpdk_buffer_validate_trajectory_all (u32 * uninitialized)
655 {
656  dpdk_main_t *dm = &dpdk_main;
657  struct dpdk_validate_buf_result counter = { 0 };
658  int i;
659 
660  for (i = 0; i < vec_len (dm->pktmbuf_pools); i++)
661  rte_mempool_obj_iter (dm->pktmbuf_pools[i],
662  dpdk_buffer_validate_trajectory, &counter);
663  if (uninitialized)
664  *uninitialized = counter.uninitialized;
665  return counter.invalid;
666 }
667 
668 static void
669 dpdk_buffer_poison_trajectory (struct rte_mempool *mp, void *opaque,
670  void *obj, unsigned obj_idx)
671 {
672  vlib_buffer_t *b;
673  b = vlib_buffer_from_rte_mbuf ((struct rte_mbuf *) obj);
674  b->pre_data[0] = DPDK_TRAJECTORY_POISON;
675 }
676 
677 void
678 dpdk_buffer_poison_trajectory_all (void)
679 {
680  dpdk_main_t *dm = &dpdk_main;
681  int i;
682 
683  for (i = 0; i < vec_len (dm->pktmbuf_pools); i++)
684  rte_mempool_obj_iter (dm->pktmbuf_pools[i], dpdk_buffer_poison_trajectory,
685  0);
686 }
687 #endif
688 
689 static clib_error_t *
691 {
694 
695  vec_validate_aligned (dbm->ptd, tm->n_vlib_mains - 1,
697 
698  dbm->vfio_container_fd = -1;
699 
700  return 0;
701 }
702 
704 
705 /* *INDENT-OFF* */
706 VLIB_BUFFER_REGISTER_CALLBACKS (dpdk, static) = {
707  .vlib_buffer_fill_free_list_cb = &dpdk_buffer_fill_free_list,
708  .vlib_buffer_free_cb = &dpdk_buffer_free,
709  .vlib_buffer_free_no_next_cb = &dpdk_buffer_free_no_next,
710  .vlib_packet_template_init_cb = &dpdk_packet_template_init,
711  .vlib_buffer_delete_free_list_cb = &dpdk_buffer_delete_free_list,
712 };
713 /* *INDENT-ON* */
714 
715 #if __x86_64__
722 
723 static void __clib_constructor
725 {
726  vlib_buffer_callbacks_t *cb = &__dpdk_buffer_callbacks;
727  if (dpdk_buffer_fill_free_list_avx512 && clib_cpu_supports_avx512f ())
728  {
732  }
733  else if (dpdk_buffer_fill_free_list_avx2 && clib_cpu_supports_avx2 ())
734  {
738  }
739 }
740 #endif
741 #endif
742 
743 /** @endcond */
744 /*
745  * fd.io coding-style-patch-verification: ON
746  *
747  * Local Variables:
748  * eval: (c-set-style "gnu")
749  * End:
750  */
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define vlib_buffer_from_rte_mbuf(x)
Definition: dpdk_priv.h:17
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:63
dpdk_buffer_main_t dpdk_buffer_main
Definition: buffer.c:100
static clib_error_t * dpdk_buffer_init(vlib_main_t *vm)
Definition: buffer.c:690
static void dpdk_packet_template_init(vlib_main_t *vm, void *vt, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, u8 *name)
Definition: buffer.c:412
#define VLIB_PHYSMEM_F_HUGETLB
Definition: physmem.h:58
dpdk_main_t dpdk_main
Definition: init.c:42
static_always_inline void dpdk_prefetch_buffer(vlib_buffer_t *b)
Definition: buffer.c:271
#define PREDICT_TRUE(x)
Definition: clib.h:106
void * mheap_alloc(void *memory, uword size)
Definition: mheap.c:963
u32 * vlib_buffer_state_validation_lock
Definition: buffer.c:605
#define NULL
Definition: clib.h:55
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx512
Definition: buffer.c:718
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:140
static clib_error_t * vlib_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
int i
static void dpdk_buffer_delete_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer.c:152
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:448
void( vlib_buffer_free_cb_t)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:384
vlib_main_t ** vlib_mains
Definition: buffer.c:303
unsigned char u8
Definition: types.h:56
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:212
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:600
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx2
Definition: buffer.c:717
#define VLIB_PHYSMEM_F_SHARED
Definition: physmem.h:59
#define static_always_inline
Definition: clib.h:93
i64 word
Definition: types.h:111
uword * vlib_buffer_state_validation_hash
Definition: buffer.c:606
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:156
u8 vlib_buffer_pool_create(vlib_main_t *vm, vlib_physmem_region_index_t pri, u16 buffer_size)
Definition: buffer.c:904
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx512
Definition: buffer.c:720
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:184
vlib_buffer_free_no_next_cb_t * vlib_buffer_free_no_next_cb
Definition: buffer.h:393
static void __clib_constructor dpdk_input_multiarch_select(void)
Definition: buffer.c:724
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:164
unsigned int u32
Definition: types.h:88
#define fl(x, y)
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
Definition: physmem_funcs.h:44
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
#define rte_mbuf_from_vlib_buffer(x)
Definition: dpdk_priv.h:16
uword size
clib_error_t * dpdk_buffer_pool_create(vlib_main_t *vm, unsigned num_mbufs, unsigned socket_id)
Definition: buffer.c:542
static clib_error_t * buffer_state_validation_init(vlib_main_t *vm)
Definition: buffer.c:610
static_always_inline void dpdk_rte_pktmbuf_free(vlib_main_t *vm, u32 thread_index, vlib_buffer_t *b, int maybe_next)
Definition: buffer.c:103
unsigned short u16
Definition: types.h:57
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
static clib_error_t * scan_vfio_fd(void *arg, u8 *path_name, u8 *file_name)
Definition: buffer.c:429
uword CLIB_MULTIARCH_FN() dpdk_buffer_fill_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
Definition: buffer.c:178
#define PREDICT_FALSE(x)
Definition: clib.h:105
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: main.h:108
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
Definition: buffer_funcs.h:160
static void vlib_buffer_add_to_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f, u32 buffer_index, u8 do_init)
u32 flags
Definition: vhost_user.h:110
clib_error_t * dpdk_pool_create(vlib_main_t *vm, u8 *pool_name, u32 elt_size, u32 num_elts, u32 pool_priv_size, u16 cache_size, u8 numa, struct rte_mempool **_mp, vlib_physmem_region_index_t *pri)
Definition: buffer.c:454
struct rte_pktmbuf_pool_private mbp_priv
Definition: dpdk_priv.h:57
STATIC_ASSERT(VLIB_BUFFER_PRE_DATA_SIZE==RTE_PKTMBUF_HEADROOM,"VLIB_BUFFER_PRE_DATA_SIZE must be equal to RTE_PKTMBUF_HEADROOM")
vlib_buffer_fill_free_list_cb_t * vlib_buffer_fill_free_list_cb
Definition: buffer.h:391
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.c:623
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:221
u32(* buffer_free_callback)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 follow_buffer_next)
Definition: buffer.h:436
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:77
vlib_main_t * vm
Definition: buffer.c:294
vlib_buffer_fill_free_list_cb_t __clib_weak dpdk_buffer_fill_free_list_avx512
Definition: buffer.c:716
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
static_always_inline void clib_memcpy64_x4(void *d0, void *d1, void *d2, void *d3, void *s)
Definition: string.h:89
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:226
#define clib_warning(format, args...)
Definition: error.h:59
void CLIB_MULTIARCH_FN() dpdk_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:403
static void del_free_list(vlib_main_t *vm, vlib_buffer_free_list_t *f)
Definition: buffer.c:132
void(* buffers_added_to_freelist_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl)
Definition: buffer.h:376
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:235
void * vlib_buffer_state_heap
Definition: buffer.c:607
#define VLIB_BUFFER_DATA_SIZE
Definition: buffer.h:51
#define VLIB_BUFFER_HDR_SIZE
Definition: buffer.h:175
void CLIB_MULTIARCH_FN() dpdk_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.c:395
void( vlib_buffer_free_no_next_cb_t)(struct vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Definition: buffer.h:386
signed int i32
Definition: types.h:81
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define ASSERT(truth)
vlib_buffer_free_cb_t __clib_weak dpdk_buffer_free_avx2
Definition: buffer.c:719
vlib_buffer_free_cb_t * vlib_buffer_free_cb
Definition: buffer.h:392
int vfio_container_fd
Definition: buffer.c:96
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:126
static vlib_buffer_free_list_t * vlib_buffer_get_buffer_free_list(vlib_main_t *vm, vlib_buffer_t *b, vlib_buffer_free_list_index_t *index)
Definition: buffer_funcs.h:652
dpdk_buffer_per_thread_data * ptd
Definition: buffer.c:97
struct rte_mempool ** pktmbuf_pools
Definition: dpdk.h:443
#define clib_error_report(e)
Definition: error.h:113
VLIB_BUFFER_REGISTER_CALLBACKS(dpdk, static)
static uword pointer_to_uword(const void *p)
Definition: types.h:131
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:138
#define clib_max(x, y)
Definition: clib.h:282
static_always_inline void recycle_or_free(vlib_main_t *vm, vlib_buffer_main_t *bm, u32 bi, vlib_buffer_t *b, u32 follow_buffer_next)
Definition: buffer.c:571
clib_error_t * foreach_directory_file(char *dir_name, clib_error_t *(*f)(void *arg, u8 *path_name, u8 *file_name), void *arg, int scan_dirs)
Definition: util.c:49
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:546
u64 uword
Definition: types.h:112
#define clib_unix_warning(format, args...)
Definition: error.h:68
int container_fd
Definition: vfio.h:31
static void vlib_buffer_init_for_free_list(vlib_buffer_t *dst, vlib_buffer_free_list_t *fl)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1513
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
vlib_buffer_main_t buffer_main
Definition: buffer.c:52
u8 vlib_buffer_free_list_index_t
Definition: buffer.h:54
vlib_buffer_free_no_next_cb_t __clib_weak dpdk_buffer_free_no_next_avx2
Definition: buffer.c:721
static vlib_buffer_free_list_t * vlib_buffer_get_free_list(vlib_main_t *vm, vlib_buffer_free_list_index_t free_list_index)
Definition: buffer_funcs.h:662
uword( vlib_buffer_fill_free_list_cb_t)(struct vlib_main_t *vm, vlib_buffer_free_list_t *fl, uword min_free_buffers)
Definition: buffer.h:381
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:128
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:111
u8 vlib_physmem_region_index_t
Definition: physmem.h:43
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
vlib_buffer_free_list_t ** buffer_announce_list
Definition: main.h:111
linux_vfio_main_t vfio_main
Definition: vfio.c:35
struct rte_mbuf ** mbuf_alloc_list
Definition: buffer.c:91
#define CLIB_MULTIARCH_FN(fn)
Definition: cpu.h:59
#define vec_resize_aligned(V, N, A)
Resize a vector (no header, alignment specified).
Definition: vec.h:253