FD.io VPP  v17.04-9-g99c0734
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vppinfra/vec.h>
17 #include <vppinfra/format.h>
18 #include <vlib/unix/cj.h>
19 #include <assert.h>
20 
21 #include <vnet/ethernet/ethernet.h>
22 #include <dpdk/device/dpdk.h>
23 
24 #include <dpdk/device/dpdk_priv.h>
25 #include <vppinfra/error.h>
26 
27 #define foreach_dpdk_tx_func_error \
28  _(BAD_RETVAL, "DPDK tx function returned an error") \
29  _(RING_FULL, "Tx packet drops (ring full)") \
30  _(PKT_DROP, "Tx packet drops (dpdk tx failure)") \
31  _(REPL_FAIL, "Tx packet drops (replication failure)")
32 
33 typedef enum
34 {
35 #define _(f,s) DPDK_TX_FUNC_ERROR_##f,
37 #undef _
40 
41 static char *dpdk_tx_func_error_strings[] = {
42 #define _(n,s) s,
44 #undef _
45 };
46 
49 {
50  int error;
51  dpdk_main_t *dm = &dpdk_main;
53 
54  error = rte_eth_dev_default_mac_addr_set (xd->device_index,
55  (struct ether_addr *) address);
56 
57  if (error)
58  {
59  return clib_error_return (0, "mac address set failed: %d", error);
60  }
61  else
62  {
64  vec_add (xd->default_mac_address, address, sizeof (address));
65  return NULL;
66  }
67 }
68 
71  struct ether_addr mc_addr_vec[], int naddr)
72 {
73  int error;
74  dpdk_main_t *dm = &dpdk_main;
76 
77  error = rte_eth_dev_set_mc_addr_list (xd->device_index, mc_addr_vec, naddr);
78 
79  if (error)
80  {
81  return clib_error_return (0, "mc addr list failed: %d", error);
82  }
83  else
84  {
85  return NULL;
86  }
87 }
88 
89 struct rte_mbuf *
91 {
92  dpdk_main_t *dm = &dpdk_main;
93  struct rte_mbuf **mbufs = 0, *s, *d;
94  u8 nb_segs;
95  unsigned socket_id = rte_socket_id ();
96  int i;
97 
98  ASSERT (dm->pktmbuf_pools[socket_id]);
100  nb_segs = s->nb_segs;
101  vec_validate (mbufs, nb_segs - 1);
102 
103  if (rte_pktmbuf_alloc_bulk (dm->pktmbuf_pools[socket_id], mbufs, nb_segs))
104  {
105  vec_free (mbufs);
106  return 0;
107  }
108 
109  d = mbufs[0];
110  d->nb_segs = s->nb_segs;
111  d->data_len = s->data_len;
112  d->pkt_len = s->pkt_len;
113  d->data_off = s->data_off;
114  clib_memcpy (d->buf_addr, s->buf_addr, RTE_PKTMBUF_HEADROOM + s->data_len);
115 
116  for (i = 1; i < nb_segs; i++)
117  {
118  d->next = mbufs[i];
119  d = mbufs[i];
120  s = s->next;
121  d->data_len = s->data_len;
122  clib_memcpy (d->buf_addr, s->buf_addr,
123  RTE_PKTMBUF_HEADROOM + s->data_len);
124  }
125 
126  d = mbufs[0];
127  vec_free (mbufs);
128  return d;
129 }
130 
131 static void
133  vlib_node_runtime_t * node,
134  dpdk_device_t * xd,
135  u16 queue_id, u32 buffer_index, vlib_buffer_t * buffer)
136 {
139  struct rte_mbuf *mb;
140 
141  mb = rte_mbuf_from_vlib_buffer (buffer);
142 
143  t0 = vlib_add_trace (vm, node, buffer, sizeof (t0[0]));
144  t0->queue_index = queue_id;
145  t0->device_index = xd->device_index;
146  t0->buffer_index = buffer_index;
147  clib_memcpy (&t0->mb, mb, sizeof (t0->mb));
148  clib_memcpy (&t0->buffer, buffer,
149  sizeof (buffer[0]) - sizeof (buffer->pre_data));
150  clib_memcpy (t0->buffer.pre_data, buffer->data + buffer->current_data,
151  sizeof (t0->buffer.pre_data));
152 }
153 
156  int maybe_multiseg)
157 {
158  struct rte_mbuf *mb, *first_mb, *last_mb;
159 
160  /* buffer is coming from non-dpdk source so we need to init
161  rte_mbuf header */
163  {
164  vlib_buffer_t *b2 = b;
165  last_mb = mb = rte_mbuf_from_vlib_buffer (b2);
166  rte_pktmbuf_reset (mb);
167  while (maybe_multiseg && (b2->flags & VLIB_BUFFER_NEXT_PRESENT))
168  {
169  b2 = vlib_get_buffer (vm, b2->next_buffer);
170  mb = rte_mbuf_from_vlib_buffer (b2);
171  rte_pktmbuf_reset (mb);
172  }
173  }
174 
175  last_mb = first_mb = mb = rte_mbuf_from_vlib_buffer (b);
176  first_mb->nb_segs = 1;
177  mb->data_len = b->current_length;
178  mb->pkt_len = maybe_multiseg ? vlib_buffer_length_in_chain (vm, b) :
179  b->current_length;
180  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
181 
182  while (maybe_multiseg && (b->flags & VLIB_BUFFER_NEXT_PRESENT))
183  {
184  b = vlib_get_buffer (vm, b->next_buffer);
185  mb = rte_mbuf_from_vlib_buffer (b);
186  last_mb->next = mb;
187  last_mb = mb;
188  mb->data_len = b->current_length;
189  mb->pkt_len = b->current_length;
190  mb->data_off = VLIB_BUFFER_PRE_DATA_SIZE + b->current_data;
191  first_mb->nb_segs++;
192  if (PREDICT_FALSE (b->n_add_refs))
193  {
194  rte_mbuf_refcnt_update (mb, b->n_add_refs);
195  b->n_add_refs = 0;
196  }
197  }
198 }
199 
200 /*
201  * This function calls the dpdk's tx_burst function to transmit the packets
202  * on the tx_vector. It manages a lock per-device if the device does not
203  * support multiple queues. It returns the number of packets untransmitted
204  * on the tx_vector. If all packets are transmitted (the normal case), the
205  * function returns 0.
206  *
207  * The function assumes there is at least one packet on the tx_vector.
208  */
211  dpdk_device_t * xd,
212  struct rte_mbuf **tx_vector)
213 {
214  dpdk_main_t *dm = &dpdk_main;
215  u32 n_packets;
216  u32 tx_head;
217  u32 tx_tail;
218  u32 n_retry;
219  int rv;
220  int queue_id;
221  tx_ring_hdr_t *ring;
222 
223  ring = vec_header (tx_vector, sizeof (*ring));
224 
225  n_packets = ring->tx_head - ring->tx_tail;
226 
227  tx_head = ring->tx_head % xd->nb_tx_desc;
228 
229  /*
230  * Ensure rte_eth_tx_burst is not called with 0 packets, which can lead to
231  * unpredictable results.
232  */
233  ASSERT (n_packets > 0);
234 
235  /*
236  * Check for tx_vector overflow. If this fails it is a system configuration
237  * error. The ring should be sized big enough to handle the largest un-flowed
238  * off burst from a traffic manager. A larger size also helps performance
239  * a bit because it decreases the probability of having to issue two tx_burst
240  * calls due to a ring wrap.
241  */
242  ASSERT (n_packets < xd->nb_tx_desc);
243  ASSERT (ring->tx_tail == 0);
244 
245  n_retry = 16;
246  queue_id = vm->cpu_index;
247 
248  do
249  {
250  /* start the burst at the tail */
251  tx_tail = ring->tx_tail % xd->nb_tx_desc;
252 
253  /*
254  * This device only supports one TX queue,
255  * and we're running multi-threaded...
256  */
257  if (PREDICT_FALSE (xd->lockp != 0))
258  {
259  queue_id = queue_id % xd->tx_q_used;
260  while (__sync_lock_test_and_set (xd->lockp[queue_id], 1))
261  /* zzzz */
262  queue_id = (queue_id + 1) % xd->tx_q_used;
263  }
264 
265  if (PREDICT_FALSE (xd->flags & DPDK_DEVICE_FLAG_HQOS)) /* HQoS ON */
266  {
267  /* no wrap, transmit in one burst */
269  &xd->hqos_wt[vm->cpu_index];
270 
271  ASSERT (hqos->swq != NULL);
272 
274  &tx_vector[tx_tail], tx_head - tx_tail);
275  rv = rte_ring_sp_enqueue_burst (hqos->swq,
276  (void **) &tx_vector[tx_tail],
277  (uint16_t) (tx_head - tx_tail));
278  }
279  else if (PREDICT_TRUE (xd->flags & DPDK_DEVICE_FLAG_PMD))
280  {
281  /* no wrap, transmit in one burst */
282  rv = rte_eth_tx_burst (xd->device_index,
283  (uint16_t) queue_id,
284  &tx_vector[tx_tail],
285  (uint16_t) (tx_head - tx_tail));
286  }
287  else
288  {
289  ASSERT (0);
290  rv = 0;
291  }
292 
293  if (PREDICT_FALSE (xd->lockp != 0))
294  *xd->lockp[queue_id] = 0;
295 
296  if (PREDICT_FALSE (rv < 0))
297  {
298  // emit non-fatal message, bump counter
299  vnet_main_t *vnm = dm->vnet_main;
301  u32 node_index;
302 
303  node_index = vec_elt_at_index (im->hw_interfaces,
304  xd->vlib_hw_if_index)->tx_node_index;
305 
306  vlib_error_count (vm, node_index, DPDK_TX_FUNC_ERROR_BAD_RETVAL, 1);
307  clib_warning ("rte_eth_tx_burst[%d]: error %d", xd->device_index,
308  rv);
309  return n_packets; // untransmitted packets
310  }
311  ring->tx_tail += (u16) rv;
312  n_packets -= (uint16_t) rv;
313  }
314  while (rv && n_packets && (n_retry > 0));
315 
316  return n_packets;
317 }
318 
321 {
322  vlib_buffer_t *b;
323  struct rte_mbuf *mb;
324  b = vlib_get_buffer (vm, bi);
325  mb = rte_mbuf_from_vlib_buffer (b);
328 }
329 
332  vlib_buffer_t * b, u32 bi, struct rte_mbuf **mbp)
333 {
334  dpdk_main_t *dm = &dpdk_main;
335  u32 my_cpu = vm->cpu_index;
336  struct rte_mbuf *mb_new;
337 
338  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_RECYCLE) == 0)
339  return;
340 
341  mb_new = dpdk_replicate_packet_mb (b);
342  if (PREDICT_FALSE (mb_new == 0))
343  {
344  vlib_error_count (vm, node->node_index,
345  DPDK_TX_FUNC_ERROR_REPL_FAIL, 1);
347  }
348  else
349  *mbp = mb_new;
350 
351  vec_add1 (dm->recycle[my_cpu], bi);
352 }
353 
354 /*
355  * Transmits the packets on the frame to the interface associated with the
356  * node. It first copies packets on the frame to a tx_vector containing the
357  * rte_mbuf pointers. It then passes this vector to tx_burst_vector_internal
358  * which calls the dpdk tx_burst function.
359  */
360 static uword
362  vlib_node_runtime_t * node, vlib_frame_t * f)
363 {
364  dpdk_main_t *dm = &dpdk_main;
365  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
367  u32 n_packets = f->n_vectors;
368  u32 n_left;
369  u32 *from;
370  struct rte_mbuf **tx_vector;
371  u16 i;
372  u16 nb_tx_desc = xd->nb_tx_desc;
373  int queue_id;
374  u32 my_cpu;
375  u32 tx_pkts = 0;
376  tx_ring_hdr_t *ring;
377  u32 n_on_ring;
378 
379  my_cpu = vm->cpu_index;
380 
381  queue_id = my_cpu;
382 
383  tx_vector = xd->tx_vectors[queue_id];
384  ring = vec_header (tx_vector, sizeof (*ring));
385 
386  n_on_ring = ring->tx_head - ring->tx_tail;
387  from = vlib_frame_vector_args (f);
388 
389  ASSERT (n_packets <= VLIB_FRAME_SIZE);
390 
391  if (PREDICT_FALSE (n_on_ring + n_packets > nb_tx_desc))
392  {
393  /*
394  * Overflowing the ring should never happen.
395  * If it does then drop the whole frame.
396  */
397  vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_RING_FULL,
398  n_packets);
399 
400  while (n_packets--)
401  {
402  u32 bi0 = from[n_packets];
403  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
404  struct rte_mbuf *mb0 = rte_mbuf_from_vlib_buffer (b0);
405  rte_pktmbuf_free (mb0);
406  }
407  return n_on_ring;
408  }
409 
410  if (PREDICT_FALSE (dm->tx_pcap_enable))
411  {
412  n_left = n_packets;
413  while (n_left > 0)
414  {
415  u32 bi0 = from[0];
416  vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0);
417  if (dm->pcap_sw_if_index == 0 ||
418  dm->pcap_sw_if_index == vnet_buffer (b0)->sw_if_index[VLIB_TX])
419  pcap_add_buffer (&dm->pcap_main, vm, bi0, 512);
420  from++;
421  n_left--;
422  }
423  }
424 
425  from = vlib_frame_vector_args (f);
426  n_left = n_packets;
427  i = ring->tx_head % nb_tx_desc;
428 
429  while (n_left >= 8)
430  {
431  u32 bi0, bi1, bi2, bi3;
432  struct rte_mbuf *mb0, *mb1, *mb2, *mb3;
433  vlib_buffer_t *b0, *b1, *b2, *b3;
434  u32 or_flags;
435 
436  dpdk_prefetch_buffer_by_index (vm, from[4]);
437  dpdk_prefetch_buffer_by_index (vm, from[5]);
438  dpdk_prefetch_buffer_by_index (vm, from[6]);
439  dpdk_prefetch_buffer_by_index (vm, from[7]);
440 
441  bi0 = from[0];
442  bi1 = from[1];
443  bi2 = from[2];
444  bi3 = from[3];
445  from += 4;
446 
447  b0 = vlib_get_buffer (vm, bi0);
448  b1 = vlib_get_buffer (vm, bi1);
449  b2 = vlib_get_buffer (vm, bi2);
450  b3 = vlib_get_buffer (vm, bi3);
451 
452  or_flags = b0->flags | b1->flags | b2->flags | b3->flags;
453 
454  if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
455  {
456  dpdk_validate_rte_mbuf (vm, b0, 1);
457  dpdk_validate_rte_mbuf (vm, b1, 1);
458  dpdk_validate_rte_mbuf (vm, b2, 1);
459  dpdk_validate_rte_mbuf (vm, b3, 1);
460  }
461  else
462  {
463  dpdk_validate_rte_mbuf (vm, b0, 0);
464  dpdk_validate_rte_mbuf (vm, b1, 0);
465  dpdk_validate_rte_mbuf (vm, b2, 0);
466  dpdk_validate_rte_mbuf (vm, b3, 0);
467  }
468 
469  mb0 = rte_mbuf_from_vlib_buffer (b0);
470  mb1 = rte_mbuf_from_vlib_buffer (b1);
471  mb2 = rte_mbuf_from_vlib_buffer (b2);
472  mb3 = rte_mbuf_from_vlib_buffer (b3);
473 
474  if (PREDICT_FALSE (or_flags & VLIB_BUFFER_RECYCLE))
475  {
476  dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
477  dpdk_buffer_recycle (vm, node, b1, bi1, &mb1);
478  dpdk_buffer_recycle (vm, node, b2, bi2, &mb2);
479  dpdk_buffer_recycle (vm, node, b3, bi3, &mb3);
480 
481  /* dont enqueue packets if replication failed as they must
482  be sent back to recycle */
483  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
484  tx_vector[i++ % nb_tx_desc] = mb0;
485  if (PREDICT_TRUE ((b1->flags & VLIB_BUFFER_REPL_FAIL) == 0))
486  tx_vector[i++ % nb_tx_desc] = mb1;
487  if (PREDICT_TRUE ((b2->flags & VLIB_BUFFER_REPL_FAIL) == 0))
488  tx_vector[i++ % nb_tx_desc] = mb2;
489  if (PREDICT_TRUE ((b3->flags & VLIB_BUFFER_REPL_FAIL) == 0))
490  tx_vector[i++ % nb_tx_desc] = mb3;
491  }
492  else
493  {
494  if (PREDICT_FALSE (i + 3 >= nb_tx_desc))
495  {
496  tx_vector[i++ % nb_tx_desc] = mb0;
497  tx_vector[i++ % nb_tx_desc] = mb1;
498  tx_vector[i++ % nb_tx_desc] = mb2;
499  tx_vector[i++ % nb_tx_desc] = mb3;
500  i %= nb_tx_desc;
501  }
502  else
503  {
504  tx_vector[i++] = mb0;
505  tx_vector[i++] = mb1;
506  tx_vector[i++] = mb2;
507  tx_vector[i++] = mb3;
508  }
509  }
510 
511 
513  {
514  if (b0->flags & VLIB_BUFFER_IS_TRACED)
515  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0);
516  if (b1->flags & VLIB_BUFFER_IS_TRACED)
517  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi1, b1);
518  if (b2->flags & VLIB_BUFFER_IS_TRACED)
519  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi2, b2);
520  if (b3->flags & VLIB_BUFFER_IS_TRACED)
521  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi3, b3);
522  }
523 
524  n_left -= 4;
525  }
526  while (n_left > 0)
527  {
528  u32 bi0;
529  struct rte_mbuf *mb0;
530  vlib_buffer_t *b0;
531 
532  bi0 = from[0];
533  from++;
534 
535  b0 = vlib_get_buffer (vm, bi0);
536 
537  dpdk_validate_rte_mbuf (vm, b0, 1);
538 
539  mb0 = rte_mbuf_from_vlib_buffer (b0);
540  dpdk_buffer_recycle (vm, node, b0, bi0, &mb0);
541 
543  if (b0->flags & VLIB_BUFFER_IS_TRACED)
544  dpdk_tx_trace_buffer (dm, node, xd, queue_id, bi0, b0);
545 
546  if (PREDICT_TRUE ((b0->flags & VLIB_BUFFER_REPL_FAIL) == 0))
547  {
548  tx_vector[i % nb_tx_desc] = mb0;
549  i++;
550  }
551  n_left--;
552  }
553 
554  /* account for additional packets in the ring */
555  ring->tx_head += n_packets;
556  n_on_ring = ring->tx_head - ring->tx_tail;
557 
558  /* transmit as many packets as possible */
559  n_packets = tx_burst_vector_internal (vm, xd, tx_vector);
560 
561  /*
562  * tx_pkts is the number of packets successfully transmitted
563  * This is the number originally on ring minus the number remaining on ring
564  */
565  tx_pkts = n_on_ring - n_packets;
566 
567  {
568  /* If there is no callback then drop any non-transmitted packets */
569  if (PREDICT_FALSE (n_packets))
570  {
572  vnet_main_t *vnm = vnet_get_main ();
573 
576 
577  vlib_increment_simple_counter (cm, my_cpu, xd->vlib_sw_if_index,
578  n_packets);
579 
580  vlib_error_count (vm, node->node_index, DPDK_TX_FUNC_ERROR_PKT_DROP,
581  n_packets);
582 
583  while (n_packets--)
584  rte_pktmbuf_free (tx_vector[ring->tx_tail + n_packets]);
585  }
586 
587  /* Reset head/tail to avoid unnecessary wrap */
588  ring->tx_head = 0;
589  ring->tx_tail = 0;
590  }
591 
592  /* Recycle replicated buffers */
593  if (PREDICT_FALSE (vec_len (dm->recycle[my_cpu])))
594  {
595  vlib_buffer_free (vm, dm->recycle[my_cpu],
596  vec_len (dm->recycle[my_cpu]));
597  _vec_len (dm->recycle[my_cpu]) = 0;
598  }
599 
600  ASSERT (ring->tx_head >= ring->tx_tail);
601 
602  return tx_pkts;
603 }
604 
605 static void
607 {
608  dpdk_main_t *dm = &dpdk_main;
609  dpdk_device_t *xd = vec_elt_at_index (dm->devices, instance);
610 
611  /*
612  * Set the "last_cleared_stats" to the current stats, so that
613  * things appear to clear from a display perspective.
614  */
616 
617  clib_memcpy (&xd->last_cleared_stats, &xd->stats, sizeof (xd->stats));
620  sizeof (xd->last_cleared_xstats[0]));
621 
622 }
623 
624 static clib_error_t *
626 {
627  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
628  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
629  dpdk_main_t *dm = &dpdk_main;
631  int rv = 0;
632 
633  if (is_up)
634  {
635  f64 now = vlib_time_now (dm->vlib_main);
636 
637  if ((xd->flags & DPDK_DEVICE_FLAG_ADMIN_UP) == 0)
638  {
639  rv = rte_eth_dev_start (xd->device_index);
640  if (!rv && xd->default_mac_address)
641  rv = rte_eth_dev_default_mac_addr_set (xd->device_index,
642  (struct ether_addr *)
643  xd->default_mac_address);
644  }
645 
647  rte_eth_promiscuous_enable (xd->device_index);
648  else
649  rte_eth_promiscuous_disable (xd->device_index);
650 
651  rte_eth_allmulticast_enable (xd->device_index);
653  dpdk_update_counters (xd, now);
654  dpdk_update_link_state (xd, now);
655  }
656  else
657  {
659 
660  rte_eth_allmulticast_disable (xd->device_index);
662  rte_eth_dev_stop (xd->device_index);
663 
664  /* For bonded interface, stop slave links */
665  if (xd->pmd == VNET_DPDK_PMD_BOND)
666  {
667  u8 slink[16];
668  int nlink = rte_eth_bond_slaves_get (xd->device_index, slink, 16);
669  while (nlink >= 1)
670  {
671  u8 dpdk_port = slink[--nlink];
672  rte_eth_dev_stop (dpdk_port);
673  }
674  }
675  }
676 
677  if (rv < 0)
678  clib_warning ("rte_eth_dev_%s error: %d", is_up ? "start" : "stop", rv);
679 
680  return /* no error */ 0;
681 }
682 
683 /*
684  * Dynamically redirect all pkts from a specific interface
685  * to the specified node
686  */
687 static void
689  u32 node_index)
690 {
691  dpdk_main_t *xm = &dpdk_main;
692  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
694 
695  /* Shut off redirection */
696  if (node_index == ~0)
697  {
698  xd->per_interface_next_index = node_index;
699  return;
700  }
701 
703  vlib_node_add_next (xm->vlib_main, dpdk_input_node.index, node_index);
704 }
705 
706 
707 static clib_error_t *
709  u32 hw_if_index,
710  struct vnet_sw_interface_t *st, int is_add)
711 {
712  dpdk_main_t *xm = &dpdk_main;
713  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
716  int r, vlan_offload;
717  u32 prev_subifs = xd->num_subifs;
718  clib_error_t *err = 0;
719 
720  if (is_add)
721  xd->num_subifs++;
722  else if (xd->num_subifs)
723  xd->num_subifs--;
724 
725  if ((xd->flags & DPDK_DEVICE_FLAG_PMD) == 0)
726  goto done;
727 
728  /* currently we program VLANS only for IXGBE VF and I40E VF */
729  if ((xd->pmd != VNET_DPDK_PMD_IXGBEVF) && (xd->pmd != VNET_DPDK_PMD_I40EVF))
730  goto done;
731 
732  if (t->sub.eth.flags.no_tags == 1)
733  goto done;
734 
735  if ((t->sub.eth.flags.one_tag != 1) || (t->sub.eth.flags.exact_match != 1))
736  {
737  xd->num_subifs = prev_subifs;
738  err = clib_error_return (0, "unsupported VLAN setup");
739  goto done;
740  }
741 
742  vlan_offload = rte_eth_dev_get_vlan_offload (xd->device_index);
743  vlan_offload |= ETH_VLAN_FILTER_OFFLOAD;
744 
745  if ((r = rte_eth_dev_set_vlan_offload (xd->device_index, vlan_offload)))
746  {
747  xd->num_subifs = prev_subifs;
748  err = clib_error_return (0, "rte_eth_dev_set_vlan_offload[%d]: err %d",
749  xd->device_index, r);
750  goto done;
751  }
752 
753 
754  if ((r =
755  rte_eth_dev_vlan_filter (xd->device_index, t->sub.eth.outer_vlan_id,
756  is_add)))
757  {
758  xd->num_subifs = prev_subifs;
759  err = clib_error_return (0, "rte_eth_dev_vlan_filter[%d]: err %d",
760  xd->device_index, r);
761  goto done;
762  }
763 
764 done:
765  if (xd->num_subifs)
767  else
769 
770  return err;
771 }
772 
773 /* *INDENT-OFF* */
775  .name = "dpdk",
776  .tx_function = dpdk_interface_tx,
777  .tx_function_n_errors = DPDK_TX_FUNC_N_ERROR,
778  .tx_function_error_strings = dpdk_tx_func_error_strings,
779  .format_device_name = format_dpdk_device_name,
780  .format_device = format_dpdk_device,
781  .format_tx_trace = format_dpdk_tx_dma_trace,
782  .clear_counters = dpdk_clear_hw_interface_counters,
783  .admin_up_down_function = dpdk_interface_admin_up_down,
784  .subif_add_del_function = dpdk_subif_add_del_function,
785  .rx_redirect_to_node = dpdk_set_interface_next_node,
786  .mac_addr_change_function = dpdk_set_mac_address,
787 };
788 
790 /* *INDENT-ON* */
791 
792 #define UP_DOWN_FLAG_EVENT 1
793 
794 uword
797 {
798  clib_error_t *error = 0;
799  uword event_type;
800  uword *event_data = 0;
801  u32 sw_if_index;
802  u32 flags;
803 
804  while (1)
805  {
807 
808  event_type = vlib_process_get_events (vm, &event_data);
809 
811 
812  switch (event_type)
813  {
814  case UP_DOWN_FLAG_EVENT:
815  {
816  if (vec_len (event_data) == 2)
817  {
818  sw_if_index = event_data[0];
819  flags = event_data[1];
820  error =
822  flags);
823  clib_error_report (error);
824  }
825  }
826  break;
827  }
828 
829  vec_reset_length (event_data);
830 
832 
833  }
834  return 0; /* or not */
835 }
836 
837 /* *INDENT-OFF* */
839  .function = admin_up_down_process,
840  .type = VLIB_NODE_TYPE_PROCESS,
841  .name = "admin-up-down-process",
842  .process_log2_n_stack_bytes = 17, // 256KB
843 };
844 /* *INDENT-ON* */
845 
846 /*
847  * fd.io coding-style-patch-verification: ON
848  *
849  * Local Variables:
850  * eval: (c-set-style "gnu")
851  * End:
852  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:436
#define DPDK_DEVICE_FLAG_PROMISC
Definition: dpdk.h:196
u8 * default_mac_address
Definition: dpdk.h:238
vmrglw vmrglh hi
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:530
vlib_buffer_t buffer
Definition: dpdk.h:431
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:290
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:463
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:603
#define VLIB_BUFFER_RECYCLE
Definition: buffer.h:91
dpdk_main_t dpdk_main
Definition: dpdk.h:422
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
vnet_interface_main_t interface_main
Definition: vnet.h:57
vnet_device_class_t dpdk_device_class
#define PREDICT_TRUE(x)
Definition: clib.h:98
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define NULL
Definition: clib.h:55
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:185
static_always_inline u32 tx_burst_vector_internal(vlib_main_t *vm, dpdk_device_t *xd, struct rte_mbuf **tx_vector)
Definition: device.c:210
static char * dpdk_tx_func_error_strings[]
Definition: device.c:41
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
u16 flags
Definition: dpdk.h:194
static void dpdk_tx_trace_buffer(dpdk_main_t *dm, vlib_node_runtime_t *node, dpdk_device_t *xd, u16 queue_id, u32 buffer_index, vlib_buffer_t *buffer)
Definition: device.c:132
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
#define DPDK_DEVICE_FLAG_HQOS
Definition: dpdk.h:201
u32 per_interface_next_index
Definition: dpdk.h:182
u64 tx_tail
Definition: dpdk.h:138
struct rte_eth_xstat * last_cleared_xstats
Definition: dpdk.h:233
u16 num_subifs
Definition: dpdk.h:209
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:100
format_function_t format_dpdk_tx_dma_trace
Definition: dpdk.h:493
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1062
VNET_DEVICE_CLASS(af_packet_device_class)
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
#define DPDK_DEVICE_FLAG_PMD
Definition: dpdk.h:197
#define vec_add(V, E, N)
Add N elements to end of vector V (no header, unspecified alignment)
Definition: vec.h:599
struct rte_eth_stats stats
Definition: dpdk.h:229
struct vnet_sub_interface_t::@117::@118::@120 flags
#define VLIB_BUFFER_NEXT_PRESENT
Definition: buffer.h:87
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:67
#define static_always_inline
Definition: clib.h:85
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:526
#define VLIB_BUFFER_REPL_FAIL
Definition: buffer.h:90
vlib_node_registration_t dpdk_input_node
(constructor) VLIB_REGISTER_NODE (dpdk_input_node)
Definition: node.c:655
static void dpdk_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:688
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:111
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:144
u32 cpu_index
Definition: main.h:159
static uword dpdk_interface_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *f)
Definition: device.c:361
static vlib_node_registration_t admin_up_down_process_node
(constructor) VLIB_REGISTER_NODE (admin_up_down_process_node)
Definition: device.c:838
A collection of simple counters.
Definition: counter.h:58
u32 device_index
Definition: dpdk.h:176
dpdk_device_hqos_per_worker_thread_t * hqos_wt
Definition: dpdk.h:220
u32 pcap_sw_if_index
Definition: dpdk.h:378
vnet_hw_interface_t * hw_interfaces
Definition: interface.h:603
#define rte_mbuf_from_vlib_buffer(x)
Definition: dpdk_priv.h:16
vnet_sub_interface_t sub
Definition: interface.h:569
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:71
u32 vlib_hw_if_index
Definition: dpdk.h:178
pcap_main_t pcap_main
Definition: dpdk.h:376
void dpdk_hqos_metadata_set(dpdk_device_hqos_per_worker_thread_t *hqos, struct rte_mbuf **pkts, u32 n_pkts)
Definition: hqos.c:644
struct rte_mbuf mb
Definition: dpdk.h:429
static void pcap_add_buffer(pcap_main_t *pm, vlib_main_t *vm, u32 buffer_index, u32 n_bytes_in_trace)
Add buffer (vlib_buffer_t) to the trace.
Definition: pcap.h:201
#define PREDICT_FALSE(x)
Definition: clib.h:97
#define VLIB_FRAME_SIZE
Definition: node.h:328
vlib_simple_counter_main_t * sw_if_counters
Definition: interface.h:625
u16 tx_q_used
Definition: dpdk.h:212
u32 node_index
Node index.
Definition: node.h:436
dpdk_tx_func_error_t
Definition: device.c:33
struct rte_mbuf * dpdk_replicate_packet_mb(vlib_buffer_t *b)
Definition: device.c:90
#define DPDK_DEVICE_FLAG_ADMIN_UP
Definition: dpdk.h:195
u32 ** recycle
Definition: dpdk.h:360
u16 n_vectors
Definition: node.h:344
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
dpdk_device_t * devices
Definition: dpdk.h:355
vlib_main_t * vm
Definition: buffer.c:276
static void dpdk_update_counters(dpdk_device_t *xd, f64 now)
Definition: dpdk_priv.h:78
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:340
volatile u32 ** lockp
Definition: dpdk.h:173
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_BUFFER_IS_TRACED
Definition: buffer.h:85
#define clib_memcpy(a, b, c)
Definition: string.h:69
dpdk_pmd_t pmd
Definition: dpdk.h:191
struct vnet_sub_interface_t::@117 eth
format_function_t format_dpdk_device
Definition: dpdk.h:492
static_always_inline void dpdk_validate_rte_mbuf(vlib_main_t *vm, vlib_buffer_t *b, int maybe_multiseg)
Definition: device.c:155
struct rte_eth_xstat * xstats
Definition: dpdk.h:232
#define VNET_SW_INTERFACE_FLAG_ADMIN_UP
Definition: interface.h:536
#define ASSERT(truth)
format_function_t format_dpdk_device_name
Definition: dpdk.h:491
unsigned int u32
Definition: types.h:88
u64 tx_head
Definition: dpdk.h:137
uword admin_up_down_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: device.c:795
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:109
struct rte_mempool ** pktmbuf_pools
Definition: dpdk.h:416
#define DPDK_DEVICE_FLAG_HAVE_SUBIF
Definition: dpdk.h:200
#define clib_error_report(e)
Definition: error.h:125
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:259
void dpdk_update_link_state(dpdk_device_t *xd, f64 now)
Definition: init.c:1688
u8 n_add_refs
Number of additional references to this buffer.
Definition: buffer.h:124
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
static void dpdk_clear_hw_interface_counters(u32 instance)
Definition: device.c:606
Definition: defs.h:47
unsigned short u16
Definition: types.h:57
#define foreach_dpdk_tx_func_error
Definition: device.c:27
clib_error_t * dpdk_set_mac_address(vnet_hw_interface_t *hi, char *address)
Definition: device.c:48
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
u8 admin_up_down_in_progress
Definition: dpdk.h:391
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:253
#define UP_DOWN_FLAG_EVENT
Definition: device.c:792
static clib_error_t * dpdk_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:625
#define VLIB_BUFFER_EXT_HDR_VALID
Definition: buffer.h:93
#define vnet_buffer(b)
Definition: buffer.h:294
static_always_inline void dpdk_prefetch_buffer_by_index(vlib_main_t *vm, u32 bi)
Definition: device.c:320
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
static void * vec_header(void *v, uword header_bytes)
Find a user vector header.
Definition: vec_bootstrap.h:92
u8 data[0]
Packet data.
Definition: buffer.h:152
clib_error_t * vnet_sw_interface_set_flags(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
Definition: interface.c:538
u16 flags
Copy of main node flags.
Definition: node.h:449
struct rte_eth_stats last_cleared_stats
Definition: dpdk.h:231
static clib_error_t * dpdk_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
Definition: device.c:708
static_always_inline void dpdk_buffer_recycle(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b, u32 bi, struct rte_mbuf **mbp)
Definition: device.c:331
u32 flags
Definition: vhost-user.h:78
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:74
int tx_pcap_enable
Definition: dpdk.h:375
vnet_main_t * vnet_main
Definition: dpdk.h:412
u16 nb_tx_desc
Definition: dpdk.h:203
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 cpu_index, u32 index, u64 increment)
Increment a simple counter.
Definition: counter.h:78
#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH(dev, fn)
Definition: interface.h:217
clib_error_t * dpdk_set_mc_filter(vnet_hw_interface_t *hi, struct ether_addr mc_addr_vec[], int naddr)
Definition: device.c:70
CLIB vectors are ubiquitous dynamically resized arrays with by user defined "headers".
vlib_main_t * vlib_main
Definition: dpdk.h:411