FD.io VPP  v19.01.1-17-ge106252
Vector Packet Processing
vmxnet3.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vppinfra/types.h>
17 #include <vlib/vlib.h>
18 #include <vlib/pci/pci.h>
19 #include <vnet/ethernet/ethernet.h>
20 #include <vnet/plugin/plugin.h>
21 #include <vpp/app/version.h>
22 
23 #include <vmxnet3/vmxnet3.h>
24 
25 #define PCI_VENDOR_ID_VMWARE 0x15ad
26 #define PCI_DEVICE_ID_VMWARE_VMXNET3 0x07b0
27 
29 
30 static pci_device_id_t vmxnet3_pci_device_ids[] = {
31  {
33  .device_id = PCI_DEVICE_ID_VMWARE_VMXNET3},
34  {0},
35 };
36 
37 static clib_error_t *
39  u32 flags)
40 {
41  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
44  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
45 
46  if (vd->flags & VMXNET3_DEVICE_F_ERROR)
47  return clib_error_return (0, "device is in error state");
48 
49  if (is_up)
50  {
53  vd->flags |= VMXNET3_DEVICE_F_ADMIN_UP;
54  }
55  else
56  {
58  vd->flags &= ~VMXNET3_DEVICE_F_ADMIN_UP;
59  }
60  return 0;
61 }
62 
63 static clib_error_t *
66 {
68  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
70  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
71 
73  rxq->int_mode = 0;
74  else
75  rxq->int_mode = 1;
76 
77  return 0;
78 }
79 
80 static void
82  u32 node_index)
83 {
85  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
87 
88  /* Shut off redirection */
89  if (node_index == ~0)
90  {
91  vd->per_interface_next_index = node_index;
92  return;
93  }
94 
97  node_index);
98 }
99 
100 static void
102 {
103  vmxnet3_main_t *vmxm = &vmxnet3_main;
104  vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, instance);
105  vmxnet3_queues *q = &vd->dma->queues;
106 
107  /*
108  * Set the "last_cleared_stats" to the current stats, so that
109  * things appear to clear from a display perspective.
110  */
111  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_STATS);
112 
113  clib_memcpy (&vd->tx_stats, &q->tx.stats, sizeof (vd->tx_stats));
114  clib_memcpy (&vd->rx_stats, &q->rx.stats, sizeof (vd->rx_stats));
115 }
116 
118 #define _(n,s) s,
120 #undef _
121 };
122 
123 /* *INDENT-OFF* */
125 {
126  .name = "VMXNET3 interface",
127  .format_device = format_vmxnet3_device,
128  .format_device_name = format_vmxnet3_device_name,
129  .admin_up_down_function = vmxnet3_interface_admin_up_down,
130  .clear_counters = vmxnet3_clear_hw_interface_counters,
131  .rx_mode_change_function = vmxnet3_interface_rx_mode_change,
132  .rx_redirect_to_node = vmxnet3_set_interface_next_node,
133  .tx_function_n_errors = VMXNET3_TX_N_ERROR,
134  .tx_function_error_strings = vmxnet3_tx_func_error_strings,
135 };
136 /* *INDENT-ON* */
137 
138 static u32
140 {
141  return 0;
142 }
143 
144 static void
146 {
147  u32 val;
148 
149  memcpy (&val, vd->mac_addr, 4);
150  vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACL, val);
151 
152  val = 0;
153  memcpy (&val, vd->mac_addr + 4, 2);
154  vmxnet3_reg_write (vd, 1, VMXNET3_REG_MACH, val);
155 }
156 
157 static clib_error_t *
159 {
160  vmxnet3_shared *shared;
161  vmxnet3_queues *q;
162  u64 shared_dma;
163  u16 qid = 0, rid;
164  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, qid);
165  vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, qid);
166 
167  vd->dma = vlib_physmem_alloc_aligned_on_numa (vm, sizeof (*vd->dma), 512,
168  vd->numa_node);
169  if (vd->dma == 0)
170  return vlib_physmem_last_error (vm);
171 
172  clib_memset (vd->dma, 0, sizeof (*vd->dma));
173 
174  q = &vd->dma->queues;
175  q->tx.cfg.desc_address = vmxnet3_dma_addr (vm, vd, txq->tx_desc);
176  q->tx.cfg.comp_address = vmxnet3_dma_addr (vm, vd, txq->tx_comp);
177  q->tx.cfg.num_desc = txq->size;
178  q->tx.cfg.num_comp = txq->size;
179  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
180  {
181  q->rx.cfg.desc_address[rid] = vmxnet3_dma_addr (vm, vd,
182  rxq->rx_desc[rid]);
183  q->rx.cfg.num_desc[rid] = rxq->size;
184  }
185  q->rx.cfg.comp_address = vmxnet3_dma_addr (vm, vd, rxq->rx_comp);
186  q->rx.cfg.num_comp = rxq->size;
187 
188  shared = &vd->dma->shared;
189  shared->magic = VMXNET3_SHARED_MAGIC;
190  shared->misc.version = VMXNET3_VERSION_MAGIC;
191  if (sizeof (void *) == 4)
192  shared->misc.guest_info = VMXNET3_GOS_BITS_32;
193  else
194  shared->misc.guest_info = VMXNET3_GOS_BITS_64;
195  shared->misc.guest_info |= VMXNET3_GOS_TYPE_LINUX;
196  shared->misc.version_support = VMXNET3_VERSION_SELECT;
197  shared->misc.upt_version_support = VMXNET3_UPT_VERSION_SELECT;
198  shared->misc.queue_desc_address = vmxnet3_dma_addr (vm, vd, q);
199  shared->misc.queue_desc_len = sizeof (*q);
200  shared->misc.mtu = VMXNET3_MTU;
201  shared->misc.num_tx_queues = vd->num_tx_queues;
202  shared->misc.num_rx_queues = vd->num_rx_queues;
203  shared->interrupt.num_intrs = vd->num_intrs;
204  shared->interrupt.event_intr_index = 1;
205  shared->interrupt.control = VMXNET3_IC_DISABLE_ALL;
206  shared->rx_filter.mode = VMXNET3_RXMODE_UCAST | VMXNET3_RXMODE_BCAST |
207  VMXNET3_RXMODE_ALL_MULTI | VMXNET3_RXMODE_PROMISC;
208  shared_dma = vmxnet3_dma_addr (vm, vd, shared);
209 
210  vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAL, shared_dma);
211  vmxnet3_reg_write (vd, 1, VMXNET3_REG_DSAH, shared_dma >> 32);
212 
213  return 0;
214 }
215 
216 static inline void
218 {
219  int i;
220  vmxnet3_shared *shared = &vd->dma->shared;
221 
222  shared->interrupt.control &= ~VMXNET3_IC_DISABLE_ALL;
223  for (i = 0; i < vd->num_intrs; i++)
224  vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 0);
225 }
226 
227 static inline void
229 {
230  int i;
231  vmxnet3_shared *shared = &vd->dma->shared;
232 
233  shared->interrupt.control |= VMXNET3_IC_DISABLE_ALL;
234  for (i = 0; i < vd->num_intrs; i++)
235  vmxnet3_reg_write (vd, 0, VMXNET3_REG_IMR + i * 8, 1);
236 }
237 
238 static clib_error_t *
240 {
241  vmxnet3_rxq_t *rxq;
242  u16 rid;
243 
245  rxq = vec_elt_at_index (vd->rxqs, qid);
246  clib_memset (rxq, 0, sizeof (*rxq));
247  rxq->size = qsz;
248  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
249  {
251  (vm, qsz * sizeof (*rxq->rx_desc[rid]), 512, vd->numa_node);
252 
253  if (rxq->rx_desc[rid] == 0)
254  return vlib_physmem_last_error (vm);
255 
256  clib_memset (rxq->rx_desc[rid], 0, qsz * sizeof (*rxq->rx_desc[rid]));
257  }
258  rxq->rx_comp =
259  vlib_physmem_alloc_aligned_on_numa (vm, qsz * sizeof (*rxq->rx_comp), 512,
260  vd->numa_node);
261  if (rxq->rx_comp == 0)
262  return vlib_physmem_last_error (vm);
263 
264  clib_memset (rxq->rx_comp, 0, qsz * sizeof (*rxq->rx_comp));
265  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
266  {
267  vmxnet3_rx_ring *ring;
268 
269  ring = &rxq->rx_ring[rid];
270  ring->gen = VMXNET3_RXF_GEN;
271  ring->rid = rid;
273  }
275 
276  return 0;
277 }
278 
279 static clib_error_t *
281 {
282  vmxnet3_txq_t *txq;
283 
284  if (qid >= vd->num_tx_queues)
285  {
286  qid = qid % vd->num_tx_queues;
287  txq = vec_elt_at_index (vd->txqs, qid);
288  if (txq->lock == 0)
289  clib_spinlock_init (&txq->lock);
290  vd->flags |= VMXNET3_DEVICE_F_SHARED_TXQ_LOCK;
291  return 0;
292  }
293 
295  txq = vec_elt_at_index (vd->txqs, qid);
296  clib_memset (txq, 0, sizeof (*txq));
297  txq->size = qsz;
298  txq->tx_desc =
299  vlib_physmem_alloc_aligned_on_numa (vm, qsz * sizeof (*txq->tx_desc), 512,
300  vd->numa_node);
301  if (txq->tx_desc == 0)
302  return vlib_physmem_last_error (vm);
303 
304  memset (txq->tx_desc, 0, qsz * sizeof (*txq->tx_desc));
305  txq->tx_comp =
306  vlib_physmem_alloc_aligned_on_numa (vm, qsz * sizeof (*txq->tx_comp), 512,
307  vd->numa_node);
308  if (txq->tx_comp == 0)
309  return vlib_physmem_last_error (vm);
310 
311  clib_memset (txq->tx_comp, 0, qsz * sizeof (*txq->tx_comp));
313  txq->tx_ring.gen = VMXNET3_TXF_GEN;
315 
316  return 0;
317 }
318 
319 static clib_error_t *
322 {
323  clib_error_t *error = 0;
324  u32 ret, i;
326 
327  vd->num_tx_queues = 1;
328  vd->num_rx_queues = 1;
329  vd->num_intrs = 2;
330 
331  /* Quiesce the device */
332  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
333  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
334  if (ret != 0)
335  {
336  error = clib_error_return (0, "error on quiescing device rc (%u)", ret);
337  return error;
338  }
339 
340  /* Reset the device */
341  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
342  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
343  if (ret != 0)
344  {
345  error = clib_error_return (0, "error on resetting device rc (%u)", ret);
346  return error;
347  }
348 
349  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_VRRS);
350  vd->version = count_leading_zeros (ret);
351  vd->version = uword_bits - vd->version;
352 
353  if (vd->version == 0)
354  {
355  error = clib_error_return (0, "unsupported hardware version %u",
356  vd->version);
357  return error;
358  }
359 
360  /* cap support version to 3 */
362  1 << (clib_min (3, vd->version) - 1));
363 
364  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_UVRS);
365  if (ret & 1)
367  else
368  {
369  error = clib_error_return (0, "unsupported upt version %u", ret);
370  return error;
371  }
372 
373  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
374  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
375  if (ret & 1)
376  {
377  vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
378  vd->link_speed = ret >> 16;
379  }
380  else
381  {
382  vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
383  }
384 
385  /* Get the mac address */
386  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACL);
387  clib_memcpy (vd->mac_addr, &ret, 4);
388  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_MACH);
389  clib_memcpy (vd->mac_addr + 4, &ret, 2);
390 
391  error = vmxnet3_rxq_init (vm, vd, 0, args->rxq_size);
392  if (error)
393  return error;
394 
395  for (i = 0; i < tm->n_vlib_mains; i++)
396  {
397  error = vmxnet3_txq_init (vm, vd, i, args->txq_size);
398  if (error)
399  return error;
400  }
401 
402  error = vmxnet3_provision_driver_shared (vm, vd);
403  if (error)
404  return error;
405 
406  vmxnet3_write_mac (vd);
407 
408  /* Activate device */
409  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_ACTIVATE_DEV);
410  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
411  if (ret != 0)
412  {
413  error =
414  clib_error_return (0, "error on activating device rc (%u)", ret);
415  return error;
416  }
417 
418  /* Disable interrupts */
420 
421  vec_foreach_index (i, vd->rxqs)
422  {
423  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
424 
425  vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
426  vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
427  }
428  vd->flags |= VMXNET3_DEVICE_F_INITIALIZED;
429 
431 
432  return error;
433 }
434 
435 static void
437 {
438  vnet_main_t *vnm = vnet_get_main ();
439  vmxnet3_main_t *vmxm = &vmxnet3_main;
440  uword pd = vlib_pci_get_private_data (vm, h);
441  vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
442  u16 qid = line;
443 
444  if (vec_len (vd->rxqs) > qid && vd->rxqs[qid].int_mode != 0)
446 }
447 
448 static void
450 {
451  vnet_main_t *vnm = vnet_get_main ();
452  vmxnet3_main_t *vmxm = &vmxnet3_main;
453  uword pd = vlib_pci_get_private_data (vm, h);
454  vmxnet3_device_t *vd = pool_elt_at_index (vmxm->devices, pd);
455  u32 ret;
456 
457  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
458  ret = vmxnet3_reg_read (vd, 1, VMXNET3_REG_CMD);
459  if (ret & 1)
460  {
461  vd->flags |= VMXNET3_DEVICE_F_LINK_UP;
462  vd->link_speed = ret >> 16;
464  vd->link_speed * 1000);
467  }
468  else
469  {
470  vd->flags &= ~VMXNET3_DEVICE_F_LINK_UP;
472  }
473 }
474 
475 static u8
477 {
478  if (qsz < 64 || qsz > 4096)
479  return 0;
480  if ((qsz % 64) != 0)
481  return 0;
482  return 1;
483 }
484 
485 void
487 {
488  vnet_main_t *vnm = vnet_get_main ();
489  vmxnet3_main_t *vmxm = &vmxnet3_main;
490  vmxnet3_device_t *vd;
492  clib_error_t *error = 0;
493 
494  if (args->rxq_size == 0)
496  if (args->txq_size == 0)
498 
499  if (!vmxnet3_queue_size_valid (args->rxq_size) ||
501  {
502  args->rv = VNET_API_ERROR_INVALID_VALUE;
503  args->error =
504  clib_error_return (error,
505  "queue size must be <= 4096, >= 64, "
506  "and multiples of 64");
507  vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
508  format_vlib_pci_addr, &args->addr,
509  "queue size must be <= 4096, >= 64, and multiples of 64");
510  return;
511  }
512 
513  /* *INDENT-OFF* */
514  pool_foreach (vd, vmxm->devices, ({
515  if (vd->pci_addr.as_u32 == args->addr.as_u32)
516  {
517  args->rv = VNET_API_ERROR_INVALID_VALUE;
518  args->error =
519  clib_error_return (error, "PCI address in use");
520  vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
521  format_vlib_pci_addr, &args->addr, "pci address in use");
522  return;
523  }
524  }));
525  /* *INDENT-ON* */
526 
527  pool_get (vmxm->devices, vd);
528  vd->dev_instance = vd - vmxm->devices;
529  vd->per_interface_next_index = ~0;
530  vd->pci_addr = args->addr;
531 
532  if (args->enable_elog)
533  vd->flags |= VMXNET3_DEVICE_F_ELOG;
534 
535  if ((error =
536  vlib_pci_device_open (vm, &args->addr, vmxnet3_pci_device_ids, &h)))
537  {
538  pool_put (vmxm->devices, vd);
539  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
540  args->error =
541  clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
542  &args->addr);
543  vlib_log (VLIB_LOG_LEVEL_ERR, vmxm->log_default, "%U: %s",
544  format_vlib_pci_addr, &args->addr,
545  "error encountered on pci device open");
546  return;
547  }
548 
549  /*
550  * Do not use vmxnet3_log_error prior to this line since the macro
551  * references vd->pci_dev_handle
552  */
553  vd->pci_dev_handle = h;
554  vd->numa_node = vlib_pci_get_numa_node (vm, h);
555  vlib_pci_set_private_data (vm, h, vd->dev_instance);
556 
557  if ((error = vlib_pci_bus_master_enable (vm, h)))
558  {
559  vmxnet3_log_error (vd, "error encountered on pci bus master enable");
560  goto error;
561  }
562 
563  if ((error = vlib_pci_map_region (vm, h, 0, (void **) &vd->bar[0])))
564  {
565  vmxnet3_log_error (vd, "error encountered on pci map region for bar 0");
566  goto error;
567  }
568 
569  if ((error = vlib_pci_map_region (vm, h, 1, (void **) &vd->bar[1])))
570  {
571  vmxnet3_log_error (vd, "error encountered on pci map region for bar 1");
572  goto error;
573  }
574 
575  if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
577  {
578  vmxnet3_log_error (vd,
579  "error encountered on pci register msix handler 0");
580  goto error;
581  }
582 
583  if ((error = vlib_pci_register_msix_handler (vm, h, 1, 1,
585  {
586  vmxnet3_log_error (vd,
587  "error encountered on pci register msix handler 1");
588  goto error;
589  }
590 
591  if ((error = vlib_pci_enable_msix_irq (vm, h, 0, 2)))
592  {
593  vmxnet3_log_error (vd, "error encountered on pci enable msix irq");
594  goto error;
595  }
596 
597  if ((error = vlib_pci_intr_enable (vm, h)))
598  {
599  vmxnet3_log_error (vd, "error encountered on pci interrupt enable");
600  goto error;
601  }
602 
603  if ((error = vmxnet3_device_init (vm, vd, args)))
604  {
605  vmxnet3_log_error (vd, "error encountered on device init");
606  goto error;
607  }
608 
609  /* create interface */
611  vd->dev_instance, vd->mac_addr,
612  &vd->hw_if_index, vmxnet3_flag_change);
613 
614  if (error)
615  {
616  vmxnet3_log_error (vd,
617  "error encountered on ethernet register interface");
618  goto error;
619  }
620 
621  vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vd->hw_if_index);
622  vd->sw_if_index = sw->sw_if_index;
623  args->sw_if_index = sw->sw_if_index;
624 
625  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vd->hw_if_index);
627  vnet_hw_interface_set_input_node (vnm, vd->hw_if_index,
628  vmxnet3_input_node.index);
629  vnet_hw_interface_assign_rx_thread (vnm, vd->hw_if_index, 0, ~0);
630  vnet_hw_interface_set_link_speed (vnm, vd->hw_if_index,
631  vd->link_speed * 1000);
632  if (vd->flags & VMXNET3_DEVICE_F_LINK_UP)
633  vnet_hw_interface_set_flags (vnm, vd->hw_if_index,
635  else
636  vnet_hw_interface_set_flags (vnm, vd->hw_if_index, 0);
637  return;
638 
639 error:
640  vmxnet3_delete_if (vm, vd);
641  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
642  args->error = error;
643 }
644 
645 void
647 {
648  vnet_main_t *vnm = vnet_get_main ();
649  vmxnet3_main_t *vmxm = &vmxnet3_main;
650  u32 i, bi;
651  u16 desc_idx;
652 
653  /* Quiesce the device */
654  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_QUIESCE_DEV);
655 
656  /* Reset the device */
657  vmxnet3_reg_write (vd, 1, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
658 
659  if (vd->hw_if_index)
660  {
664  }
665 
667 
668  /* *INDENT-OFF* */
669  vec_foreach_index (i, vd->rxqs)
670  {
671  vmxnet3_rxq_t *rxq = vec_elt_at_index (vd->rxqs, i);
672  u16 mask = rxq->size - 1;
673  u16 rid;
674 
675  for (rid = 0; rid < VMXNET3_RX_RING_SIZE; rid++)
676  {
677  vmxnet3_rx_ring *ring;
678 
679  ring = &rxq->rx_ring[rid];
680  desc_idx = (ring->consume + 1) & mask;
681  vlib_buffer_free_from_ring (vm, ring->bufs, desc_idx, rxq->size,
682  ring->fill);
683  vec_free (ring->bufs);
684  vlib_physmem_free (vm, rxq->rx_desc[rid]);
685  }
686  vlib_physmem_free (vm, rxq->rx_comp);
687  }
688  /* *INDENT-ON* */
689  vec_free (vd->rxqs);
690 
691  /* *INDENT-OFF* */
692  vec_foreach_index (i, vd->txqs)
693  {
694  vmxnet3_txq_t *txq = vec_elt_at_index (vd->txqs, i);
695  u16 mask = txq->size - 1;
696  u16 end_idx;
697 
698  desc_idx = txq->tx_ring.consume;
699  end_idx = txq->tx_ring.produce;
700  while (desc_idx != end_idx)
701  {
702  bi = txq->tx_ring.bufs[desc_idx];
703  vlib_buffer_free_no_next (vm, &bi, 1);
704  desc_idx++;
705  desc_idx &= mask;
706  }
707  clib_spinlock_free (&txq->lock);
708  vec_free (txq->tx_ring.bufs);
709  vlib_physmem_free (vm, txq->tx_desc);
710  vlib_physmem_free (vm, txq->tx_comp);
711  }
712  /* *INDENT-ON* */
713  vec_free (vd->txqs);
714 
715  vlib_physmem_free (vm, vd->dma);
716 
717  clib_error_free (vd->error);
718  clib_memset (vd, 0, sizeof (*vd));
719  pool_put (vmxm->devices, vd);
720 }
721 
722 /*
723  * fd.io coding-style-patch-verification: ON
724  *
725  * Local Variables:
726  * eval: (c-set-style "gnu")
727  * End:
728  */
static char * vmxnet3_tx_func_error_strings[]
Definition: vmxnet3.c:117
vmrglw vmrglh hi
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1197
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring0(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:590
void vmxnet3_delete_if(vlib_main_t *vm, vmxnet3_device_t *vd)
Definition: vmxnet3.c:646
format_function_t format_vlib_pci_addr
Definition: pci.h:321
#define vec_foreach_index(var, v)
Iterate over vector indices.
u32 flags
Definition: vhost_user.h:115
#define clib_min(x, y)
Definition: clib.h:295
static void * vlib_physmem_alloc_aligned_on_numa(vlib_main_t *vm, uword n_bytes, uword alignment, u32 numa_node)
Definition: physmem_funcs.h:63
format_function_t format_vmxnet3_device_name
Definition: vmxnet3.h:533
#define VMXNET3_GOS_BITS_32
Definition: vmxnet3.h:109
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:323
#define VMXNET3_REG_VRRS
Definition: vmxnet3.h:68
static clib_error_t * vlib_pci_bus_master_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:268
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
#define count_leading_zeros(x)
Definition: clib.h:138
vmxnet3_rx_desc * rx_desc[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:423
static_always_inline uword vmxnet3_dma_addr(vlib_main_t *vm, vmxnet3_device_t *vd, void *p)
Definition: vmxnet3.h:572
unsigned long u64
Definition: types.h:89
vlib_pci_dev_handle_t pci_dev_handle
Definition: vmxnet3.h:471
#define VMXNET3_REG_DSAL
Definition: vmxnet3.h:70
clib_spinlock_t lock
Definition: vmxnet3.h:448
vmxnet3_tx_stats tx_stats
Definition: vmxnet3.h:492
#define VMXNET3_REG_MACL
Definition: vmxnet3.h:73
static clib_error_t * vlib_physmem_last_error(struct vlib_main_t *vm)
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define foreach_vmxnet3_tx_func_error
Definition: vmxnet3.h:19
#define VMXNET3_MTU
Definition: vmxnet3.h:80
vmxnet3_main_t vmxnet3_main
Definition: vmxnet3.c:28
int i
#define VMXNET3_NUM_TX_DESC
Definition: vmxnet3.h:97
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
void vmxnet3_create_if(vlib_main_t *vm, vmxnet3_create_if_args_t *args)
Definition: vmxnet3.c:486
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
vmxnet3_dma * dma
Definition: vmxnet3.h:489
static clib_error_t * vmxnet3_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: vmxnet3.c:38
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:236
static void vmxnet3_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: vmxnet3.c:81
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1122
unsigned char u8
Definition: types.h:56
static_always_inline void vmxnet3_reg_write(vmxnet3_device_t *vd, u8 bar, u32 addr, u32 val)
Definition: vmxnet3.h:554
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:64
#define VMXNET3_RXCF_GEN
Definition: vmxnet3.h:86
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define clib_memcpy(d, s, n)
Definition: string.h:180
static_always_inline u32 vmxnet3_reg_read(vmxnet3_device_t *vd, u8 bar, u32 addr)
Definition: vmxnet3.h:561
#define VMXNET3_VERSION_SELECT
Definition: vmxnet3.h:104
vmxnet3_rxq_t * rxqs
Definition: vmxnet3.h:476
vlib_log_class_t log_default
Definition: vmxnet3.h:500
static void vmxnet3_clear_hw_interface_counters(u32 instance)
Definition: vmxnet3.c:101
vnet_hw_interface_rx_mode
Definition: interface.h:52
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:490
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
vnet_hw_interface_flags_t flags
Definition: interface.h:516
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
unsigned int u32
Definition: types.h:88
vlib_pci_addr_t addr
Definition: vmxnet3.h:507
#define VMXNET3_REG_UVRS
Definition: vmxnet3.h:69
#define VMXNET3_REG_MACH
Definition: vmxnet3.h:74
static clib_error_t * vmxnet3_rxq_init(vlib_main_t *vm, vmxnet3_device_t *vd, u16 qid, u16 qsz)
Definition: vmxnet3.c:239
#define VMXNET3_REG_IMR
Definition: vmxnet3.h:61
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:584
static clib_error_t * vmxnet3_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
Definition: vmxnet3.c:64
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
#define VMXNET3_REG_CMD
Definition: vmxnet3.h:72
static void vmxnet3_write_mac(vmxnet3_device_t *vd)
Definition: vmxnet3.c:145
static u8 vmxnet3_queue_size_valid(u16 qsz)
Definition: vmxnet3.c:476
unsigned short u16
Definition: types.h:57
#define VMXNET3_GOS_TYPE_LINUX
Definition: vmxnet3.h:111
#define VMXNET3_GOS_BITS_64
Definition: vmxnet3.h:110
vmxnet3_rx_comp_ring rx_comp_ring
Definition: vmxnet3.h:425
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:148
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
vmxnet3_tx_comp_ring tx_comp_ring
Definition: vmxnet3.h:453
#define VMXNET3_UPT_VERSION_SELECT
Definition: vmxnet3.h:105
#define VMXNET3_RX_RING_SIZE
Definition: vmxnet3.h:95
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
vmxnet3_tx_comp * tx_comp
Definition: vmxnet3.h:451
static void vmxnet3_irq_0_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: vmxnet3.c:436
#define VMXNET3_VERSION_MAGIC
Definition: vmxnet3.h:102
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:865
clib_error_t * vlib_pci_map_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, void **result)
Definition: pci.c:1128
static void vmxnet3_enable_interrupt(vmxnet3_device_t *vd)
Definition: vmxnet3.c:217
#define VMXNET3_TXF_GEN
Definition: vmxnet3.h:89
#define VMXNET3_IC_DISABLE_ALL
Definition: vmxnet3.h:107
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
Definition: buffer_funcs.h:550
static void vmxnet3_disable_interrupt(vmxnet3_device_t *vd)
Definition: vmxnet3.c:228
vlib_main_t * vm
Definition: buffer.c:301
#define uword_bits
Definition: types.h:102
static void vlib_physmem_free(vlib_main_t *vm, void *p)
Definition: physmem_funcs.h:89
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:170
void vlib_log(vlib_log_level_t level, vlib_log_class_t class, char *fmt,...)
Definition: log.c:138
u8 mac_addr[6]
Definition: vmxnet3.h:484
#define PCI_VENDOR_ID_VMWARE
Definition: vmxnet3.c:25
u32 per_interface_next_index
Definition: vmxnet3.h:465
VNET_DEVICE_CLASS(vmxnet3_device_class,)
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:808
vmxnet3_txq_t * txqs
Definition: vmxnet3.h:477
clib_error_t * error
Definition: vmxnet3.h:487
static clib_error_t * vmxnet3_device_init(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_create_if_args_t *args)
Definition: vmxnet3.c:320
#define VMXNET3_RXF_GEN
Definition: vmxnet3.h:83
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
format_function_t format_vmxnet3_device
Definition: vmxnet3.h:532
#define VMXNET3_REG_DSAH
Definition: vmxnet3.h:71
#define VMXNET3_TXCF_GEN
Definition: vmxnet3.h:92
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:277
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static clib_error_t * vmxnet3_provision_driver_shared(vlib_main_t *vm, vmxnet3_device_t *vd)
Definition: vmxnet3.c:158
vlib_node_registration_t vmxnet3_input_node
(constructor) VLIB_REGISTER_NODE (vmxnet3_input_node)
Definition: input.c:370
#define VMXNET3_SHARED_MAGIC
Definition: vmxnet3.h:103
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define PCI_DEVICE_ID_VMWARE_VMXNET3
Definition: vmxnet3.c:26
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:504
vmxnet3_tx_desc * tx_desc
Definition: vmxnet3.h:450
u64 uword
Definition: types.h:112
vnet_device_class_t vmxnet3_device_class
static u32 vmxnet3_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: vmxnet3.c:139
static void vmxnet3_irq_1_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: vmxnet3.c:449
vmxnet3_rx_comp * rx_comp
Definition: vmxnet3.h:424
#define clib_error_free(e)
Definition: error.h:86
vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:422
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
vmxnet3_rx_stats rx_stats
Definition: vmxnet3.h:493
vl_api_gbp_vxlan_tunnel_mode_t mode
Definition: gbp.api:349
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
vmxnet3_device_t * devices
Definition: vmxnet3.h:498
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring1(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:635
#define vmxnet3_log_error(dev, f,...)
Definition: vmxnet3.h:541
#define VMXNET3_NUM_RX_DESC
Definition: vmxnet3.h:99
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:236
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:155
vmxnet3_tx_ring tx_ring
Definition: vmxnet3.h:452
clib_error_t * error
Definition: vmxnet3.h:514
static void vnet_hw_interface_set_link_speed(vnet_main_t *vnm, u32 hw_if_index, u32 link_speed)
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
static clib_error_t * vmxnet3_txq_init(vlib_main_t *vm, vmxnet3_device_t *vd, u16 qid, u16 qsz)
Definition: vmxnet3.c:280
u16 vendor_id
Definition: pci.h:124
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1256