FD.io VPP  v19.04.4-rc0-5-ge88582fac
Vector Packet Processing
pci.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <fcntl.h>
17 #include <sys/ioctl.h>
18 
19 #include <vppinfra/types.h>
20 #include <vlib/vlib.h>
21 #include <vlib/pci/pci.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/ip/ip4_packet.h>
24 #include <vnet/ip/ip6_packet.h>
27 
28 #define PCI_VENDOR_ID_VIRTIO 0x1af4
29 #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000
30 /* Doesn't support modern device */
31 #define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041
32 
33 #define PCI_CAPABILITY_LIST 0x34
34 #define PCI_CAP_ID_VNDR 0x09
35 #define PCI_CAP_ID_MSIX 0x11
36 
37 #define PCI_MSIX_ENABLE 0x8000
38 
39 #define PCI_CONFIG_SIZE(vif) ((vif->msix_enabled == VIRTIO_MSIX_ENABLED) ? \
40  24 : 20)
41 
42 static pci_device_id_t virtio_pci_device_ids[] = {
43  {
45  .device_id = PCI_DEVICE_ID_VIRTIO_NIC},
46  {
47  .vendor_id = PCI_VENDOR_ID_VIRTIO,
48  .device_id = PCI_DEVICE_ID_VIRTIO_NIC_MODERN},
49  {0},
50 };
51 
52 static void
54  int len, u32 addr)
55 {
56  u32 size = 0;
58 
59  while (len > 0)
60  {
61  if (len >= 4)
62  {
63  size = 4;
64  vlib_pci_read_io_u32 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
65  }
66  else if (len >= 2)
67  {
68  size = 2;
69  vlib_pci_read_io_u16 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
70  }
71  else
72  {
73  size = 1;
74  vlib_pci_read_io_u8 (vm, h, PCI_CONFIG_SIZE (vif) + addr, dst);
75  }
76  dst = (u8 *) dst + size;
77  addr += size;
78  len -= size;
79  }
80 }
81 
82 static void
84  void *src, int len, u32 addr)
85 {
86  u32 size = 0;
88 
89  while (len > 0)
90  {
91  if (len >= 4)
92  {
93  size = 4;
94  vlib_pci_write_io_u32 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
95  }
96  else if (len >= 2)
97  {
98  size = 2;
99  vlib_pci_write_io_u16 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
100  }
101  else
102  {
103  size = 1;
104  vlib_pci_write_io_u8 (vm, h, PCI_CONFIG_SIZE (vif) + addr, src);
105  }
106  src = (u8 *) src + size;
107  addr += size;
108  len -= size;
109  }
110 }
111 
112 static u64
114 {
115  u32 features;
116  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
117  &features);
118  return features;
119 }
120 
121 static u32
123  u64 features)
124 {
125  if ((features >> 32) != 0)
126  {
127  clib_warning ("only 32 bit features are allowed for legacy virtio!");
128  }
129  u32 feature = 0, guest_features = (u32) features;
130  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
131  &guest_features);
132  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
133  &feature);
134  return feature;
135 }
136 
137 static u8
139 {
140  u8 status = 0;
141  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
142  return status;
143 }
144 
145 static void
147 {
148  if (status != VIRTIO_CONFIG_STATUS_RESET)
149  status |= virtio_pci_legacy_get_status (vm, vif);
150  vlib_pci_write_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
151 }
152 
153 static u8
155 {
156  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_RESET);
157  return virtio_pci_legacy_get_status (vm, vif);
158 }
159 
160 static u8
162 {
163  u8 isr = 0;
164  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &isr);
165  return isr;
166 }
167 
168 static u16
170  u16 queue_id)
171 {
172  u16 queue_num = 0;
173  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
174  &queue_id);
175  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
176  &queue_num);
177  return queue_num;
178 }
179 
180 
181 static void
183  u16 queue_id, void *p)
184 {
186  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
187  &queue_id);
188  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
189  (u32 *) & addr);
190 }
191 
192 static void
194  u16 queue_id)
195 {
196  u32 src = 0;
197  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
198  &queue_id);
199  vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN, &src);
200 }
201 
202 inline void
204  u16 queue_id)
205 {
206  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
207  &queue_id);
208 }
209 
210 /* Enable one vector (0) for Link State Intrerrupt */
211 static u16
213  u16 vec)
214 {
215  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
216  &vec);
217  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
218  &vec);
219  return vec;
220 }
221 
222 static u16
224  u16 queue_id)
225 {
226  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
227  &queue_id);
228  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
229  &vec);
230  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
231  &vec);
232  return vec;
233 }
234 
235 static u32
237  u32 flags)
238 {
239  return 0;
240 }
241 
242 static clib_error_t *
244 {
245  virtio_main_t *vim = &virtio_main;
246  virtio_net_config_t config;
247  clib_error_t *error = 0;
248  u16 max_queue_pairs = 1;
249 
250  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
251  {
253  sizeof (config.max_virtqueue_pairs),
255  max_virtqueue_pairs));
256  max_queue_pairs = config.max_virtqueue_pairs;
257  }
258 
259  virtio_log_debug (vim, vif, "max queue pair is %x", max_queue_pairs);
260  if (max_queue_pairs < 1 || max_queue_pairs > 0x8000)
261  return clib_error_return (error, "max queue pair is %x", max_queue_pairs);
262 
263  vif->max_queue_pairs = max_queue_pairs;
264  return error;
265 }
266 
267 static void
269 {
271  sizeof (vif->mac_addr), 0);
272 }
273 
274 static u32
276 {
277  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC))
278  {
280  sizeof (vif->mac_addr), 0);
281  return 0;
282  }
283  return 1;
284 }
285 
286 static u16
288 {
289  /*
290  * Minimal driver: assumes link is up
291  */
292  u16 status = 1;
293  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_STATUS))
294  virtio_pci_legacy_read_config (vm, vif, &status, sizeof (status), /* mac */
296  status));
297  return status;
298 }
299 
300 static void
302 {
303  vnet_main_t *vnm = vnet_get_main ();
304  virtio_main_t *vim = &virtio_main;
305  uword pd = vlib_pci_get_private_data (vm, h);
306  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
307  u16 qid = line;
308 
310 }
311 
312 static void
314 {
315  vnet_main_t *vnm = vnet_get_main ();
316  virtio_main_t *vim = &virtio_main;
317  uword pd = vlib_pci_get_private_data (vm, h);
318  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
319 
321  {
322  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
325  }
326  else
327  {
328  vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
330  }
331 }
332 
333 static void
335 {
336  virtio_main_t *vim = &virtio_main;
337  uword pd = vlib_pci_get_private_data (vm, h);
338  virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
339  u8 isr = 0;
340  u16 line = 0;
341 
342  isr = virtio_pci_legacy_get_isr (vm, vif);
343 
344  /*
345  * If the lower bit is set: look through the used rings of
346  * all virtqueues for the device, to see if any progress has
347  * been made by the device which requires servicing.
348  */
349  if (isr & VIRTIO_PCI_ISR_INTR)
350  virtio_pci_irq_0_handler (vm, h, line);
351 
352  if (isr & VIRTIO_PCI_ISR_CONFIG)
353  virtio_pci_irq_1_handler (vm, h, line);
354 }
355 
356 inline void
358 {
359  struct status_struct
360  {
361  u8 bit;
362  char *str;
363  };
364  struct status_struct *status_entry;
365  static struct status_struct status_array[] = {
366 #define _(s,b) { .str = #s, .bit = b, },
368 #undef _
369  {.str = NULL}
370  };
371 
372  vlib_cli_output (vm, " status 0x%x", vif->status);
373 
374  status_entry = (struct status_struct *) &status_array;
375  while (status_entry->str)
376  {
377  if (vif->status & status_entry->bit)
378  vlib_cli_output (vm, " %s (%x)", status_entry->str,
379  status_entry->bit);
380  status_entry++;
381  }
382 }
383 
384 inline void
386 {
387  u32 data_u32;
388  u16 data_u16;
389  u8 data_u8;
390  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
391  &data_u32);
392  vlib_cli_output (vm, "remote features 0x%lx", data_u32);
393  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
394  &data_u32);
395  vlib_cli_output (vm, "guest features 0x%lx", data_u32);
396  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
397  &data_u32);
398  vlib_cli_output (vm, "queue address 0x%lx", data_u32);
399  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
400  &data_u16);
401  vlib_cli_output (vm, "queue size 0x%x", data_u16);
402  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
403  &data_u16);
404  vlib_cli_output (vm, "queue select 0x%x", data_u16);
405  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
406  &data_u16);
407  vlib_cli_output (vm, "queue notify 0x%x", data_u16);
408  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &data_u8);
409  vlib_cli_output (vm, "status 0x%x", data_u8);
410  vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &data_u8);
411  vlib_cli_output (vm, "isr 0x%x", data_u8);
412 
413  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
414  {
415  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
416  &data_u16);
417  vlib_cli_output (vm, "config vector 0x%x", data_u16);
418  u16 queue_id = 0;
419  vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
420  &queue_id);
421  vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
422  &data_u16);
423  vlib_cli_output (vm, "queue vector for queue (0) 0x%x", data_u16);
424  }
425 
426  u8 mac[6];
427  virtio_pci_legacy_read_config (vm, vif, mac, sizeof (mac), 0);
428  vlib_cli_output (vm, "mac %U", format_ethernet_address, mac);
429  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to status */
430  6);
431  vlib_cli_output (vm, "link up/down status 0x%x", data_u16);
432  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16),
433  /* offset to max_virtqueue */ 8);
434  vlib_cli_output (vm, "num of virtqueue 0x%x", data_u16);
435  virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to mtu */
436  10);
437  vlib_cli_output (vm, "mtu 0x%x", data_u16);
438 
439  u32 i = PCI_CONFIG_SIZE (vif) + 12, a = 4;
440  i += a;
441  i &= ~a;
442  for (; i < 64; i += 4)
443  {
444  u32 data = 0;
445  vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, i, &data);
446  vlib_cli_output (vm, "0x%lx", data);
447  }
448 }
449 
451 {
452  struct virtio_net_ctrl_hdr ctrl;
453  struct virtio_net_ctrl_mq num_mqs;
454  virtio_net_ctrl_ack status;
455 };
456 
457 static int
459  u16 num_queues)
460 {
461  virtio_main_t *vim = &virtio_main;
462  virtio_vring_t *vring = vif->cxq_vring;
463  u32 buffer_index;
464  vlib_buffer_t *b;
465  u16 used, next, avail;
466  u16 sz = vring->size;
467  u16 mask = sz - 1;
468  struct virtio_ctrl_mq_status_hdr mq_hdr, result;
469  virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
470 
471  mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ;
473  mq_hdr.status = VIRTIO_NET_ERR;
474  mq_hdr.num_mqs.virtqueue_pairs = num_queues;
475 
476  used = vring->desc_in_use;
477  next = vring->desc_next;
478  avail = vring->avail->idx;
479  struct vring_desc *d = &vring->desc[next];
480 
481  if (vlib_buffer_alloc (vm, &buffer_index, 1))
482  b = vlib_get_buffer (vm, buffer_index);
483  else
484  return VIRTIO_NET_ERR;
485  /*
486  * current_data may not be initialized with 0 and may contain
487  * previous offset.
488  */
489  b->current_data = 0;
490  clib_memcpy (vlib_buffer_get_current (b), &mq_hdr,
491  sizeof (struct virtio_ctrl_mq_status_hdr));
492  d->flags = VRING_DESC_F_NEXT;
493  d->addr = vlib_buffer_get_current_pa (vm, b);
494  d->len = sizeof (struct virtio_net_ctrl_hdr);
495  vring->avail->ring[avail & mask] = next;
496  avail++;
497  next = (next + 1) & mask;
498  d->next = next;
499  used++;
500 
501  d = &vring->desc[next];
502  d->flags = VRING_DESC_F_NEXT;
503  d->addr = vlib_buffer_get_current_pa (vm, b) +
505  d->len = sizeof (struct virtio_net_ctrl_mq);
506  next = (next + 1) & mask;
507  d->next = next;
508  used++;
509 
510  d = &vring->desc[next];
511  d->flags = VRING_DESC_F_WRITE;
512  d->addr = vlib_buffer_get_current_pa (vm, b) +
514  d->len = sizeof (mq_hdr.status);
515  next = (next + 1) & mask;
516  used++;
517 
519  vring->avail->idx = avail;
520  vring->desc_next = next;
521  vring->desc_in_use = used;
522 
523  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
524  {
525  virtio_kick (vm, vring, vif);
526  }
527 
528  clib_memset (&result, 0, sizeof (result));
529  u16 last = vring->last_used_idx, n_left = 0;
530  n_left = vring->used->idx - last;
531 
532  while (n_left)
533  {
534  struct vring_used_elem *e = &vring->used->ring[last & mask];
535  u16 slot = e->id;
536 
537  d = &vring->desc[slot];
538  while (d->flags & VRING_DESC_F_NEXT)
539  {
540  used--;
541  slot = d->next;
542  d = &vring->desc[slot];
543  }
544  used--;
545  last++;
546  n_left--;
547  }
548  vring->desc_in_use = used;
549  vring->last_used_idx = last;
550 
552  clib_memcpy (&result, vlib_buffer_get_current (b),
553  sizeof (struct virtio_ctrl_mq_status_hdr));
554 
555  virtio_log_debug (vim, vif, "multi-queue enable status on Ctrl queue : %u",
556  result.status);
557  status = result.status;
558  vlib_buffer_free (vm, &buffer_index, 1);
559  return status;
560 }
561 
562 static u8
564 {
565  if (qsz < 64 || qsz > 4096)
566  return 0;
567  if ((qsz % 64) != 0)
568  return 0;
569  return 1;
570 }
571 
572 clib_error_t *
574  u16 queue_num)
575 {
576  clib_error_t *error = 0;
577  virtio_main_t *vim = &virtio_main;
578  u16 queue_size = 0;
579  virtio_vring_t *vring;
580  struct vring vr;
581  u32 i = 0;
582  void *ptr = NULL;
583 
584  queue_size = virtio_pci_legacy_get_queue_num (vm, vif, queue_num);
585  if (!virtio_pci_queue_size_valid (queue_size))
586  clib_warning ("queue size is not valid");
587 
588  if (!is_pow2 (queue_size))
589  return clib_error_return (0, "ring size must be power of 2");
590 
591  if (queue_size > 32768)
592  return clib_error_return (0, "ring size must be 32768 or lower");
593 
594  if (queue_size == 0)
595  queue_size = 256;
596 
598  vring = vec_elt_at_index (vif->cxq_vring, 0);
599  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
601  ptr =
603  vif->numa_node);
604  if (!ptr)
605  return vlib_physmem_last_error (vm);
606  clib_memset (ptr, 0, i);
607  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
608  vring->desc = vr.desc;
609  vring->avail = vr.avail;
610  vring->used = vr.used;
611  vring->queue_id = queue_num;
612  vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
613 
614  ASSERT (vring->buffers == 0);
615 
616  vring->size = queue_size;
617  virtio_log_debug (vim, vif, "control-queue: number %u, size %u", queue_num,
618  queue_size);
619  virtio_pci_legacy_setup_queue (vm, vif, queue_num, ptr);
620  vring->kick_fd = -1;
621 
622  return error;
623 }
624 
625 clib_error_t *
627 {
628  clib_error_t *error = 0;
629  virtio_main_t *vim = &virtio_main;
631  u16 queue_size = 0;
632  virtio_vring_t *vring;
633  struct vring vr;
634  u32 i = 0;
635  void *ptr = NULL;
636 
637  queue_size = virtio_pci_legacy_get_queue_num (vm, vif, queue_num);
638  if (!virtio_pci_queue_size_valid (queue_size))
639  clib_warning ("queue size is not valid");
640 
641  if (!is_pow2 (queue_size))
642  return clib_error_return (0, "ring size must be power of 2");
643 
644  if (queue_size > 32768)
645  return clib_error_return (0, "ring size must be 32768 or lower");
646 
647  if (queue_size == 0)
648  queue_size = 256;
649 
650  if (queue_num % 2)
651  {
652  if (TX_QUEUE_ACCESS (queue_num) > vtm->n_vlib_mains)
653  return error;
656  vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num));
657  clib_spinlock_init (&vring->lockp);
658  }
659  else
660  {
663  vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num));
664  }
665  i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
667  ptr =
669  vif->numa_node);
670  if (!ptr)
671  return vlib_physmem_last_error (vm);
672  clib_memset (ptr, 0, i);
673  vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
674  vring->desc = vr.desc;
675  vring->avail = vr.avail;
676  vring->used = vr.used;
677  vring->queue_id = queue_num;
678  vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
679 
680  ASSERT (vring->buffers == 0);
681  vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
682  if (queue_num % 2)
683  {
684  vif->num_txqs++;
685  virtio_log_debug (vim, vif, "tx-queue: number %u, size %u", queue_num,
686  queue_size);
687  }
688  else
689  {
690  vif->num_rxqs++;
691  virtio_log_debug (vim, vif, "rx-queue: number %u, size %u", queue_num,
692  queue_size);
693  }
694  vring->size = queue_size;
695  virtio_pci_legacy_setup_queue (vm, vif, queue_num, ptr);
696  vring->kick_fd = -1;
697 
698  return error;
699 }
700 
701 static void
703  u64 req_features)
704 {
705  /*
706  * if features are not requested
707  * default: all supported features
708  */
709  u64 supported_features = VIRTIO_FEATURE (VIRTIO_NET_F_MTU)
710  | VIRTIO_FEATURE (VIRTIO_NET_F_MAC)
711  | VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)
712  | VIRTIO_FEATURE (VIRTIO_NET_F_STATUS)
713  | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)
714  | VIRTIO_FEATURE (VIRTIO_NET_F_MQ)
715  | VIRTIO_FEATURE (VIRTIO_F_NOTIFY_ON_EMPTY)
716  | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)
717  | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC);
718 
719  if (req_features == 0)
720  {
721  req_features = supported_features;
722  }
723 
724  vif->features = req_features & vif->remote_features & supported_features;
725 
727  {
728  virtio_net_config_t config;
729  virtio_pci_legacy_read_config (vm, vif, &config.mtu,
730  sizeof (config.mtu),
732  mtu));
733  if (config.mtu < 64)
735  }
736 
737  vif->features = virtio_pci_legacy_set_features (vm, vif, vif->features);
738 }
739 
740 void
742 {
744 }
745 
746 int
748 {
749  u8 status = 0;
750 
751  /*
752  * Reset the device
753  */
754  status = virtio_pci_legacy_reset (vm, vif);
755 
756  /*
757  * Set the Acknowledge status bit
758  */
759  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_ACK);
760 
761  /*
762  * Set the Driver status bit
763  */
764  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER);
765 
766  /*
767  * Read the status and verify it
768  */
769  status = virtio_pci_legacy_get_status (vm, vif);
770  if (!
771  ((status & VIRTIO_CONFIG_STATUS_ACK)
772  && (status & VIRTIO_CONFIG_STATUS_DRIVER)))
773  return -1;
774  vif->status = status;
775 
776  return 0;
777 }
778 
779 clib_error_t *
781 {
782  clib_error_t *error = 0;
783  virtio_main_t *vim = &virtio_main;
784  struct virtio_pci_cap cap;
785  u8 pos, common_cfg = 0, notify_base = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
787 
788  if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos)))
789  {
790  virtio_log_error (vim, vif, "error in reading capabilty list position");
791  clib_error_return (error, "error in reading capabilty list position");
792  }
793  while (pos)
794  {
795  if ((error =
796  vlib_pci_read_write_config (vm, h, VLIB_READ, pos, &cap,
797  sizeof (cap))))
798  {
799  virtio_log_error (vim, vif, "%s [%2x]",
800  "error in reading the capability at", pos);
801  clib_error_return (error,
802  "error in reading the capability at [%2x]", pos);
803  }
804 
805  if (cap.cap_vndr == PCI_CAP_ID_MSIX)
806  {
807  u16 flags, table_size, table_size_mask = 0x07FF;
808 
809  if ((error =
810  vlib_pci_read_write_config (vm, h, VLIB_READ, pos + 2, &flags,
811  sizeof (flags))))
812  clib_error_return (error,
813  "error in reading the capability at [%2x]",
814  pos + 2);
815 
816  table_size = flags & table_size_mask;
817  virtio_log_debug (vim, vif, "flags:0x%x %s 0x%x", flags,
818  "msix interrupt vector table-size", table_size);
819 
820  if (flags & PCI_MSIX_ENABLE)
821  {
822  virtio_log_debug (vim, vif, "msix interrupt enabled");
824  }
825  else
826  {
827  virtio_log_debug (vim, vif, "msix interrupt disabled");
829  }
830  }
831 
832  if (cap.cap_vndr != PCI_CAP_ID_VNDR)
833  {
834  virtio_log_debug (vim, vif, "[%2x] %s %2x ", pos,
835  "skipping non VNDR cap id:", cap.cap_vndr);
836  goto next;
837  }
838 
839  virtio_log_debug (vim, vif,
840  "[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u",
841  pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
842  switch (cap.cfg_type)
843  {
845  common_cfg = 1;
846  break;
848  notify_base = 1;
849  break;
851  dev_cfg = 1;
852  break;
854  isr = 1;
855  break;
857  if (cap.bar == 0)
858  pci_cfg = 1;
859  break;
860  }
861  next:
862  pos = cap.cap_next;
863  }
864 
865  if (common_cfg == 0 || notify_base == 0 || dev_cfg == 0 || isr == 0)
866  {
867  virtio_log_debug (vim, vif, "legacy virtio pci device found");
868  return error;
869  }
870 
871  if (!pci_cfg)
872  clib_error_return (error, "modern virtio pci device found");
873 
874  virtio_log_debug (vim, vif, "transitional virtio pci device found");
875  return error;
876 }
877 
878 static clib_error_t *
881 {
882  clib_error_t *error = 0;
883  virtio_main_t *vim = &virtio_main;
884  u8 status = 0;
885 
886  if ((error = virtio_pci_read_caps (vm, vif)))
887  clib_error_return (error, "Device is not supported");
888 
889  if (virtio_pci_reset_device (vm, vif) < 0)
890  {
891  virtio_log_error (vim, vif, "Failed to reset the device");
892  clib_error_return (error, "Failed to reset the device");
893  }
894  /*
895  * read device features and negotiate (user) requested features
896  */
898  virtio_negotiate_features (vm, vif, args->features);
899 
900  /*
901  * After FEATURE_OK, driver should not accept new feature bits
902  */
903  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_FEATURES_OK);
904  status = virtio_pci_legacy_get_status (vm, vif);
905  if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
906  {
907  virtio_log_error (vim, vif,
908  "error encountered: Device doesn't support requested features");
909  clib_error_return (error, "Device doesn't support requested features");
910  }
911  vif->status = status;
912 
913  /*
914  * get or set the mac address
915  */
916  if (virtio_pci_get_mac (vm, vif))
917  {
918  f64 now = vlib_time_now (vm);
919  u32 rnd;
920  rnd = (u32) (now * 1e6);
921  rnd = random_u32 (&rnd);
922 
923  memcpy (vif->mac_addr + 2, &rnd, sizeof (rnd));
924  vif->mac_addr[0] = 2;
925  vif->mac_addr[1] = 0xfe;
926  virtio_pci_set_mac (vm, vif);
927  }
928 
930 
931  /*
932  * Initialize the virtqueues
933  */
934  if ((error = virtio_pci_get_max_virtqueue_pairs (vm, vif)))
935  goto err;
936 
937  for (int i = 0; i < vif->max_queue_pairs; i++)
938  {
939  if ((error = virtio_pci_vring_init (vm, vif, RX_QUEUE (i))))
940  virtio_log_warning (vim, vif, "%s (%u) %s", "error in rxq-queue",
941  RX_QUEUE (i), "initialization");
942 
943  if ((error = virtio_pci_vring_init (vm, vif, TX_QUEUE (i))))
944  virtio_log_warning (vim, vif, "%s (%u) %s", "error in txq-queue",
945  TX_QUEUE (i), "initialization");
946  }
947 
948  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
949  {
950  if ((error =
951  virtio_pci_control_vring_init (vm, vif, vif->max_queue_pairs * 2)))
952  {
953  virtio_log_warning (vim, vif, "%s (%u) %s",
954  "error in control-queue",
955  vif->max_queue_pairs * 2, "initialization");
956  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
957  vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MQ);
958  }
959  }
960  else
961  {
962  virtio_log_debug (vim, vif, "control queue is not available");
963  vif->cxq_vring = NULL;
964  }
965 
966  /*
967  * set the msix interrupts
968  */
969  if (vif->msix_enabled == VIRTIO_MSIX_ENABLED)
970  {
971  if (virtio_pci_legacy_set_config_irq (vm, vif, 1) ==
972  VIRTIO_MSI_NO_VECTOR)
973  virtio_log_warning (vim, vif, "config vector 1 is not set");
974  if (virtio_pci_legacy_set_queue_irq (vm, vif, 0, 0) ==
975  VIRTIO_MSI_NO_VECTOR)
976  virtio_log_warning (vim, vif, "queue vector 0 is not set");
977  }
978 
979  /*
980  * set the driver status OK
981  */
982  virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER_OK);
983  vif->status = virtio_pci_legacy_get_status (vm, vif);
984 err:
985  return error;
986 }
987 
988 void
990 {
991  vnet_main_t *vnm = vnet_get_main ();
992  virtio_main_t *vim = &virtio_main;
993  virtio_if_t *vif;
995  clib_error_t *error = 0;
996 
997  if (args->rxq_size == 0)
999  if (args->txq_size == 0)
1000  args->txq_size = VIRTIO_NUM_TX_DESC;
1001 
1002  if (!virtio_pci_queue_size_valid (args->rxq_size) ||
1004  {
1005  args->rv = VNET_API_ERROR_INVALID_VALUE;
1006  args->error =
1007  clib_error_return (error,
1008  "queue size must be <= 4096, >= 64, "
1009  "and multiples of 64");
1010  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1011  format_vlib_pci_addr, &args->addr,
1012  "queue size must be <= 4096, >= 64, and multiples of 64");
1013  return;
1014  }
1015 
1016  /* *INDENT-OFF* */
1017  pool_foreach (vif, vim->interfaces, ({
1018  if (vif->pci_addr.as_u32 == args->addr)
1019  {
1020  args->rv = VNET_API_ERROR_INVALID_VALUE;
1021  args->error =
1022  clib_error_return (error, "PCI address in use");
1023  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1024  format_vlib_pci_addr, &args->addr,
1025  " PCI address in use");
1026  return;
1027  }
1028  }));
1029  /* *INDENT-ON* */
1030 
1031  pool_get (vim->interfaces, vif);
1032  vif->dev_instance = vif - vim->interfaces;
1033  vif->per_interface_next_index = ~0;
1034  vif->pci_addr.as_u32 = args->addr;
1035 
1036  if ((error =
1037  vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr,
1038  virtio_pci_device_ids, &h)))
1039  {
1040  pool_put (vim->interfaces, vif);
1041  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1042  args->error =
1043  clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
1044  &vif->pci_addr);
1045  vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
1046  format_vlib_pci_addr, &vif->pci_addr,
1047  "error encountered on pci device open");
1048  return;
1049  }
1050  vif->pci_dev_handle = h;
1051  vlib_pci_set_private_data (vm, h, vif->dev_instance);
1052  vif->numa_node = vlib_pci_get_numa_node (vm, h);
1053 
1054  if ((error = vlib_pci_bus_master_enable (vm, h)))
1055  {
1056  virtio_log_error (vim, vif,
1057  "error encountered on pci bus master enable");
1058  goto error;
1059  }
1060 
1061  if ((error = vlib_pci_io_region (vm, h, 0)))
1062  {
1063  virtio_log_error (vim, vif, "error encountered on pci io region");
1064  goto error;
1065  }
1066 
1068  {
1069  if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
1071  {
1072  virtio_log_error (vim, vif,
1073  "error encountered on pci register msix handler 0");
1074  goto error;
1075  }
1076  if ((error = vlib_pci_register_msix_handler (vm, h, 1, 1,
1078  {
1079  virtio_log_error (vim, vif,
1080  "error encountered on pci register msix handler 1");
1081  goto error;
1082  }
1083 
1084  if ((error = vlib_pci_enable_msix_irq (vm, h, 0, 2)))
1085  {
1086  virtio_log_error (vim, vif,
1087  "error encountered on pci enable msix irq");
1088  goto error;
1089  }
1090  vif->support_int_mode = 1;
1091  virtio_log_debug (vim, vif, "device supports msix interrupts");
1092  }
1093  else if (vlib_pci_get_num_msix_interrupts (vm, h) == 1)
1094  {
1095  /*
1096  * if msix table-size is 1, fall back to intX.
1097  */
1098  if ((error =
1100  {
1101  virtio_log_error (vim, vif,
1102  "error encountered on pci register interrupt handler");
1103  goto error;
1104  }
1105  vif->support_int_mode = 1;
1106  virtio_log_debug (vim, vif, "pci register interrupt handler");
1107  }
1108  else
1109  {
1110  /*
1111  * WARN: intX is showing some weird behaviour.
1112  * Please don't use interrupt mode with UIO driver.
1113  */
1114  vif->support_int_mode = 0;
1115  virtio_log_debug (vim, vif, "driver is configured in poll mode only");
1116  }
1117 
1118  if ((error = vlib_pci_intr_enable (vm, h)))
1119  {
1120  virtio_log_error (vim, vif,
1121  "error encountered on pci interrupt enable");
1122  goto error;
1123  }
1124 
1125  if ((error = virtio_pci_device_init (vm, vif, args)))
1126  {
1127  virtio_log_error (vim, vif, "error encountered on device init");
1128  goto error;
1129  }
1130 
1131  vif->type = VIRTIO_IF_TYPE_PCI;
1132  /* create interface */
1134  vif->dev_instance, vif->mac_addr,
1135  &vif->hw_if_index,
1137 
1138  if (error)
1139  {
1140  virtio_log_error (vim, vif,
1141  "error encountered on ethernet register interface");
1142  goto error;
1143  }
1144 
1145  vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vif->hw_if_index);
1146  vif->sw_if_index = sw->sw_if_index;
1147  args->sw_if_index = sw->sw_if_index;
1148 
1149  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
1151  vnet_hw_interface_set_input_node (vnm, vif->hw_if_index,
1152  virtio_input_node.index);
1153  u32 i = 0;
1154  vec_foreach_index (i, vif->rxq_vrings)
1155  {
1156  vnet_hw_interface_assign_rx_thread (vnm, vif->hw_if_index, i, ~0);
1158  /* Set default rx mode to POLLING */
1159  vnet_hw_interface_set_rx_mode (vnm, vif->hw_if_index, i,
1161  }
1163  {
1164  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
1165  vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
1167  }
1168  else
1169  vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
1170 
1171  if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) &&
1172  (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ)))
1173  {
1174  if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs))
1175  virtio_log_warning (vim, vif, "multiqueue is not set");
1176  }
1177  return;
1178 
1179 error:
1180  virtio_pci_delete_if (vm, vif);
1181  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1182  args->error = error;
1183 }
1184 
1185 int
1187 {
1188  vnet_main_t *vnm = vnet_get_main ();
1189  virtio_main_t *vim = &virtio_main;
1190  u32 i = 0;
1191 
1192  if (vif->type != VIRTIO_IF_TYPE_PCI)
1193  return VNET_API_ERROR_INVALID_INTERFACE;
1194 
1196 
1197  for (i = 0; i < vif->max_queue_pairs; i++)
1198  {
1199  virtio_pci_legacy_del_queue (vm, vif, RX_QUEUE (i));
1200  virtio_pci_legacy_del_queue (vm, vif, TX_QUEUE (i));
1201  }
1202 
1203  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
1204  virtio_pci_legacy_del_queue (vm, vif, vif->max_queue_pairs * 2);
1205 
1206  virtio_pci_legacy_reset (vm, vif);
1207 
1208  if (vif->hw_if_index)
1209  {
1211  vec_foreach_index (i, vif->rxq_vrings)
1212  {
1214  }
1216  }
1217 
1219 
1220  vec_foreach_index (i, vif->rxq_vrings)
1221  {
1222  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i);
1223  if (vring->kick_fd != -1)
1224  close (vring->kick_fd);
1225  if (vring->used)
1226  {
1227  virtio_free_rx_buffers (vm, vring);
1228  }
1229  vec_free (vring->buffers);
1230  vlib_physmem_free (vm, vring->desc);
1231  }
1232 
1233  vec_foreach_index (i, vif->txq_vrings)
1234  {
1235  virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i);
1236  if (vring->kick_fd != -1)
1237  close (vring->kick_fd);
1238  if (vring->used)
1239  {
1240  virtio_free_used_desc (vm, vring);
1241  }
1242  vec_free (vring->buffers);
1243  vlib_physmem_free (vm, vring->desc);
1244  }
1245 
1246  if (vif->cxq_vring != NULL)
1247  {
1248  u16 last = vif->cxq_vring->last_used_idx;
1249  u16 n_left = vif->cxq_vring->used->idx - last;
1250  while (n_left)
1251  {
1252  last++;
1253  n_left--;
1254  }
1255 
1256  vif->cxq_vring->last_used_idx = last;
1257  vlib_physmem_free (vm, vif->cxq_vring->desc);
1258  }
1259 
1260  vec_free (vif->rxq_vrings);
1261  vec_free (vif->txq_vrings);
1262  vec_free (vif->cxq_vring);
1263 
1264  if (vif->fd != -1)
1265  vif->fd = -1;
1266  if (vif->tap_fd != -1)
1267  vif->tap_fd = -1;
1268  clib_error_free (vif->error);
1269  memset (vif, 0, sizeof (*vif));
1270  pool_put (vim->interfaces, vif);
1271 
1272  return 0;
1273 }
1274 
1275 /*
1276  * fd.io coding-style-patch-verification: ON
1277  *
1278  * Local Variables:
1279  * eval: (c-set-style "gnu")
1280  * End:
1281  */
static u8 virtio_pci_legacy_get_isr(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:161
struct vring_used * used
Definition: virtio.h:105
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:427
#define vec_foreach_index(var, v)
Iterate over vector indices.
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:239
vlib_node_registration_t virtio_input_node
(constructor) VLIB_REGISTER_NODE (virtio_input_node)
Definition: node.c:384
void virtio_set_net_hdr_size(virtio_if_t *vif)
Definition: virtio.c:257
virtio_if_t * interfaces
Definition: virtio.h:191
u32 flags
Definition: vhost_user.h:115
clib_error_t * virtio_pci_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:626
static u32 virtio_pci_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: pci.c:236
static void virtio_pci_legacy_del_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:193
static clib_error_t * vlib_pci_bus_master_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:271
static void * vlib_physmem_alloc_aligned_on_numa(vlib_main_t *vm, uword n_bytes, uword alignment, u32 numa_node)
Definition: physmem_funcs.h:63
#define VIRTIO_PCI_CAP_ISR_CFG
Definition: pci.h:113
a
Definition: bitmap.h:538
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
u16 max_virtqueue_pairs
Definition: virtio.h:95
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:324
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
static u16 virtio_pci_legacy_get_queue_num(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:169
#define VIRTIO_MSI_QUEUE_VECTOR
Definition: pci.h:36
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1275
#define VIRTIO_PCI_STATUS
Definition: pci.h:31
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:118
#define VIRTIO_PCI_QUEUE_PFN
Definition: pci.h:27
#define NULL
Definition: clib.h:58
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static u8 virtio_pci_legacy_get_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:138
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:255
u32 vlib_pci_get_num_msix_interrupts(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:177
static clib_error_t * vlib_physmem_last_error(struct vlib_main_t *vm)
static u32 virtio_pci_legacy_set_features(vlib_main_t *vm, virtio_if_t *vif, u64 features)
Definition: pci.c:122
static clib_error_t * vlib_pci_intr_disable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:255
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_device_class_t virtio_device_class
static clib_error_t * virtio_pci_device_init(vlib_main_t *vm, virtio_if_t *vif, virtio_pci_create_if_args_t *args)
Definition: pci.c:879
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
int i
void virtio_vring_set_numa_node(vlib_main_t *vm, virtio_if_t *vif, u32 idx)
Definition: virtio.c:241
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:65
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:883
#define VIRTIO_MSI_CONFIG_VECTOR
Definition: pci.h:35
u8 data[128]
Definition: ipsec.api:248
vl_api_ip4_address_t dst
Definition: ipsec_gre.api:39
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:236
vhost_vring_addr_t addr
Definition: vhost_user.h:121
#define VIRTIO_PCI_HOST_FEATURES
Definition: pci.h:25
static void virtio_pci_set_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:268
unsigned char u8
Definition: types.h:56
void virtio_pci_create_if(vlib_main_t *vm, virtio_pci_create_if_args_t *args)
Definition: pci.c:989
double f64
Definition: types.h:142
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define clib_memcpy(d, s, n)
Definition: string.h:180
static void virtio_pci_irq_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:334
clib_error_t * vlib_pci_read_write_config(vlib_main_t *vm, vlib_pci_dev_handle_t h, vlib_read_or_write_t read_or_write, uword address, void *data, u32 n_bytes)
Definition: pci.c:1017
u32 msix_enabled
Definition: virtio.h:154
#define VIRTIO_PCI_CAP_COMMON_CFG
Definition: pci.h:109
struct vring_avail * avail
Definition: virtio.h:106
u64 features
Definition: virtio.h:163
int virtio_pci_reset_device(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:747
u32 hw_if_index
Definition: virtio.h:140
u8 * format_ethernet_address(u8 *s, va_list *args)
Definition: format.c:44
static u8 virtio_pci_queue_size_valid(u16 qsz)
Definition: pci.c:563
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:493
static u16 virtio_pci_legacy_set_queue_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec, u16 queue_id)
Definition: pci.c:223
#define PCI_DEVICE_ID_VIRTIO_NIC
Definition: pci.c:29
#define TX_QUEUE_ACCESS(X)
Definition: virtio.h:80
void device_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:357
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
static void virtio_negotiate_features(vlib_main_t *vm, virtio_if_t *vif, u64 req_features)
Definition: pci.c:702
virtio_net_ctrl_ack status
Definition: pci.c:454
vnet_hw_interface_flags_t flags
Definition: interface.h:494
void virtio_free_rx_buffers(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:155
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
clib_spinlock_t lockp
Definition: virtio.h:107
struct virtio_net_ctrl_hdr ctrl
Definition: pci.c:452
#define clib_error_return(e, args...)
Definition: error.h:99
#define VIRTIO_PCI_CAP_NOTIFY_CFG
Definition: pci.h:111
unsigned int u32
Definition: types.h:88
static void virtio_pci_legacy_read_config(vlib_main_t *vm, virtio_if_t *vif, void *dst, int len, u32 addr)
Definition: pci.c:53
#define virtio_log_debug(vim, vif, f,...)
Definition: pci.h:123
u16 queue_id
Definition: virtio.h:114
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
u16 num_txqs
Definition: virtio.h:170
u32 pci_dev_handle
Definition: virtio.h:159
#define TX_QUEUE(X)
Definition: virtio.h:78
static void virtio_pci_legacy_set_status(vlib_main_t *vm, virtio_if_t *vif, u8 status)
Definition: pci.c:146
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
uword size
#define VIRTIO_PCI_GUEST_FEATURES
Definition: pci.h:26
#define VIRTIO_NUM_RX_DESC
Definition: virtio.h:73
#define VIRTIO_NET_F_MTU
Definition: pci.h:104
int tap_fd
Definition: virtio.h:158
static void virtio_pci_irq_1_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:313
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:155
unsigned short u16
Definition: types.h:57
#define VIRTIO_PCI_ISR_CONFIG
Definition: pci.h:48
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:826
static u64 virtio_pci_legacy_get_features(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:113
#define VIRTIO_NUM_TX_DESC
Definition: virtio.h:74
#define PCI_MSIX_ENABLE
Definition: pci.c:37
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:148
vl_api_ip4_address_t src
Definition: ipsec_gre.api:38
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
static u16 virtio_pci_legacy_set_config_irq(vlib_main_t *vm, virtio_if_t *vif, u16 vec)
Definition: pci.c:212
void virtio_pci_read_device_feature(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:741
clib_error_t * virtio_pci_read_caps(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:780
static clib_error_t * virtio_pci_get_max_virtqueue_pairs(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:243
#define PCI_CAPABILITY_LIST
Definition: pci.c:33
u8 status
Definition: virtio.h:171
u8 len
Definition: ip_types.api:49
u16 desc_next
Definition: virtio.h:109
u8 * format_vlib_pci_addr(u8 *s, va_list *va)
Definition: pci.c:141
#define VIRTIO_PCI_VRING_ALIGN
Definition: pci.h:121
u16 num_rxqs
Definition: virtio.h:169
virtio_vring_t * rxq_vrings
Definition: virtio.h:161
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:170
u16 last_used_idx
Definition: virtio.h:118
#define VIRTIO_PCI_QUEUE_NOTIFY
Definition: pci.h:30
vlib_main_t * vm
Definition: buffer.c:312
#define VIRTIO_PCI_ISR_INTR
Definition: pci.h:46
static void vlib_physmem_free(vlib_main_t *vm, void *p)
Definition: physmem_funcs.h:89
#define VIRTIO_RING_FLAG_MASK_INT
Definition: virtio.h:99
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
#define clib_warning(format, args...)
Definition: error.h:59
static u16 virtio_pci_is_link_up(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:287
void debug_device_config_space(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:385
void vlib_log(vlib_log_level_t level, vlib_log_class_t class, char *fmt,...)
Definition: log.c:138
u8 mac_addr[6]
Definition: virtio.h:172
static void virtio_pci_legacy_write_config(vlib_main_t *vm, virtio_if_t *vif, void *src, int len, u32 addr)
Definition: pci.c:83
u32 flags
Definition: virtio.h:137
clib_error_t * virtio_pci_control_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 queue_num)
Definition: pci.c:573
clib_error_t * error
Definition: virtio.h:166
vlib_log_class_t log_default
Definition: virtio.h:189
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:241
void virtio_pci_legacy_notify_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id)
Definition: pci.c:203
int virtio_pci_delete_if(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:1186
u32 numa_node
Definition: virtio.h:142
virtio_if_type_t type
Definition: virtio.h:144
void virtio_free_used_desc(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:192
#define PCI_DEVICE_ID_VIRTIO_NIC_MODERN
Definition: pci.c:31
#define PCI_CONFIG_SIZE(vif)
Definition: pci.c:39
#define ASSERT(truth)
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
u64 remote_features
Definition: virtio.h:163
#define VIRTIO_FEATURE(X)
Definition: virtio.h:76
clib_error_t * vlib_pci_register_intx_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, pci_intx_handler_function_t *intx_handler)
Definition: pci.c:775
clib_error_t * error
Definition: pci.h:238
virtio_main_t virtio_main
Definition: virtio.c:37
static uword is_pow2(uword x)
Definition: clib.h:235
#define VIRTIO_PCI_QUEUE_SEL
Definition: pci.h:29
Definition: defs.h:56
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
#define RX_QUEUE_ACCESS(X)
Definition: virtio.h:81
#define VIRTIO_PCI_CAP_DEVICE_CFG
Definition: pci.h:115
#define PCI_CAP_ID_VNDR
Definition: pci.c:34
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:278
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:504
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
int fd
Definition: virtio.h:153
static void virtio_pci_irq_0_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: pci.c:301
#define virtio_log_warning(vim, vif, f,...)
Definition: pci.h:130
#define virtio_log_error(vim, vif, f,...)
Definition: pci.h:137
#define clib_error_free(e)
Definition: error.h:86
clib_error_t * vlib_pci_io_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource)
Definition: pci.c:1160
u32 * buffers
Definition: virtio.h:117
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
static u8 virtio_pci_legacy_reset(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:154
virtio_vring_t * cxq_vring
Definition: virtio.h:183
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
struct vring_desc * desc
Definition: virtio.h:104
static void virtio_pci_legacy_setup_queue(vlib_main_t *vm, virtio_if_t *vif, u16 queue_id, void *p)
Definition: pci.c:182
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:115
int vnet_hw_interface_set_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode mode)
Definition: devices.c:253
#define VIRTIO_PCI_CAP_PCI_CFG
Definition: pci.h:117
u16 max_queue_pairs
Definition: virtio.h:168
struct virtio_net_ctrl_mq num_mqs
Definition: pci.c:453
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define PCI_VENDOR_ID_VIRTIO
Definition: pci.c:28
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:612
#define PCI_CAP_ID_MSIX
Definition: pci.c:35
#define VIRTIO_NET_S_LINK_UP
Definition: pci.h:105
vl_api_mac_address_t mac
Definition: gbp.api:120
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:762
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1215
#define RX_QUEUE(X)
Definition: virtio.h:79
static u32 virtio_pci_get_mac(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:275
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
#define VIRTIO_NET_CTRL_MQ
Definition: vhost_user.h:40
#define VIRTIO_PCI_QUEUE_NUM
Definition: pci.h:28
u16 vendor_id
Definition: pci.h:127
static int virtio_pci_enable_multiqueue(vlib_main_t *vm, virtio_if_t *vif, u16 num_queues)
Definition: pci.c:458
#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET
Definition: vhost_user.h:41
u16 desc_in_use
Definition: virtio.h:108
virtio_vring_t * txq_vrings
Definition: virtio.h:162
#define VIRTIO_PCI_QUEUE_ADDR_SHIFT
Definition: pci.h:119
static_always_inline void virtio_kick(vlib_main_t *vm, virtio_vring_t *vring, virtio_if_t *vif)
Definition: virtio.h:216
#define VIRTIO_PCI_ISR
Definition: pci.h:32