FD.io VPP  v19.04.4-rc0-5-ge88582fac
Vector Packet Processing
virtio.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2017 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <fcntl.h>
21 #include <net/if.h>
22 #include <linux/if_tun.h>
23 #include <sys/ioctl.h>
24 #include <linux/virtio_net.h>
25 #include <linux/vhost.h>
26 #include <sys/eventfd.h>
27 
28 #include <vlib/vlib.h>
29 #include <vlib/pci/pci.h>
30 #include <vlib/unix/unix.h>
31 #include <vnet/ethernet/ethernet.h>
32 #include <vnet/ip/ip4_packet.h>
33 #include <vnet/ip/ip6_packet.h>
36 
38 
39 #define _IOCTL(fd,a,...) \
40  if (ioctl (fd, a, __VA_ARGS__) < 0) \
41  { \
42  err = clib_error_return_unix (0, "ioctl(" #a ")"); \
43  goto error; \
44  }
45 
46 static clib_error_t *
48 {
50  vnet_main_t *vnm = vnet_get_main ();
51  u16 qid = uf->private_data & 0xFFFF;
52  virtio_if_t *vif =
53  vec_elt_at_index (nm->interfaces, uf->private_data >> 16);
54  u64 b;
55 
56  CLIB_UNUSED (ssize_t size) = read (uf->file_descriptor, &b, sizeof (b));
57  if ((qid & 1) == 0)
59 
60  return 0;
61 }
62 
63 
66 {
67  clib_error_t *err = 0;
68  virtio_vring_t *vring;
69  struct vhost_vring_state state = { 0 };
70  struct vhost_vring_addr addr = { 0 };
71  struct vhost_vring_file file = { 0 };
72  clib_file_t t = { 0 };
73  int i;
74 
75  if (!is_pow2 (sz))
76  return clib_error_return (0, "ring size must be power of 2");
77 
78  if (sz > 32768)
79  return clib_error_return (0, "ring size must be 32768 or lower");
80 
81  if (sz == 0)
82  sz = 256;
83 
84  if (idx % 2)
85  {
89  vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (idx));
90  if (thm->n_vlib_mains > 1)
91  clib_spinlock_init (&vring->lockp);
92  }
93  else
94  {
97  vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (idx));
98  }
99  i = sizeof (struct vring_desc) * sz;
102  clib_memset (vring->desc, 0, i);
103 
104  i = sizeof (struct vring_avail) + sz * sizeof (vring->avail->ring[0]);
107  clib_memset (vring->avail, 0, i);
108  // tell kernel that we don't need interrupt
109  vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
110 
111  i = sizeof (struct vring_used) + sz * sizeof (struct vring_used_elem);
114  clib_memset (vring->used, 0, i);
115 
116  vring->queue_id = idx;
117  ASSERT (vring->buffers == 0);
119 
120  vring->size = sz;
121  vring->call_fd = eventfd (0, EFD_NONBLOCK | EFD_CLOEXEC);
122  vring->kick_fd = eventfd (0, EFD_CLOEXEC);
123 
125  t.file_descriptor = vring->call_fd;
126  t.private_data = vif->dev_instance << 16 | idx;
127  t.description = format (0, "%U vring %u", format_virtio_device_name,
128  vif->dev_instance, idx);
129  vring->call_file_index = clib_file_add (&file_main, &t);
130 
131  state.index = idx;
132  state.num = sz;
133  _IOCTL (vif->fd, VHOST_SET_VRING_NUM, &state);
134 
135  addr.index = idx;
136  addr.flags = 0;
137  addr.desc_user_addr = pointer_to_uword (vring->desc);
138  addr.avail_user_addr = pointer_to_uword (vring->avail);
139  addr.used_user_addr = pointer_to_uword (vring->used);
140  _IOCTL (vif->fd, VHOST_SET_VRING_ADDR, &addr);
141 
142  file.index = idx;
143  file.fd = vring->kick_fd;
144  _IOCTL (vif->fd, VHOST_SET_VRING_KICK, &file);
145  file.fd = vring->call_fd;
146  _IOCTL (vif->fd, VHOST_SET_VRING_CALL, &file);
147  file.fd = vif->tap_fd;
148  _IOCTL (vif->fd, VHOST_NET_SET_BACKEND, &file);
149 
150 error:
151  return err;
152 }
153 
154 inline void
156 {
157  u16 used = vring->desc_in_use;
158  u16 last = vring->last_used_idx;
159  u16 mask = vring->size - 1;
160 
161  while (used)
162  {
163  vlib_buffer_free (vm, &vring->buffers[last & mask], 1);
164  last++;
165  used--;
166  }
167 }
168 
169 clib_error_t *
171 {
172  virtio_vring_t *vring =
174 
176  close (vring->kick_fd);
177  close (vring->call_fd);
178  if (vring->used)
179  {
180  virtio_free_rx_buffers (vm, vring);
181  clib_mem_free (vring->used);
182  }
183  if (vring->desc)
184  clib_mem_free (vring->desc);
185  if (vring->avail)
186  clib_mem_free (vring->avail);
187  vec_free (vring->buffers);
188  return 0;
189 }
190 
191 inline void
193 {
194  u16 used = vring->desc_in_use;
195  u16 sz = vring->size;
196  u16 mask = sz - 1;
197  u16 last = vring->last_used_idx;
198  u16 n_left = vring->used->idx - last;
199 
200  if (n_left == 0)
201  return;
202 
203  while (n_left)
204  {
205  struct vring_used_elem *e = &vring->used->ring[last & mask];
206  u16 slot = e->id;
207 
208  vlib_buffer_free (vm, &vring->buffers[slot], 1);
209  used--;
210  last++;
211  n_left--;
212  }
213  vring->desc_in_use = used;
214  vring->last_used_idx = last;
215 }
216 
217 clib_error_t *
219 {
220  virtio_vring_t *vring =
222 
224  close (vring->kick_fd);
225  close (vring->call_fd);
226  if (vring->used)
227  {
228  virtio_free_used_desc (vm, vring);
229  clib_mem_free (vring->used);
230  }
231  if (vring->desc)
232  clib_mem_free (vring->desc);
233  if (vring->avail)
234  clib_mem_free (vring->avail);
235  vec_free (vring->buffers);
236  clib_spinlock_free (&vring->lockp);
237  return 0;
238 }
239 
240 void
242 {
243  vnet_main_t *vnm = vnet_get_main ();
244  u32 thread_index;
245  virtio_vring_t *vring =
247  thread_index =
249  RX_QUEUE_ACCESS (idx));
250  vring->buffer_pool_index =
252  vlib_mains
253  [thread_index]->numa_node);
254 }
255 
256 inline void
258 {
259  if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF) ||
260  vif->features & VIRTIO_FEATURE (VIRTIO_F_VERSION_1))
261  vif->virtio_net_hdr_sz = sizeof (struct virtio_net_hdr_v1);
262  else
263  vif->virtio_net_hdr_sz = sizeof (struct virtio_net_hdr);
264 }
265 
266 inline void
267 virtio_show (vlib_main_t * vm, u32 * hw_if_indices, u8 show_descr, u32 type)
268 {
269  u32 i, j, hw_if_index;
270  virtio_if_t *vif;
271  vnet_main_t *vnm = &vnet_main;
272  virtio_main_t *mm = &virtio_main;
273  virtio_vring_t *vring;
274  struct feat_struct
275  {
276  u8 bit;
277  char *str;
278  };
279  struct feat_struct *feat_entry;
280 
281  static struct feat_struct feat_array[] = {
282 #define _(s,b) { .str = #s, .bit = b, },
284 #undef _
285  {.str = NULL}
286  };
287 
288  struct feat_struct *flag_entry;
289  static struct feat_struct flags_array[] = {
290 #define _(b,e,s) { .bit = b, .str = s, },
292 #undef _
293  {.str = NULL}
294  };
295 
296  if (!hw_if_indices)
297  return;
298 
299  for (hw_if_index = 0; hw_if_index < vec_len (hw_if_indices); hw_if_index++)
300  {
302  vnet_get_hw_interface (vnm, hw_if_indices[hw_if_index]);
303  vif = pool_elt_at_index (mm->interfaces, hi->dev_instance);
304  if (vif->type != type)
305  continue;
306  vlib_cli_output (vm, "Interface: %U (ifindex %d)",
308  hw_if_indices[hw_if_index], vif->hw_if_index);
309  if (type == VIRTIO_IF_TYPE_PCI)
310  {
311  vlib_cli_output (vm, " PCI Address: %U", format_vlib_pci_addr,
312  &vif->pci_addr);
313  }
314  if (type == VIRTIO_IF_TYPE_TAP)
315  {
316  if (vif->host_if_name)
317  vlib_cli_output (vm, " name \"%s\"", vif->host_if_name);
318  if (vif->net_ns)
319  vlib_cli_output (vm, " host-ns \"%s\"", vif->net_ns);
320  vlib_cli_output (vm, " fd %d", vif->fd);
321  vlib_cli_output (vm, " tap-fd %d", vif->tap_fd);
322  vlib_cli_output (vm, " gso-enabled %d", vif->gso_enabled);
323  }
324  vlib_cli_output (vm, " Mac Address: %U", format_ethernet_address,
325  vif->mac_addr);
326  vlib_cli_output (vm, " Device instance: %u", vif->dev_instance);
327  vlib_cli_output (vm, " flags 0x%x", vif->flags);
328  flag_entry = (struct feat_struct *) &flags_array;
329  while (flag_entry->str)
330  {
331  if (vif->flags & (1ULL << flag_entry->bit))
332  vlib_cli_output (vm, " %s (%d)", flag_entry->str,
333  flag_entry->bit);
334  flag_entry++;
335  }
336  if (type == VIRTIO_IF_TYPE_PCI)
337  {
338  device_status (vm, vif);
339  }
340  vlib_cli_output (vm, " features 0x%lx", vif->features);
341  feat_entry = (struct feat_struct *) &feat_array;
342  while (feat_entry->str)
343  {
344  if (vif->features & (1ULL << feat_entry->bit))
345  vlib_cli_output (vm, " %s (%d)", feat_entry->str,
346  feat_entry->bit);
347  feat_entry++;
348  }
349  vlib_cli_output (vm, " remote-features 0x%lx", vif->remote_features);
350  feat_entry = (struct feat_struct *) &feat_array;
351  while (feat_entry->str)
352  {
353  if (vif->remote_features & (1ULL << feat_entry->bit))
354  vlib_cli_output (vm, " %s (%d)", feat_entry->str,
355  feat_entry->bit);
356  feat_entry++;
357  }
358  vlib_cli_output (vm, " Number of RX Virtqueue %u", vif->num_rxqs);
359  vlib_cli_output (vm, " Number of TX Virtqueue %u", vif->num_txqs);
360  if (vif->cxq_vring != NULL
361  && vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
362  vlib_cli_output (vm, " Number of CTRL Virtqueue 1");
363  vec_foreach_index (i, vif->rxq_vrings)
364  {
365  vring = vec_elt_at_index (vif->rxq_vrings, i);
366  vlib_cli_output (vm, " Virtqueue (RX) %d", vring->queue_id);
367  vlib_cli_output (vm,
368  " qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d",
369  vring->size, vring->last_used_idx, vring->desc_next,
370  vring->desc_in_use);
371  vlib_cli_output (vm,
372  " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d",
373  vring->avail->flags, vring->avail->idx,
374  vring->used->flags, vring->used->idx);
375  if (type == VIRTIO_IF_TYPE_TAP)
376  {
377  vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd,
378  vring->call_fd);
379  }
380  if (show_descr)
381  {
382  vlib_cli_output (vm, "\n descriptor table:\n");
383  vlib_cli_output (vm,
384  " id addr len flags next user_addr\n");
385  vlib_cli_output (vm,
386  " ===== ================== ===== ====== ===== ==================\n");
387  for (j = 0; j < vring->size; j++)
388  {
389  struct vring_desc *desc = &vring->desc[j];
390  vlib_cli_output (vm,
391  " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
392  j, desc->addr,
393  desc->len,
394  desc->flags, desc->next, desc->addr);
395  }
396  }
397  }
398  vec_foreach_index (i, vif->txq_vrings)
399  {
400  vring = vec_elt_at_index (vif->txq_vrings, i);
401  vlib_cli_output (vm, " Virtqueue (TX) %d", vring->queue_id);
402  vlib_cli_output (vm,
403  " qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d",
404  vring->size, vring->last_used_idx, vring->desc_next,
405  vring->desc_in_use);
406  vlib_cli_output (vm,
407  " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d",
408  vring->avail->flags, vring->avail->idx,
409  vring->used->flags, vring->used->idx);
410  if (type == VIRTIO_IF_TYPE_TAP)
411  {
412  vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd,
413  vring->call_fd);
414  }
415  if (show_descr)
416  {
417  vlib_cli_output (vm, "\n descriptor table:\n");
418  vlib_cli_output (vm,
419  " id addr len flags next user_addr\n");
420  vlib_cli_output (vm,
421  " ===== ================== ===== ====== ===== ==================\n");
422  for (j = 0; j < vring->size; j++)
423  {
424  struct vring_desc *desc = &vring->desc[j];
425  vlib_cli_output (vm,
426  " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
427  j, desc->addr,
428  desc->len,
429  desc->flags, desc->next, desc->addr);
430  }
431  }
432  }
433  if (vif->cxq_vring != NULL
434  && vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ))
435  {
436  vring = vif->cxq_vring;
437  vlib_cli_output (vm, " Virtqueue (CTRL) %d", vring->queue_id);
438  vlib_cli_output (vm,
439  " qsz %d, last_used_idx %d, desc_next %d, desc_in_use %d",
440  vring->size, vring->last_used_idx,
441  vring->desc_next, vring->desc_in_use);
442  vlib_cli_output (vm,
443  " avail.flags 0x%x avail.idx %d used.flags 0x%x used.idx %d",
444  vring->avail->flags, vring->avail->idx,
445  vring->used->flags, vring->used->idx);
446  if (type == VIRTIO_IF_TYPE_TAP)
447  {
448  vlib_cli_output (vm, " kickfd %d, callfd %d", vring->kick_fd,
449  vring->call_fd);
450  }
451  if (show_descr)
452  {
453  vlib_cli_output (vm, "\n descriptor table:\n");
454  vlib_cli_output (vm,
455  " id addr len flags next user_addr\n");
456  vlib_cli_output (vm,
457  " ===== ================== ===== ====== ===== ==================\n");
458  for (j = 0; j < vring->size; j++)
459  {
460  struct vring_desc *desc = &vring->desc[j];
461  vlib_cli_output (vm,
462  " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
463  j, desc->addr,
464  desc->len,
465  desc->flags, desc->next, desc->addr);
466  }
467  }
468  }
469 
470  }
471 
472 }
473 
474 /*
475  * fd.io coding-style-patch-verification: ON
476  *
477  * Local Variables:
478  * eval: (c-set-style "gnu")
479  * End:
480  */
struct vring_used * used
Definition: virtio.h:105
vmrglw vmrglh hi
#define vec_foreach_index(var, v)
Iterate over vector indices.
format_function_t format_vnet_hw_if_index_name
void virtio_set_net_hdr_size(virtio_if_t *vif)
Definition: virtio.c:257
virtio_if_t * interfaces
Definition: virtio.h:191
#define CLIB_UNUSED(x)
Definition: clib.h:82
void virtio_show(vlib_main_t *vm, u32 *hw_if_indices, u8 show_descr, u32 type)
Definition: virtio.c:267
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
unsigned long u64
Definition: types.h:89
int gso_enabled
Definition: virtio.h:181
u32 dev_instance
Definition: virtio.h:139
#define NULL
Definition: clib.h:58
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
#define foreach_virtio_net_features
Definition: virtio.h:26
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 file_descriptor
Definition: file.h:54
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
int i
void virtio_vring_set_numa_node(vlib_main_t *vm, virtio_if_t *vif, u32 idx)
Definition: virtio.c:241
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
vlib_main_t ** vlib_mains
Definition: buffer.c:321
unsigned char u8
Definition: types.h:56
clib_error_t * virtio_vring_free_tx(vlib_main_t *vm, virtio_if_t *vif, u32 idx)
Definition: virtio.c:218
clib_file_function_t * read_function
Definition: file.h:67
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:64
struct vring_avail * avail
Definition: virtio.h:106
u64 features
Definition: virtio.h:163
u32 hw_if_index
Definition: virtio.h:140
u8 * format_ethernet_address(u8 *s, va_list *args)
Definition: format.c:44
#define TX_QUEUE_ACCESS(X)
Definition: virtio.h:80
void device_status(vlib_main_t *vm, virtio_if_t *vif)
Definition: pci.c:357
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
void virtio_free_rx_buffers(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:155
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
clib_spinlock_t lockp
Definition: virtio.h:107
#define clib_error_return(e, args...)
Definition: error.h:99
clib_file_main_t file_main
Definition: main.c:63
unsigned int u32
Definition: types.h:88
u16 queue_id
Definition: virtio.h:114
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
u16 num_txqs
Definition: virtio.h:170
u8 * description
Definition: file.h:70
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
uword size
static_always_inline uword vnet_get_device_input_thread_index(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:127
int tap_fd
Definition: virtio.h:158
unsigned short u16
Definition: types.h:57
pci_addr_t pci_addr
Definition: virtio.h:148
vnet_main_t vnet_main
Definition: misc.c:43
u8 buffer_pool_index
Definition: virtio.h:112
u16 desc_next
Definition: virtio.h:109
u16 virtio_net_hdr_sz
Definition: virtio.h:143
u16 num_rxqs
Definition: virtio.h:169
virtio_vring_t * rxq_vrings
Definition: virtio.h:161
format_function_t format_virtio_device_name
Definition: virtio.h:213
u16 last_used_idx
Definition: virtio.h:118
vlib_main_t * vm
Definition: buffer.c:312
#define VIRTIO_RING_FLAG_MASK_INT
Definition: virtio.h:99
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
u32 call_file_index
Definition: virtio.h:116
u8 * net_ns
Definition: virtio.h:174
u8 mac_addr[6]
Definition: virtio.h:172
u32 flags
Definition: virtio.h:137
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:241
virtio_if_type_t type
Definition: virtio.h:144
void virtio_free_used_desc(vlib_main_t *vm, virtio_vring_t *vring)
Definition: virtio.c:192
#define ASSERT(truth)
u64 remote_features
Definition: virtio.h:163
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
Definition: file.h:119
#define VIRTIO_FEATURE(X)
Definition: virtio.h:76
clib_error_t * virtio_vring_init(vlib_main_t *vm, virtio_if_t *vif, u16 idx, u16 sz)
Definition: virtio.c:65
static void clib_mem_free(void *p)
Definition: mem.h:205
static uword pointer_to_uword(const void *p)
Definition: types.h:131
virtio_main_t virtio_main
Definition: virtio.c:37
static uword is_pow2(uword x)
Definition: clib.h:235
#define RX_QUEUE_ACCESS(X)
Definition: virtio.h:81
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
int fd
Definition: virtio.h:153
u32 * buffers
Definition: virtio.h:117
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:140
virtio_vring_t * cxq_vring
Definition: virtio.h:183
clib_error_t * virtio_vring_free_rx(vlib_main_t *vm, virtio_if_t *vif, u32 idx)
Definition: virtio.c:170
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
uword private_data
Definition: file.h:64
struct vring_desc * desc
Definition: virtio.h:104
Definition: file.h:51
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static u8 vlib_buffer_pool_get_default_for_numa(vlib_main_t *vm, u32 numa_node)
Definition: buffer_funcs.h:163
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:762
format_function_t format_vlib_pci_addr
Definition: pci.h:324
u16 desc_in_use
Definition: virtio.h:108
virtio_vring_t * txq_vrings
Definition: virtio.h:162
static clib_error_t * call_read_ready(clib_file_t *uf)
Definition: virtio.c:47
u8 * host_if_name
Definition: virtio.h:173