FD.io VPP  v18.11-rc0-18-g2a3fb1a
Vector Packet Processing
vhost_user.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * vhost.c - vhost-user
4  *
5  * Copyright (c) 2014-2018 Cisco and/or its affiliates.
6  * Licensed under the Apache License, Version 2.0 (the "License");
7  * you may not use this file except in compliance with the License.
8  * You may obtain a copy of the License at:
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  *------------------------------------------------------------------
18  */
19 
20 #include <fcntl.h> /* for open */
21 #include <sys/ioctl.h>
22 #include <sys/socket.h>
23 #include <sys/un.h>
24 #include <sys/stat.h>
25 #include <sys/types.h>
26 #include <sys/uio.h> /* for iovec */
27 #include <netinet/in.h>
28 #include <sys/vfs.h>
29 
30 #include <linux/if_arp.h>
31 #include <linux/if_tun.h>
32 
33 #include <vlib/vlib.h>
34 #include <vlib/unix/unix.h>
35 
36 #include <vnet/ip/ip.h>
37 
38 #include <vnet/ethernet/ethernet.h>
39 #include <vnet/devices/devices.h>
40 #include <vnet/feature/feature.h>
41 
44 
45 /**
46  * @file
47  * @brief vHost User Device Driver.
48  *
49  * This file contains the source code for vHost User interface.
50  */
51 
52 
54 
55 /* *INDENT-OFF* */
56 vhost_user_main_t vhost_user_main = {
57  .mtu_bytes = 1518,
58 };
59 
60 VNET_HW_INTERFACE_CLASS (vhost_interface_class, static) = {
61  .name = "vhost-user",
62 };
63 /* *INDENT-ON* */
64 
65 static long
67 {
68  struct statfs s;
69  fstatfs (fd, &s);
70  return s.f_bsize;
71 }
72 
73 static void
75 {
76  int i, r;
77  for (i = 0; i < vui->nregions; i++)
78  {
79  if (vui->region_mmap_addr[i] != MAP_FAILED)
80  {
81 
82  long page_sz = get_huge_page_size (vui->region_mmap_fd[i]);
83 
84  ssize_t map_sz = (vui->regions[i].memory_size +
85  vui->regions[i].mmap_offset +
86  page_sz - 1) & ~(page_sz - 1);
87 
88  r =
89  munmap (vui->region_mmap_addr[i] - vui->regions[i].mmap_offset,
90  map_sz);
91 
92  DBG_SOCK
93  ("unmap memory region %d addr 0x%lx len 0x%lx page_sz 0x%x", i,
94  vui->region_mmap_addr[i], map_sz, page_sz);
95 
96  vui->region_mmap_addr[i] = MAP_FAILED;
97 
98  if (r == -1)
99  {
100  clib_warning ("failed to unmap memory region (errno %d)",
101  errno);
102  }
103  close (vui->region_mmap_fd[i]);
104  }
105  }
106  vui->nregions = 0;
107 }
108 
109 static void
111 {
112  //Let's try to assign one queue to each thread
113  u32 qid = 0;
114  u32 thread_index = 0;
115  vui->use_tx_spinlock = 0;
116  while (1)
117  {
118  for (qid = 0; qid < VHOST_VRING_MAX_N / 2; qid++)
119  {
120  vhost_user_vring_t *rxvq = &vui->vrings[VHOST_VRING_IDX_RX (qid)];
121  if (!rxvq->started || !rxvq->enabled)
122  continue;
123 
124  vui->per_cpu_tx_qid[thread_index] = qid;
125  thread_index++;
126  if (thread_index == vlib_get_thread_main ()->n_vlib_mains)
127  return;
128  }
129  //We need to loop, meaning the spinlock has to be used
130  vui->use_tx_spinlock = 1;
131  if (thread_index == 0)
132  {
133  //Could not find a single valid one
134  for (thread_index = 0;
135  thread_index < vlib_get_thread_main ()->n_vlib_mains;
136  thread_index++)
137  {
138  vui->per_cpu_tx_qid[thread_index] = 0;
139  }
140  return;
141  }
142  }
143 }
144 
145 /**
146  * @brief Unassign existing interface/queue to thread mappings and re-assign
147  * new interface/queue to thread mappings
148  */
149 static void
151 {
153  vhost_user_intf_t *vui;
154  vhost_user_vring_t *txvq;
155  vnet_main_t *vnm = vnet_get_main ();
156  u32 qid;
157  int rv;
158  u16 *queue;
159 
160  // Scrap all existing mappings for all interfaces/queues
161  /* *INDENT-OFF* */
162  pool_foreach (vui, vum->vhost_user_interfaces, {
163  vec_foreach (queue, vui->rx_queues)
164  {
165  rv = vnet_hw_interface_unassign_rx_thread (vnm, vui->hw_if_index,
166  *queue);
167  if (rv)
168  clib_warning ("Warning: unable to unassign interface %d, "
169  "queue %d: rc=%d", vui->hw_if_index, *queue, rv);
170  }
172  });
173  /* *INDENT-ON* */
174 
175  // Create the rx_queues for all interfaces
176  /* *INDENT-OFF* */
177  pool_foreach (vui, vum->vhost_user_interfaces, {
178  for (qid = 0; qid < VHOST_VRING_MAX_N / 2; qid++)
179  {
180  txvq = &vui->vrings[VHOST_VRING_IDX_TX (qid)];
181  if (txvq->started)
182  {
183  if (txvq->mode == VNET_HW_INTERFACE_RX_MODE_UNKNOWN)
184  /* Set polling as the default */
185  txvq->mode = VNET_HW_INTERFACE_RX_MODE_POLLING;
186  vec_add1 (vui->rx_queues, qid);
187  }
188  }
189  });
190  /* *INDENT-ON* */
191 
192  // Assign new mappings for all interfaces/queues
193  /* *INDENT-OFF* */
194  pool_foreach (vui, vum->vhost_user_interfaces, {
195  vnet_hw_interface_set_input_node (vnm, vui->hw_if_index,
196  vhost_user_input_node.index);
197  vec_foreach (queue, vui->rx_queues)
198  {
199  vnet_hw_interface_assign_rx_thread (vnm, vui->hw_if_index, *queue,
200  ~0);
201  txvq = &vui->vrings[VHOST_VRING_IDX_TX (*queue)];
202  rv = vnet_hw_interface_set_rx_mode (vnm, vui->hw_if_index, *queue,
203  txvq->mode);
204  if (rv)
205  clib_warning ("Warning: unable to set rx mode for interface %d, "
206  "queue %d: rc=%d", vui->hw_if_index, *queue, rv);
207  }
208  });
209  /* *INDENT-ON* */
210 }
211 
212 /** @brief Returns whether at least one TX and one RX vring are enabled */
215 {
216  int i, found[2] = { }; //RX + TX
217 
218  for (i = 0; i < VHOST_VRING_MAX_N; i++)
219  if (vui->vrings[i].started && vui->vrings[i].enabled)
220  found[i & 1] = 1;
221 
222  return found[0] && found[1];
223 }
224 
225 static void
227 {
228  /* if we have pointers to descriptor table, go up */
229  int is_up = vhost_user_intf_ready (vui);
230  if (is_up != vui->is_up)
231  {
232  DBG_SOCK ("interface %d %s", vui->sw_if_index,
233  is_up ? "ready" : "down");
236  0);
237  vui->is_up = is_up;
238  }
241 }
242 
243 static void
245 {
246  u32 qid;
247  vnet_main_t *vnm = vnet_get_main ();
248 
249  qid = ifq & 0xff;
250  if ((qid & 1) == 0)
251  /* Only care about the odd number, or TX, virtqueue */
252  return;
253 
254  if (vhost_user_intf_ready (vui))
255  // qid >> 1 is to convert virtqueue number to vring queue index
257 }
258 
259 static clib_error_t *
261 {
262  __attribute__ ((unused)) int n;
263  u8 buff[8];
264 
265  n = read (uf->file_descriptor, ((char *) &buff), 8);
266 
267  return 0;
268 }
269 
270 static clib_error_t *
272 {
273  __attribute__ ((unused)) int n;
274  u8 buff[8];
275  vhost_user_intf_t *vui =
276  pool_elt_at_index (vhost_user_main.vhost_user_interfaces,
277  uf->private_data >> 8);
278  u32 qid = uf->private_data & 0xff;
279 
280  n = read (uf->file_descriptor, ((char *) &buff), 8);
281  DBG_SOCK ("if %d KICK queue %d", uf->private_data >> 8, qid);
282  if (!vui->vrings[qid].started ||
283  (vhost_user_intf_ready (vui) != vui->is_up))
284  {
286  vui->vrings[qid].started = 1;
289  }
290 
292  return 0;
293 }
294 
297 {
298  vhost_user_vring_t *vring = &vui->vrings[qid];
299  memset (vring, 0, sizeof (*vring));
300  vring->kickfd_idx = ~0;
301  vring->callfd_idx = ~0;
302  vring->errfd = -1;
303 
304  /*
305  * We have a bug with some qemu 2.5, and this may be a fix.
306  * Feel like interpretation holy text, but this is from vhost-user.txt.
307  * "
308  * One queue pair is enabled initially. More queues are enabled
309  * dynamically, by sending message VHOST_USER_SET_VRING_ENABLE.
310  * "
311  * Don't know who's right, but this is what DPDK does.
312  */
313  if (qid == 0 || qid == 1)
314  vring->enabled = 1;
315 }
316 
319 {
320  vhost_user_vring_t *vring = &vui->vrings[qid];
321  if (vring->kickfd_idx != ~0)
322  {
324  vring->kickfd_idx);
325  clib_file_del (&file_main, uf);
326  vring->kickfd_idx = ~0;
327  }
328  if (vring->callfd_idx != ~0)
329  {
331  vring->callfd_idx);
332  clib_file_del (&file_main, uf);
333  vring->callfd_idx = ~0;
334  }
335  if (vring->errfd != -1)
336  {
337  close (vring->errfd);
338  vring->errfd = -1;
339  }
340  vhost_user_vring_init (vui, qid);
341 }
342 
345 {
346  vnet_main_t *vnm = vnet_get_main ();
347  int q;
348 
350 
351  if (vui->clib_file_index != ~0)
352  {
354  vui->clib_file_index = ~0;
355  }
356 
357  vui->is_up = 0;
358 
359  for (q = 0; q < VHOST_VRING_MAX_N; q++)
360  vhost_user_vring_close (vui, q);
361 
362  unmap_all_mem_regions (vui);
363  DBG_SOCK ("interface ifindex %d disconnected", vui->sw_if_index);
364 }
365 
366 static clib_error_t *
368 {
369  int n, i;
370  int fd, number_of_fds = 0;
371  int fds[VHOST_MEMORY_MAX_NREGIONS];
372  vhost_user_msg_t msg;
373  struct msghdr mh;
374  struct iovec iov[1];
376  vhost_user_intf_t *vui;
377  struct cmsghdr *cmsg;
378  u8 q;
379  clib_file_t template = { 0 };
380  vnet_main_t *vnm = vnet_get_main ();
381 
382  vui = pool_elt_at_index (vum->vhost_user_interfaces, uf->private_data);
383 
384  char control[CMSG_SPACE (VHOST_MEMORY_MAX_NREGIONS * sizeof (int))];
385 
386  memset (&mh, 0, sizeof (mh));
387  memset (control, 0, sizeof (control));
388 
389  for (i = 0; i < VHOST_MEMORY_MAX_NREGIONS; i++)
390  fds[i] = -1;
391 
392  /* set the payload */
393  iov[0].iov_base = (void *) &msg;
394  iov[0].iov_len = VHOST_USER_MSG_HDR_SZ;
395 
396  mh.msg_iov = iov;
397  mh.msg_iovlen = 1;
398  mh.msg_control = control;
399  mh.msg_controllen = sizeof (control);
400 
401  n = recvmsg (uf->file_descriptor, &mh, 0);
402 
403  /* Stop workers to avoid end of the world */
405 
406  if (n != VHOST_USER_MSG_HDR_SZ)
407  {
408  if (n == -1)
409  {
410  DBG_SOCK ("recvmsg returned error %d %s", errno, strerror (errno));
411  }
412  else
413  {
414  DBG_SOCK ("n (%d) != VHOST_USER_MSG_HDR_SZ (%d)",
416  }
417  goto close_socket;
418  }
419 
420  if (mh.msg_flags & MSG_CTRUNC)
421  {
422  DBG_SOCK ("MSG_CTRUNC is set");
423  goto close_socket;
424  }
425 
426  cmsg = CMSG_FIRSTHDR (&mh);
427 
428  if (cmsg && (cmsg->cmsg_len > 0) && (cmsg->cmsg_level == SOL_SOCKET) &&
429  (cmsg->cmsg_type == SCM_RIGHTS) &&
430  (cmsg->cmsg_len - CMSG_LEN (0) <=
431  VHOST_MEMORY_MAX_NREGIONS * sizeof (int)))
432  {
433  number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) / sizeof (int);
434  clib_memcpy (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int));
435  }
436 
437  /* version 1, no reply bit set */
438  if ((msg.flags & 7) != 1)
439  {
440  DBG_SOCK ("malformed message received. closing socket");
441  goto close_socket;
442  }
443 
444  {
445  int rv;
446  rv =
447  read (uf->file_descriptor, ((char *) &msg) + VHOST_USER_MSG_HDR_SZ,
448  msg.size);
449  if (rv < 0)
450  {
451  DBG_SOCK ("read failed %s", strerror (errno));
452  goto close_socket;
453  }
454  else if (rv != msg.size)
455  {
456  DBG_SOCK ("message too short (read %dB should be %dB)", rv, msg.size);
457  goto close_socket;
458  }
459  }
460 
461  switch (msg.request)
462  {
464  msg.flags |= 4;
465  msg.u64 = (1ULL << FEAT_VIRTIO_NET_F_MRG_RXBUF) |
466  (1ULL << FEAT_VIRTIO_NET_F_CTRL_VQ) |
467  (1ULL << FEAT_VIRTIO_F_ANY_LAYOUT) |
468  (1ULL << FEAT_VIRTIO_F_INDIRECT_DESC) |
469  (1ULL << FEAT_VHOST_F_LOG_ALL) |
470  (1ULL << FEAT_VIRTIO_NET_F_GUEST_ANNOUNCE) |
471  (1ULL << FEAT_VIRTIO_NET_F_MQ) |
472  (1ULL << FEAT_VHOST_USER_F_PROTOCOL_FEATURES) |
473  (1ULL << FEAT_VIRTIO_F_VERSION_1);
474  msg.u64 &= vui->feature_mask;
475  msg.size = sizeof (msg.u64);
476  DBG_SOCK ("if %d msg VHOST_USER_GET_FEATURES - reply 0x%016llx",
477  vui->hw_if_index, msg.u64);
478  break;
479 
481  DBG_SOCK ("if %d msg VHOST_USER_SET_FEATURES features 0x%016llx",
482  vui->hw_if_index, msg.u64);
483 
484  vui->features = msg.u64;
485 
486  if (vui->features &
487  ((1 << FEAT_VIRTIO_NET_F_MRG_RXBUF) |
488  (1ULL << FEAT_VIRTIO_F_VERSION_1)))
489  vui->virtio_net_hdr_sz = 12;
490  else
491  vui->virtio_net_hdr_sz = 10;
492 
493  vui->is_any_layout =
494  (vui->features & (1 << FEAT_VIRTIO_F_ANY_LAYOUT)) ? 1 : 0;
495 
498  vui->is_up = 0;
499 
500  /*for (q = 0; q < VHOST_VRING_MAX_N; q++)
501  vhost_user_vring_close(&vui->vrings[q]); */
502 
503  break;
504 
506  DBG_SOCK ("if %d msg VHOST_USER_SET_MEM_TABLE nregions %d",
507  vui->hw_if_index, msg.memory.nregions);
508 
509  if ((msg.memory.nregions < 1) ||
510  (msg.memory.nregions > VHOST_MEMORY_MAX_NREGIONS))
511  {
512 
513  DBG_SOCK ("number of mem regions must be between 1 and %i",
514  VHOST_MEMORY_MAX_NREGIONS);
515 
516  goto close_socket;
517  }
518 
519  if (msg.memory.nregions != number_of_fds)
520  {
521  DBG_SOCK ("each memory region must have FD");
522  goto close_socket;
523  }
524  unmap_all_mem_regions (vui);
525  for (i = 0; i < msg.memory.nregions; i++)
526  {
527  clib_memcpy (&(vui->regions[i]), &msg.memory.regions[i],
528  sizeof (vhost_user_memory_region_t));
529 
530  long page_sz = get_huge_page_size (fds[i]);
531 
532  /* align size to page */
533  ssize_t map_sz = (vui->regions[i].memory_size +
534  vui->regions[i].mmap_offset +
535  page_sz - 1) & ~(page_sz - 1);
536 
537  vui->region_mmap_addr[i] = mmap (0, map_sz, PROT_READ | PROT_WRITE,
538  MAP_SHARED, fds[i], 0);
539  vui->region_guest_addr_lo[i] = vui->regions[i].guest_phys_addr;
540  vui->region_guest_addr_hi[i] = vui->regions[i].guest_phys_addr +
541  vui->regions[i].memory_size;
542 
543  DBG_SOCK
544  ("map memory region %d addr 0 len 0x%lx fd %d mapped 0x%lx "
545  "page_sz 0x%x", i, map_sz, fds[i], vui->region_mmap_addr[i],
546  page_sz);
547 
548  if (vui->region_mmap_addr[i] == MAP_FAILED)
549  {
550  clib_warning ("failed to map memory. errno is %d", errno);
551  goto close_socket;
552  }
553  vui->region_mmap_addr[i] += vui->regions[i].mmap_offset;
554  vui->region_mmap_fd[i] = fds[i];
555 
556  vui->nregions++;
557  }
558  break;
559 
561  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_NUM idx %d num %d",
562  vui->hw_if_index, msg.state.index, msg.state.num);
563 
564  if ((msg.state.num > 32768) || /* maximum ring size is 32768 */
565  (msg.state.num == 0) || /* it cannot be zero */
566  ((msg.state.num - 1) & msg.state.num)) /* must be power of 2 */
567  goto close_socket;
568  vui->vrings[msg.state.index].qsz_mask = msg.state.num - 1;
569  break;
570 
572  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ADDR idx %d",
573  vui->hw_if_index, msg.state.index);
574 
575  if (msg.state.index >= VHOST_VRING_MAX_N)
576  {
577  DBG_SOCK ("invalid vring index VHOST_USER_SET_VRING_ADDR:"
578  " %d >= %d", msg.state.index, VHOST_VRING_MAX_N);
579  goto close_socket;
580  }
581 
582  if (msg.size < sizeof (msg.addr))
583  {
584  DBG_SOCK ("vhost message is too short (%d < %d)",
585  msg.size, sizeof (msg.addr));
586  goto close_socket;
587  }
588 
589  vui->vrings[msg.state.index].desc = (vring_desc_t *)
590  map_user_mem (vui, msg.addr.desc_user_addr);
591  vui->vrings[msg.state.index].used = (vring_used_t *)
592  map_user_mem (vui, msg.addr.used_user_addr);
593  vui->vrings[msg.state.index].avail = (vring_avail_t *)
594  map_user_mem (vui, msg.addr.avail_user_addr);
595 
596  if ((vui->vrings[msg.state.index].desc == NULL) ||
597  (vui->vrings[msg.state.index].used == NULL) ||
598  (vui->vrings[msg.state.index].avail == NULL))
599  {
600  DBG_SOCK ("failed to map user memory for hw_if_index %d",
601  vui->hw_if_index);
602  goto close_socket;
603  }
604 
605  vui->vrings[msg.state.index].log_guest_addr = msg.addr.log_guest_addr;
606  vui->vrings[msg.state.index].log_used =
607  (msg.addr.flags & (1 << VHOST_VRING_F_LOG)) ? 1 : 0;
608 
609  /* Spec says: If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated,
610  the ring is initialized in an enabled state. */
611  if (!(vui->features & (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES)))
612  {
613  vui->vrings[msg.state.index].enabled = 1;
614  }
615 
616  vui->vrings[msg.state.index].last_used_idx =
617  vui->vrings[msg.state.index].last_avail_idx =
618  vui->vrings[msg.state.index].used->idx;
619 
620  /* tell driver that we don't want interrupts */
621  vui->vrings[msg.state.index].used->flags = VRING_USED_F_NO_NOTIFY;
622  break;
623 
625  DBG_SOCK ("if %d msg VHOST_USER_SET_OWNER", vui->hw_if_index);
626  break;
627 
629  DBG_SOCK ("if %d msg VHOST_USER_RESET_OWNER", vui->hw_if_index);
630  break;
631 
633  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_CALL %d",
634  vui->hw_if_index, msg.u64);
635 
636  q = (u8) (msg.u64 & 0xFF);
637 
638  /* if there is old fd, delete and close it */
639  if (vui->vrings[q].callfd_idx != ~0)
640  {
642  vui->vrings[q].callfd_idx);
643  clib_file_del (&file_main, uf);
644  vui->vrings[q].callfd_idx = ~0;
645  }
646 
647  if (!(msg.u64 & VHOST_USER_VRING_NOFD_MASK))
648  {
649  if (number_of_fds != 1)
650  {
651  DBG_SOCK ("More than one fd received !");
652  goto close_socket;
653  }
654 
655  template.read_function = vhost_user_callfd_read_ready;
656  template.file_descriptor = fds[0];
657  template.private_data =
658  ((vui - vhost_user_main.vhost_user_interfaces) << 8) + q;
659  vui->vrings[q].callfd_idx = clib_file_add (&file_main, &template);
660  }
661  else
662  vui->vrings[q].callfd_idx = ~0;
663  break;
664 
666  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_KICK %d",
667  vui->hw_if_index, msg.u64);
668 
669  q = (u8) (msg.u64 & 0xFF);
670 
671  if (vui->vrings[q].kickfd_idx != ~0)
672  {
674  vui->vrings[q].kickfd_idx);
675  clib_file_del (&file_main, uf);
676  vui->vrings[q].kickfd_idx = ~0;
677  }
678 
679  if (!(msg.u64 & VHOST_USER_VRING_NOFD_MASK))
680  {
681  if (number_of_fds != 1)
682  {
683  DBG_SOCK ("More than one fd received !");
684  goto close_socket;
685  }
686 
687  template.read_function = vhost_user_kickfd_read_ready;
688  template.file_descriptor = fds[0];
689  template.private_data =
690  (((uword) (vui - vhost_user_main.vhost_user_interfaces)) << 8) +
691  q;
692  vui->vrings[q].kickfd_idx = clib_file_add (&file_main, &template);
693  }
694  else
695  {
696  //When no kickfd is set, the queue is initialized as started
697  vui->vrings[q].kickfd_idx = ~0;
698  vui->vrings[q].started = 1;
699  }
700 
701  break;
702 
704  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ERR %d",
705  vui->hw_if_index, msg.u64);
706 
707  q = (u8) (msg.u64 & 0xFF);
708 
709  if (vui->vrings[q].errfd != -1)
710  close (vui->vrings[q].errfd);
711 
712  if (!(msg.u64 & VHOST_USER_VRING_NOFD_MASK))
713  {
714  if (number_of_fds != 1)
715  goto close_socket;
716 
717  vui->vrings[q].errfd = fds[0];
718  }
719  else
720  vui->vrings[q].errfd = -1;
721 
722  break;
723 
725  DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_BASE idx %d num %d",
726  vui->hw_if_index, msg.state.index, msg.state.num);
727 
728  vui->vrings[msg.state.index].last_avail_idx = msg.state.num;
729  break;
730 
732  if (msg.state.index >= VHOST_VRING_MAX_N)
733  {
734  DBG_SOCK ("invalid vring index VHOST_USER_GET_VRING_BASE:"
735  " %d >= %d", msg.state.index, VHOST_VRING_MAX_N);
736  goto close_socket;
737  }
738 
739  /*
740  * Copy last_avail_idx from the vring before closing it because
741  * closing the vring also initializes the vring last_avail_idx
742  */
743  msg.state.num = vui->vrings[msg.state.index].last_avail_idx;
744  msg.flags |= 4;
745  msg.size = sizeof (msg.state);
746 
747  /* Spec says: Client must [...] stop ring upon receiving VHOST_USER_GET_VRING_BASE. */
748  vhost_user_vring_close (vui, msg.state.index);
749  DBG_SOCK ("if %d msg VHOST_USER_GET_VRING_BASE idx %d num %d",
750  vui->hw_if_index, msg.state.index, msg.state.num);
751  break;
752 
753  case VHOST_USER_NONE:
754  DBG_SOCK ("if %d msg VHOST_USER_NONE", vui->hw_if_index);
755 
756  break;
757 
759  {
760  DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_BASE", vui->hw_if_index);
761 
762  if (msg.size != sizeof (msg.log))
763  {
764  DBG_SOCK
765  ("invalid msg size for VHOST_USER_SET_LOG_BASE: %d instead of %d",
766  msg.size, sizeof (msg.log));
767  goto close_socket;
768  }
769 
770  if (!
772  {
773  DBG_SOCK
774  ("VHOST_USER_PROTOCOL_F_LOG_SHMFD not set but VHOST_USER_SET_LOG_BASE received");
775  goto close_socket;
776  }
777 
778  fd = fds[0];
779  /* align size to page */
780  long page_sz = get_huge_page_size (fd);
781  ssize_t map_sz =
782  (msg.log.size + msg.log.offset + page_sz - 1) & ~(page_sz - 1);
783 
784  vui->log_base_addr = mmap (0, map_sz, PROT_READ | PROT_WRITE,
785  MAP_SHARED, fd, 0);
786 
787  DBG_SOCK
788  ("map log region addr 0 len 0x%lx off 0x%lx fd %d mapped 0x%lx",
789  map_sz, msg.log.offset, fd, vui->log_base_addr);
790 
791  if (vui->log_base_addr == MAP_FAILED)
792  {
793  clib_warning ("failed to map memory. errno is %d", errno);
794  goto close_socket;
795  }
796 
797  vui->log_base_addr += msg.log.offset;
798  vui->log_size = msg.log.size;
799 
800  msg.flags |= 4;
801  msg.size = sizeof (msg.u64);
802 
803  break;
804  }
805 
807  DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_FD", vui->hw_if_index);
808 
809  break;
810 
812  msg.flags |= 4;
813  msg.u64 = (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD) |
815  msg.size = sizeof (msg.u64);
816  DBG_SOCK
817  ("if %d msg VHOST_USER_GET_PROTOCOL_FEATURES - reply 0x%016llx",
818  vui->hw_if_index, msg.u64);
819  break;
820 
822  DBG_SOCK
823  ("if %d msg VHOST_USER_SET_PROTOCOL_FEATURES features 0x%016llx",
824  vui->hw_if_index, msg.u64);
825 
826  vui->protocol_features = msg.u64;
827 
828  break;
829 
831  msg.flags |= 4;
832  msg.u64 = VHOST_VRING_MAX_N;
833  msg.size = sizeof (msg.u64);
834  DBG_SOCK ("if %d msg VHOST_USER_GET_QUEUE_NUM - reply %d",
835  vui->hw_if_index, msg.u64);
836  break;
837 
839  DBG_SOCK ("if %d VHOST_USER_SET_VRING_ENABLE: %s queue %d",
840  vui->hw_if_index, msg.state.num ? "enable" : "disable",
841  msg.state.index);
842  if (msg.state.index >= VHOST_VRING_MAX_N)
843  {
844  DBG_SOCK ("invalid vring index VHOST_USER_SET_VRING_ENABLE:"
845  " %d >= %d", msg.state.index, VHOST_VRING_MAX_N);
846  goto close_socket;
847  }
848 
849  vui->vrings[msg.state.index].enabled = msg.state.num;
850  break;
851 
852  default:
853  DBG_SOCK ("unknown vhost-user message %d received. closing socket",
854  msg.request);
855  goto close_socket;
856  }
857 
858  /* if we need to reply */
859  if (msg.flags & 4)
860  {
861  n =
862  send (uf->file_descriptor, &msg, VHOST_USER_MSG_HDR_SZ + msg.size, 0);
863  if (n != (msg.size + VHOST_USER_MSG_HDR_SZ))
864  {
865  DBG_SOCK ("could not send message response");
866  goto close_socket;
867  }
868  }
869 
872  return 0;
873 
874 close_socket:
878  return 0;
879 }
880 
881 static clib_error_t *
883 {
886  vhost_user_intf_t *vui =
888 
889  DBG_SOCK ("socket error on if %d", vui->sw_if_index);
894  return 0;
895 }
896 
897 static clib_error_t *
899 {
900  int client_fd, client_len;
901  struct sockaddr_un client;
902  clib_file_t template = { 0 };
904  vhost_user_intf_t *vui;
905 
907 
908  client_len = sizeof (client);
909  client_fd = accept (uf->file_descriptor,
910  (struct sockaddr *) &client,
911  (socklen_t *) & client_len);
912 
913  if (client_fd < 0)
914  return clib_error_return_unix (0, "accept");
915 
916  if (vui->clib_file_index != ~0)
917  {
918  DBG_SOCK ("Close client socket for vhost interface %d, fd %d",
921  }
922 
923  DBG_SOCK ("New client socket for vhost interface %d, fd %d",
924  vui->sw_if_index, client_fd);
925  template.read_function = vhost_user_socket_read;
926  template.error_function = vhost_user_socket_error;
927  template.file_descriptor = client_fd;
928  template.private_data = vui - vhost_user_main.vhost_user_interfaces;
929  vui->clib_file_index = clib_file_add (&file_main, &template);
930  return 0;
931 }
932 
933 static clib_error_t *
935 {
936  clib_error_t *error;
939 
940  error = vlib_call_init_function (vm, ip4_init);
941  if (error)
942  return error;
943 
944  vum->coalesce_frames = 32;
945  vum->coalesce_time = 1e-3;
946 
947  vec_validate (vum->cpus, tm->n_vlib_mains - 1);
948 
949  vhost_cpu_t *cpu;
950  vec_foreach (cpu, vum->cpus)
951  {
952  /* This is actually not necessary as validate already zeroes it
953  * Just keeping the loop here for later because I am lazy. */
954  cpu->rx_buffers_len = 0;
955  }
956 
957  vum->random = random_default_seed ();
958 
960 
961  return 0;
962 }
963 
965 
966 static uword
969 {
970  vhost_user_intf_t *vui;
971  f64 timeout = 3153600000.0 /* 100 years */ ;
972  uword event_type, *event_data = 0;
974  u16 *queue;
975  f64 now, poll_time_remaining;
976  f64 next_timeout;
977  u8 stop_timer = 0;
978 
979  while (1)
980  {
981  poll_time_remaining =
983  event_type = vlib_process_get_events (vm, &event_data);
984  vec_reset_length (event_data);
985 
986  /*
987  * Use the remaining timeout if it is less than coalesce time to avoid
988  * resetting the existing timer in the middle of expiration
989  */
990  timeout = poll_time_remaining;
991  if (vlib_process_suspend_time_is_zero (timeout) ||
992  (timeout > vum->coalesce_time))
993  timeout = vum->coalesce_time;
994 
995  now = vlib_time_now (vm);
996  switch (event_type)
997  {
999  stop_timer = 1;
1000  break;
1001 
1003  stop_timer = 0;
1004  if (!vlib_process_suspend_time_is_zero (poll_time_remaining))
1005  break;
1006  /* fall through */
1007 
1008  case ~0:
1009  /* *INDENT-OFF* */
1010  pool_foreach (vui, vum->vhost_user_interfaces, {
1011  next_timeout = timeout;
1012  vec_foreach (queue, vui->rx_queues)
1013  {
1014  vhost_user_vring_t *rxvq =
1015  &vui->vrings[VHOST_VRING_IDX_RX (*queue)];
1016  vhost_user_vring_t *txvq =
1017  &vui->vrings[VHOST_VRING_IDX_TX (*queue)];
1018 
1019  if (txvq->n_since_last_int)
1020  {
1021  if (now >= txvq->int_deadline)
1022  vhost_user_send_call (vm, txvq);
1023  else
1024  next_timeout = txvq->int_deadline - now;
1025  }
1026 
1027  if (rxvq->n_since_last_int)
1028  {
1029  if (now >= rxvq->int_deadline)
1030  vhost_user_send_call (vm, rxvq);
1031  else
1032  next_timeout = rxvq->int_deadline - now;
1033  }
1034 
1035  if ((next_timeout < timeout) && (next_timeout > 0.0))
1036  timeout = next_timeout;
1037  }
1038  });
1039  /* *INDENT-ON* */
1040  break;
1041 
1042  default:
1043  clib_warning ("BUG: unhandled event type %d", event_type);
1044  break;
1045  }
1046  /* No less than 1 millisecond */
1047  if (timeout < 1e-3)
1048  timeout = 1e-3;
1049  if (stop_timer)
1050  timeout = 3153600000.0;
1051  }
1052  return 0;
1053 }
1054 
1055 /* *INDENT-OFF* */
1058  .type = VLIB_NODE_TYPE_PROCESS,
1059  .name = "vhost-user-send-interrupt-process",
1060 };
1061 /* *INDENT-ON* */
1062 
1063 static uword
1066 {
1068  vhost_user_intf_t *vui;
1069  struct sockaddr_un sun;
1070  int sockfd;
1071  clib_file_t template = { 0 };
1072  f64 timeout = 3153600000.0 /* 100 years */ ;
1073  uword *event_data = 0;
1074 
1075  sockfd = -1;
1076  sun.sun_family = AF_UNIX;
1078  template.error_function = vhost_user_socket_error;
1079 
1080  while (1)
1081  {
1083  vlib_process_get_events (vm, &event_data);
1084  vec_reset_length (event_data);
1085 
1086  timeout = 3.0;
1087 
1088  /* *INDENT-OFF* */
1089  pool_foreach (vui, vum->vhost_user_interfaces, {
1090 
1091  if (vui->unix_server_index == ~0) { //Nothing to do for server sockets
1092  if (vui->clib_file_index == ~0)
1093  {
1094  if ((sockfd < 0) &&
1095  ((sockfd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0))
1096  {
1097  /*
1098  * 1st time error or new error for this interface,
1099  * spit out the message and record the error
1100  */
1101  if (!vui->sock_errno || (vui->sock_errno != errno))
1102  {
1103  clib_unix_warning
1104  ("Error: Could not open unix socket for %s",
1105  vui->sock_filename);
1106  vui->sock_errno = errno;
1107  }
1108  continue;
1109  }
1110 
1111  /* try to connect */
1112  strncpy (sun.sun_path, (char *) vui->sock_filename,
1113  sizeof (sun.sun_path) - 1);
1114 
1115  /* Avoid hanging VPP if the other end does not accept */
1116  if (fcntl(sockfd, F_SETFL, O_NONBLOCK) < 0)
1117  clib_unix_warning ("fcntl");
1118 
1119  if (connect (sockfd, (struct sockaddr *) &sun,
1120  sizeof (struct sockaddr_un)) == 0)
1121  {
1122  /* Set the socket to blocking as it was before */
1123  if (fcntl(sockfd, F_SETFL, 0) < 0)
1124  clib_unix_warning ("fcntl2");
1125 
1126  vui->sock_errno = 0;
1127  template.file_descriptor = sockfd;
1128  template.private_data =
1129  vui - vhost_user_main.vhost_user_interfaces;
1130  vui->clib_file_index = clib_file_add (&file_main, &template);
1131 
1132  /* This sockfd is considered consumed */
1133  sockfd = -1;
1134  }
1135  else
1136  {
1137  vui->sock_errno = errno;
1138  }
1139  }
1140  else
1141  {
1142  /* check if socket is alive */
1143  int error = 0;
1144  socklen_t len = sizeof (error);
1145  int fd = UNIX_GET_FD(vui->clib_file_index);
1146  int retval =
1147  getsockopt (fd, SOL_SOCKET, SO_ERROR, &error, &len);
1148 
1149  if (retval)
1150  {
1151  DBG_SOCK ("getsockopt returned %d", retval);
1152  vhost_user_if_disconnect (vui);
1153  }
1154  }
1155  }
1156  });
1157  /* *INDENT-ON* */
1158  }
1159  return 0;
1160 }
1161 
1162 /* *INDENT-OFF* */
1164  .function = vhost_user_process,
1165  .type = VLIB_NODE_TYPE_PROCESS,
1166  .name = "vhost-user-process",
1167 };
1168 /* *INDENT-ON* */
1169 
1170 /**
1171  * Disables and reset interface structure.
1172  * It can then be either init again, or removed from used interfaces.
1173  */
1174 static void
1176 {
1177  int q;
1179 
1180  // disconnect interface sockets
1183 
1184  for (q = 0; q < VHOST_VRING_MAX_N; q++)
1185  {
1186  clib_mem_free ((void *) vui->vring_locks[q]);
1187  }
1188 
1189  if (vui->unix_server_index != ~0)
1190  {
1191  //Close server socket
1193  vui->unix_server_index);
1194  clib_file_del (&file_main, uf);
1195  vui->unix_server_index = ~0;
1196  unlink (vui->sock_filename);
1197  }
1198 
1200  &vui->if_index);
1201 }
1202 
1203 int
1205 {
1207  vhost_user_intf_t *vui;
1208  int rv = 0;
1209  vnet_hw_interface_t *hwif;
1210  u16 *queue;
1211 
1212  if (!(hwif = vnet_get_sup_hw_interface (vnm, sw_if_index)) ||
1214  return VNET_API_ERROR_INVALID_SW_IF_INDEX;
1215 
1216  DBG_SOCK ("Deleting vhost-user interface %s (instance %d)",
1217  hwif->name, hwif->dev_instance);
1218 
1220 
1221  vec_foreach (queue, vui->rx_queues)
1222  {
1223  vhost_user_vring_t *txvq;
1224 
1225  txvq = &vui->vrings[VHOST_VRING_IDX_TX (*queue)];
1226  if ((vum->ifq_count > 0) &&
1229  {
1230  vum->ifq_count--;
1231  // Stop the timer if there is no more interrupt interface/queue
1232  if ((vum->ifq_count == 0) &&
1233  (vum->coalesce_time > 0.0) && (vum->coalesce_frames > 0))
1234  {
1238  break;
1239  }
1240  }
1241  }
1242 
1243  // Disable and reset interface
1244  vhost_user_term_if (vui);
1245 
1246  // Reset renumbered iface
1247  if (hwif->dev_instance <
1250 
1251  // Delete ethernet interface
1253 
1254  // Back to pool
1255  pool_put (vum->vhost_user_interfaces, vui);
1256 
1257  return rv;
1258 }
1259 
1260 static clib_error_t *
1262 {
1263  vnet_main_t *vnm = vnet_get_main ();
1265  vhost_user_intf_t *vui;
1266 
1268  /* *INDENT-OFF* */
1269  pool_foreach (vui, vum->vhost_user_interfaces, {
1270  vhost_user_delete_if (vnm, vm, vui->sw_if_index);
1271  });
1272  /* *INDENT-ON* */
1274  return 0;
1275 }
1276 
1278 
1279 /**
1280  * Open server unix socket on specified sock_filename.
1281  */
1282 static int
1283 vhost_user_init_server_sock (const char *sock_filename, int *sock_fd)
1284 {
1285  int rv = 0;
1286  struct sockaddr_un un = { };
1287  int fd;
1288  /* create listening socket */
1289  if ((fd = socket (AF_UNIX, SOCK_STREAM, 0)) < 0)
1290  return VNET_API_ERROR_SYSCALL_ERROR_1;
1291 
1292  un.sun_family = AF_UNIX;
1293  strncpy ((char *) un.sun_path, (char *) sock_filename,
1294  sizeof (un.sun_path) - 1);
1295 
1296  /* remove if exists */
1297  unlink ((char *) sock_filename);
1298 
1299  if (bind (fd, (struct sockaddr *) &un, sizeof (un)) == -1)
1300  {
1301  rv = VNET_API_ERROR_SYSCALL_ERROR_2;
1302  goto error;
1303  }
1304 
1305  if (listen (fd, 1) == -1)
1306  {
1307  rv = VNET_API_ERROR_SYSCALL_ERROR_3;
1308  goto error;
1309  }
1310 
1311  *sock_fd = fd;
1312  return 0;
1313 
1314 error:
1315  close (fd);
1316  return rv;
1317 }
1318 
1319 /**
1320  * Create ethernet interface for vhost user interface.
1321  */
1322 static void
1324  vhost_user_intf_t * vui, u8 * hwaddress)
1325 {
1327  u8 hwaddr[6];
1328  clib_error_t *error;
1329 
1330  /* create hw and sw interface */
1331  if (hwaddress)
1332  {
1333  clib_memcpy (hwaddr, hwaddress, 6);
1334  }
1335  else
1336  {
1337  random_u32 (&vum->random);
1338  clib_memcpy (hwaddr + 2, &vum->random, sizeof (vum->random));
1339  hwaddr[0] = 2;
1340  hwaddr[1] = 0xfe;
1341  }
1342 
1344  (vnm,
1346  vui - vum->vhost_user_interfaces /* device instance */ ,
1347  hwaddr /* ethernet address */ ,
1348  &vui->hw_if_index, 0 /* flag change */ );
1349 
1350  if (error)
1351  clib_error_report (error);
1352 
1353  vnet_sw_interface_set_mtu (vnm, vui->sw_if_index, 9000);
1354 }
1355 
1356 /*
1357  * Initialize vui with specified attributes
1358  */
1359 static void
1361  vhost_user_intf_t * vui,
1362  int server_sock_fd,
1363  const char *sock_filename,
1364  u64 feature_mask, u32 * sw_if_index)
1365 {
1366  vnet_sw_interface_t *sw;
1367  int q;
1369  vnet_hw_interface_t *hw;
1370 
1371  hw = vnet_get_hw_interface (vnm, vui->hw_if_index);
1372  sw = vnet_get_hw_sw_interface (vnm, vui->hw_if_index);
1373  if (server_sock_fd != -1)
1374  {
1375  clib_file_t template = { 0 };
1377  template.file_descriptor = server_sock_fd;
1378  template.private_data = vui - vum->vhost_user_interfaces; //hw index
1379  vui->unix_server_index = clib_file_add (&file_main, &template);
1380  }
1381  else
1382  {
1383  vui->unix_server_index = ~0;
1384  }
1385 
1386  vui->sw_if_index = sw->sw_if_index;
1387  strncpy (vui->sock_filename, sock_filename,
1388  ARRAY_LEN (vui->sock_filename) - 1);
1389  vui->sock_errno = 0;
1390  vui->is_up = 0;
1391  vui->feature_mask = feature_mask;
1392  vui->clib_file_index = ~0;
1393  vui->log_base_addr = 0;
1394  vui->if_index = vui - vum->vhost_user_interfaces;
1396  &vui->if_index, 0);
1397 
1398  for (q = 0; q < VHOST_VRING_MAX_N; q++)
1399  vhost_user_vring_init (vui, q);
1400 
1403 
1404  if (sw_if_index)
1405  *sw_if_index = vui->sw_if_index;
1406 
1407  for (q = 0; q < VHOST_VRING_MAX_N; q++)
1408  {
1411  memset ((void *) vui->vring_locks[q], 0, CLIB_CACHE_LINE_BYTES);
1412  }
1413 
1415  vlib_get_thread_main ()->n_vlib_mains - 1);
1417 }
1418 
1419 int
1421  const char *sock_filename,
1422  u8 is_server,
1423  u32 * sw_if_index,
1424  u64 feature_mask,
1425  u8 renumber, u32 custom_dev_instance, u8 * hwaddr)
1426 {
1427  vhost_user_intf_t *vui = NULL;
1428  u32 sw_if_idx = ~0;
1429  int rv = 0;
1430  int server_sock_fd = -1;
1432  uword *if_index;
1433 
1434  if (sock_filename == NULL || !(strlen (sock_filename) > 0))
1435  {
1436  return VNET_API_ERROR_INVALID_ARGUMENT;
1437  }
1438 
1439  if_index = mhash_get (&vum->if_index_by_sock_name, (void *) sock_filename);
1440  if (if_index)
1441  {
1442  if (sw_if_index)
1443  {
1444  vui = &vum->vhost_user_interfaces[*if_index];
1445  *sw_if_index = vui->sw_if_index;
1446  }
1447  return VNET_API_ERROR_IF_ALREADY_EXISTS;
1448  }
1449 
1450  if (is_server)
1451  {
1452  if ((rv =
1453  vhost_user_init_server_sock (sock_filename, &server_sock_fd)) != 0)
1454  {
1455  return rv;
1456  }
1457  }
1458 
1459  pool_get (vhost_user_main.vhost_user_interfaces, vui);
1460 
1461  vhost_user_create_ethernet (vnm, vm, vui, hwaddr);
1462  vhost_user_vui_init (vnm, vui, server_sock_fd, sock_filename,
1463  feature_mask, &sw_if_idx);
1464 
1465  if (renumber)
1466  vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
1467 
1468  if (sw_if_index)
1469  *sw_if_index = sw_if_idx;
1470 
1471  // Process node must connect
1473 
1474  return rv;
1475 }
1476 
1477 int
1479  const char *sock_filename,
1480  u8 is_server,
1481  u32 sw_if_index,
1482  u64 feature_mask, u8 renumber, u32 custom_dev_instance)
1483 {
1485  vhost_user_intf_t *vui = NULL;
1486  u32 sw_if_idx = ~0;
1487  int server_sock_fd = -1;
1488  int rv = 0;
1489  vnet_hw_interface_t *hwif;
1490  uword *if_index;
1491 
1492  if (!(hwif = vnet_get_sup_hw_interface (vnm, sw_if_index)) ||
1494  return VNET_API_ERROR_INVALID_SW_IF_INDEX;
1495 
1496  if (sock_filename == NULL || !(strlen (sock_filename) > 0))
1497  return VNET_API_ERROR_INVALID_ARGUMENT;
1498 
1500 
1501  /*
1502  * Disallow changing the interface to have the same path name
1503  * as other interface
1504  */
1505  if_index = mhash_get (&vum->if_index_by_sock_name, (void *) sock_filename);
1506  if (if_index && (*if_index != vui->if_index))
1507  return VNET_API_ERROR_IF_ALREADY_EXISTS;
1508 
1509  // First try to open server socket
1510  if (is_server)
1511  if ((rv = vhost_user_init_server_sock (sock_filename,
1512  &server_sock_fd)) != 0)
1513  return rv;
1514 
1515  vhost_user_term_if (vui);
1516  vhost_user_vui_init (vnm, vui, server_sock_fd,
1517  sock_filename, feature_mask, &sw_if_idx);
1518 
1519  if (renumber)
1520  vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
1521 
1522  // Process node must connect
1524 
1525  return rv;
1526 }
1527 
1528 clib_error_t *
1530  unformat_input_t * input,
1531  vlib_cli_command_t * cmd)
1532 {
1533  unformat_input_t _line_input, *line_input = &_line_input;
1534  u8 *sock_filename = NULL;
1535  u32 sw_if_index;
1536  u8 is_server = 0;
1537  u64 feature_mask = (u64) ~ (0ULL);
1538  u8 renumber = 0;
1539  u32 custom_dev_instance = ~0;
1540  u8 hwaddr[6];
1541  u8 *hw = NULL;
1542  clib_error_t *error = NULL;
1543 
1544  /* Get a line of input. */
1545  if (!unformat_user (input, unformat_line_input, line_input))
1546  return 0;
1547 
1548  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
1549  {
1550  if (unformat (line_input, "socket %s", &sock_filename))
1551  ;
1552  else if (unformat (line_input, "server"))
1553  is_server = 1;
1554  else if (unformat (line_input, "feature-mask 0x%llx", &feature_mask))
1555  ;
1556  else
1557  if (unformat
1558  (line_input, "hwaddr %U", unformat_ethernet_address, hwaddr))
1559  hw = hwaddr;
1560  else if (unformat (line_input, "renumber %d", &custom_dev_instance))
1561  {
1562  renumber = 1;
1563  }
1564  else
1565  {
1566  error = clib_error_return (0, "unknown input `%U'",
1567  format_unformat_error, line_input);
1568  goto done;
1569  }
1570  }
1571 
1572  vnet_main_t *vnm = vnet_get_main ();
1573 
1574  int rv;
1575  if ((rv = vhost_user_create_if (vnm, vm, (char *) sock_filename,
1576  is_server, &sw_if_index, feature_mask,
1577  renumber, custom_dev_instance, hw)))
1578  {
1579  error = clib_error_return (0, "vhost_user_create_if returned %d", rv);
1580  goto done;
1581  }
1582 
1584  sw_if_index);
1585 
1586 done:
1587  vec_free (sock_filename);
1588  unformat_free (line_input);
1589 
1590  return error;
1591 }
1592 
1593 clib_error_t *
1595  unformat_input_t * input,
1596  vlib_cli_command_t * cmd)
1597 {
1598  unformat_input_t _line_input, *line_input = &_line_input;
1599  u32 sw_if_index = ~0;
1600  vnet_main_t *vnm = vnet_get_main ();
1601  clib_error_t *error = NULL;
1602 
1603  /* Get a line of input. */
1604  if (!unformat_user (input, unformat_line_input, line_input))
1605  return 0;
1606 
1607  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
1608  {
1609  if (unformat (line_input, "sw_if_index %d", &sw_if_index))
1610  ;
1611  else if (unformat
1612  (line_input, "%U", unformat_vnet_sw_interface, vnm,
1613  &sw_if_index))
1614  {
1615  vnet_hw_interface_t *hwif =
1616  vnet_get_sup_hw_interface (vnm, sw_if_index);
1617  if (hwif == NULL ||
1619  {
1620  error = clib_error_return (0, "Not a vhost interface");
1621  goto done;
1622  }
1623  }
1624  else
1625  {
1626  error = clib_error_return (0, "unknown input `%U'",
1627  format_unformat_error, line_input);
1628  goto done;
1629  }
1630  }
1631 
1632  vhost_user_delete_if (vnm, vm, sw_if_index);
1633 
1634 done:
1635  unformat_free (line_input);
1636 
1637  return error;
1638 }
1639 
1640 int
1642  vhost_user_intf_details_t ** out_vuids)
1643 {
1644  int rv = 0;
1646  vhost_user_intf_t *vui;
1647  vhost_user_intf_details_t *r_vuids = NULL;
1649  u32 *hw_if_indices = 0;
1651  u8 *s = NULL;
1652  int i;
1653 
1654  if (!out_vuids)
1655  return -1;
1656 
1658  vec_add1 (hw_if_indices, vui->hw_if_index);
1659  );
1660 
1661  for (i = 0; i < vec_len (hw_if_indices); i++)
1662  {
1663  hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
1665 
1666  vec_add2 (r_vuids, vuid, 1);
1667  vuid->sw_if_index = vui->sw_if_index;
1668  vuid->virtio_net_hdr_sz = vui->virtio_net_hdr_sz;
1669  vuid->features = vui->features;
1670  vuid->num_regions = vui->nregions;
1671  vuid->is_server = vui->unix_server_index != ~0;
1672  vuid->sock_errno = vui->sock_errno;
1673  strncpy ((char *) vuid->sock_filename, (char *) vui->sock_filename,
1674  sizeof (vuid->sock_filename));
1675  vuid->sock_filename[ARRAY_LEN (vuid->sock_filename) - 1] = '\0';
1676  s = format (s, "%v%c", hi->name, 0);
1677 
1678  strncpy ((char *) vuid->if_name, (char *) s,
1679  ARRAY_LEN (vuid->if_name) - 1);
1680  _vec_len (s) = 0;
1681  }
1682 
1683  vec_free (s);
1684  vec_free (hw_if_indices);
1685 
1686  *out_vuids = r_vuids;
1687 
1688  return rv;
1689 }
1690 
1691 clib_error_t *
1693  unformat_input_t * input,
1694  vlib_cli_command_t * cmd)
1695 {
1696  clib_error_t *error = 0;
1697  vnet_main_t *vnm = vnet_get_main ();
1699  vhost_user_intf_t *vui;
1700  u32 hw_if_index, *hw_if_indices = 0;
1702  u16 *queue;
1703  u32 ci;
1704  int i, j, q;
1705  int show_descr = 0;
1706  struct feat_struct
1707  {
1708  u8 bit;
1709  char *str;
1710  };
1711  struct feat_struct *feat_entry;
1712 
1713  static struct feat_struct feat_array[] = {
1714 #define _(s,b) { .str = #s, .bit = b, },
1716 #undef _
1717  {.str = NULL}
1718  };
1719 
1720 #define foreach_protocol_feature \
1721  _(VHOST_USER_PROTOCOL_F_MQ) \
1722  _(VHOST_USER_PROTOCOL_F_LOG_SHMFD)
1723 
1724  static struct feat_struct proto_feat_array[] = {
1725 #define _(s) { .str = #s, .bit = s},
1727 #undef _
1728  {.str = NULL}
1729  };
1730 
1731  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1732  {
1733  if (unformat
1734  (input, "%U", unformat_vnet_hw_interface, vnm, &hw_if_index))
1735  {
1736  hi = vnet_get_hw_interface (vnm, hw_if_index);
1737  if (vhost_user_device_class.index != hi->dev_class_index)
1738  {
1739  error = clib_error_return (0, "unknown input `%U'",
1740  format_unformat_error, input);
1741  goto done;
1742  }
1743  vec_add1 (hw_if_indices, hw_if_index);
1744  }
1745  else if (unformat (input, "descriptors") || unformat (input, "desc"))
1746  show_descr = 1;
1747  else
1748  {
1749  error = clib_error_return (0, "unknown input `%U'",
1750  format_unformat_error, input);
1751  goto done;
1752  }
1753  }
1754  if (vec_len (hw_if_indices) == 0)
1755  {
1757  vec_add1 (hw_if_indices, vui->hw_if_index);
1758  );
1759  }
1760  vlib_cli_output (vm, "Virtio vhost-user interfaces");
1761  vlib_cli_output (vm, "Global:\n coalesce frames %d time %e",
1762  vum->coalesce_frames, vum->coalesce_time);
1763  vlib_cli_output (vm, " number of rx virtqueues in interrupt mode: %d",
1764  vum->ifq_count);
1765 
1766  for (i = 0; i < vec_len (hw_if_indices); i++)
1767  {
1768  hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
1770  vlib_cli_output (vm, "Interface: %s (ifindex %d)",
1771  hi->name, hw_if_indices[i]);
1772 
1773  vlib_cli_output (vm, "virtio_net_hdr_sz %d\n"
1774  " features mask (0x%llx): \n"
1775  " features (0x%llx): \n",
1776  vui->virtio_net_hdr_sz, vui->feature_mask,
1777  vui->features);
1778 
1779  feat_entry = (struct feat_struct *) &feat_array;
1780  while (feat_entry->str)
1781  {
1782  if (vui->features & (1ULL << feat_entry->bit))
1783  vlib_cli_output (vm, " %s (%d)", feat_entry->str,
1784  feat_entry->bit);
1785  feat_entry++;
1786  }
1787 
1788  vlib_cli_output (vm, " protocol features (0x%llx)",
1789  vui->protocol_features);
1790  feat_entry = (struct feat_struct *) &proto_feat_array;
1791  while (feat_entry->str)
1792  {
1793  if (vui->protocol_features & (1ULL << feat_entry->bit))
1794  vlib_cli_output (vm, " %s (%d)", feat_entry->str,
1795  feat_entry->bit);
1796  feat_entry++;
1797  }
1798 
1799  vlib_cli_output (vm, "\n");
1800 
1801  vlib_cli_output (vm, " socket filename %s type %s errno \"%s\"\n\n",
1802  vui->sock_filename,
1803  (vui->unix_server_index != ~0) ? "server" : "client",
1804  strerror (vui->sock_errno));
1805 
1806  vlib_cli_output (vm, " rx placement: ");
1807 
1808  vec_foreach (queue, vui->rx_queues)
1809  {
1810  vnet_main_t *vnm = vnet_get_main ();
1811  uword thread_index;
1813 
1814  thread_index = vnet_get_device_input_thread_index (vnm,
1815  vui->hw_if_index,
1816  *queue);
1817  vnet_hw_interface_get_rx_mode (vnm, vui->hw_if_index, *queue, &mode);
1818  vlib_cli_output (vm, " thread %d on vring %d, %U\n",
1819  thread_index, VHOST_VRING_IDX_TX (*queue),
1821  }
1822 
1823  vlib_cli_output (vm, " tx placement: %s\n",
1824  vui->use_tx_spinlock ? "spin-lock" : "lock-free");
1825 
1827  {
1828  vlib_cli_output (vm, " thread %d on vring %d\n", ci,
1829  VHOST_VRING_IDX_RX (vui->per_cpu_tx_qid[ci]));
1830  }
1831 
1832  vlib_cli_output (vm, "\n");
1833 
1834  vlib_cli_output (vm, " Memory regions (total %d)\n", vui->nregions);
1835 
1836  if (vui->nregions)
1837  {
1838  vlib_cli_output (vm,
1839  " region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr\n");
1840  vlib_cli_output (vm,
1841  " ====== ===== ================== ================== ================== ================== ==================\n");
1842  }
1843  for (j = 0; j < vui->nregions; j++)
1844  {
1845  vlib_cli_output (vm,
1846  " %d %-5d 0x%016lx 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n",
1847  j, vui->region_mmap_fd[j],
1848  vui->regions[j].guest_phys_addr,
1849  vui->regions[j].memory_size,
1850  vui->regions[j].userspace_addr,
1851  vui->regions[j].mmap_offset,
1853  }
1854  for (q = 0; q < VHOST_VRING_MAX_N; q++)
1855  {
1856  if (!vui->vrings[q].started)
1857  continue;
1858 
1859  vlib_cli_output (vm, "\n Virtqueue %d (%s%s)\n", q,
1860  (q & 1) ? "RX" : "TX",
1861  vui->vrings[q].enabled ? "" : " disabled");
1862 
1863  vlib_cli_output (vm,
1864  " qsz %d last_avail_idx %d last_used_idx %d\n",
1865  vui->vrings[q].qsz_mask + 1,
1866  vui->vrings[q].last_avail_idx,
1867  vui->vrings[q].last_used_idx);
1868 
1869  if (vui->vrings[q].avail && vui->vrings[q].used)
1870  vlib_cli_output (vm,
1871  " avail.flags %x avail.idx %d used.flags %x used.idx %d\n",
1872  vui->vrings[q].avail->flags,
1873  vui->vrings[q].avail->idx,
1874  vui->vrings[q].used->flags,
1875  vui->vrings[q].used->idx);
1876 
1877  int kickfd = UNIX_GET_FD (vui->vrings[q].kickfd_idx);
1878  int callfd = UNIX_GET_FD (vui->vrings[q].callfd_idx);
1879  vlib_cli_output (vm, " kickfd %d callfd %d errfd %d\n",
1880  kickfd, callfd, vui->vrings[q].errfd);
1881 
1882  if (show_descr)
1883  {
1884  vlib_cli_output (vm, "\n descriptor table:\n");
1885  vlib_cli_output (vm,
1886  " id addr len flags next user_addr\n");
1887  vlib_cli_output (vm,
1888  " ===== ================== ===== ====== ===== ==================\n");
1889  for (j = 0; j < vui->vrings[q].qsz_mask + 1; j++)
1890  {
1891  u32 mem_hint = 0;
1892  vlib_cli_output (vm,
1893  " %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
1894  j, vui->vrings[q].desc[j].addr,
1895  vui->vrings[q].desc[j].len,
1896  vui->vrings[q].desc[j].flags,
1897  vui->vrings[q].desc[j].next,
1899  (vui,
1900  vui->vrings[q].desc[j].
1901  addr, &mem_hint)));
1902  }
1903  }
1904  }
1905  vlib_cli_output (vm, "\n");
1906  }
1907 done:
1908  vec_free (hw_if_indices);
1909  return error;
1910 }
1911 
1912 /*
1913  * CLI functions
1914  */
1915 
1916 /*?
1917  * Create a vHost User interface. Once created, a new virtual interface
1918  * will exist with the name '<em>VirtualEthernet0/0/x</em>', where '<em>x</em>'
1919  * is the next free index.
1920  *
1921  * There are several parameters associated with a vHost interface:
1922  *
1923  * - <b>socket <socket-filename></b> - Name of the linux socket used by hypervisor
1924  * and VPP to manage the vHost interface. If in '<em>server</em>' mode, VPP will
1925  * create the socket if it does not already exist. If in '<em>client</em>' mode,
1926  * hypervisor will create the socket if it does not already exist. The VPP code
1927  * is indifferent to the file location. However, if SELinux is enabled, then the
1928  * socket needs to be created in '<em>/var/run/vpp/</em>'.
1929  *
1930  * - <b>server</b> - Optional flag to indicate that VPP should be the server for
1931  * the linux socket. If not provided, VPP will be the client. In '<em>server</em>'
1932  * mode, the VM can be reset without tearing down the vHost Interface. In
1933  * '<em>client</em>' mode, VPP can be reset without bringing down the VM and
1934  * tearing down the vHost Interface.
1935  *
1936  * - <b>feature-mask <hex></b> - Optional virtio/vhost feature set negotiated at
1937  * startup. <b>This is intended for degugging only.</b> It is recommended that this
1938  * parameter not be used except by experienced users. By default, all supported
1939  * features will be advertised. Otherwise, provide the set of features desired.
1940  * - 0x000008000 (15) - VIRTIO_NET_F_MRG_RXBUF
1941  * - 0x000020000 (17) - VIRTIO_NET_F_CTRL_VQ
1942  * - 0x000200000 (21) - VIRTIO_NET_F_GUEST_ANNOUNCE
1943  * - 0x000400000 (22) - VIRTIO_NET_F_MQ
1944  * - 0x004000000 (26) - VHOST_F_LOG_ALL
1945  * - 0x008000000 (27) - VIRTIO_F_ANY_LAYOUT
1946  * - 0x010000000 (28) - VIRTIO_F_INDIRECT_DESC
1947  * - 0x040000000 (30) - VHOST_USER_F_PROTOCOL_FEATURES
1948  * - 0x100000000 (32) - VIRTIO_F_VERSION_1
1949  *
1950  * - <b>hwaddr <mac-addr></b> - Optional ethernet address, can be in either
1951  * X:X:X:X:X:X unix or X.X.X cisco format.
1952  *
1953  * - <b>renumber <dev_instance></b> - Optional parameter which allows the instance
1954  * in the name to be specified. If instance already exists, name will be used
1955  * anyway and multiple instances will have the same name. Use with caution.
1956  *
1957  * @cliexpar
1958  * Example of how to create a vhost interface with VPP as the client and all features enabled:
1959  * @cliexstart{create vhost-user socket /var/run/vpp/vhost1.sock}
1960  * VirtualEthernet0/0/0
1961  * @cliexend
1962  * Example of how to create a vhost interface with VPP as the server and with just
1963  * multiple queues enabled:
1964  * @cliexstart{create vhost-user socket /var/run/vpp/vhost2.sock server feature-mask 0x40400000}
1965  * VirtualEthernet0/0/1
1966  * @cliexend
1967  * Once the vHost interface is created, enable the interface using:
1968  * @cliexcmd{set interface state VirtualEthernet0/0/0 up}
1969 ?*/
1970 /* *INDENT-OFF* */
1971 VLIB_CLI_COMMAND (vhost_user_connect_command, static) = {
1972  .path = "create vhost-user",
1973  .short_help = "create vhost-user socket <socket-filename> [server] "
1974  "[feature-mask <hex>] [hwaddr <mac-addr>] [renumber <dev_instance>] ",
1975  .function = vhost_user_connect_command_fn,
1976 };
1977 /* *INDENT-ON* */
1978 
1979 /*?
1980  * Delete a vHost User interface using the interface name or the
1981  * software interface index. Use the '<em>show interface</em>'
1982  * command to determine the software interface index. On deletion,
1983  * the linux socket will not be deleted.
1984  *
1985  * @cliexpar
1986  * Example of how to delete a vhost interface by name:
1987  * @cliexcmd{delete vhost-user VirtualEthernet0/0/1}
1988  * Example of how to delete a vhost interface by software interface index:
1989  * @cliexcmd{delete vhost-user sw_if_index 1}
1990 ?*/
1991 /* *INDENT-OFF* */
1992 VLIB_CLI_COMMAND (vhost_user_delete_command, static) = {
1993  .path = "delete vhost-user",
1994  .short_help = "delete vhost-user {<interface> | sw_if_index <sw_idx>}",
1995  .function = vhost_user_delete_command_fn,
1996 };
1997 
1998 /*?
1999  * Display the attributes of a single vHost User interface (provide interface
2000  * name), multiple vHost User interfaces (provide a list of interface names seperated
2001  * by spaces) or all Vhost User interfaces (omit an interface name to display all
2002  * vHost interfaces).
2003  *
2004  * @cliexpar
2005  * @parblock
2006  * Example of how to display a vhost interface:
2007  * @cliexstart{show vhost-user VirtualEthernet0/0/0}
2008  * Virtio vhost-user interfaces
2009  * Global:
2010  * coalesce frames 32 time 1e-3
2011  * Interface: VirtualEthernet0/0/0 (ifindex 1)
2012  * virtio_net_hdr_sz 12
2013  * features mask (0xffffffffffffffff):
2014  * features (0x50408000):
2015  * VIRTIO_NET_F_MRG_RXBUF (15)
2016  * VIRTIO_NET_F_MQ (22)
2017  * VIRTIO_F_INDIRECT_DESC (28)
2018  * VHOST_USER_F_PROTOCOL_FEATURES (30)
2019  * protocol features (0x3)
2020  * VHOST_USER_PROTOCOL_F_MQ (0)
2021  * VHOST_USER_PROTOCOL_F_LOG_SHMFD (1)
2022  *
2023  * socket filename /var/run/vpp/vhost1.sock type client errno "Success"
2024  *
2025  * rx placement:
2026  * thread 1 on vring 1
2027  * thread 1 on vring 5
2028  * thread 2 on vring 3
2029  * thread 2 on vring 7
2030  * tx placement: spin-lock
2031  * thread 0 on vring 0
2032  * thread 1 on vring 2
2033  * thread 2 on vring 0
2034  *
2035  * Memory regions (total 2)
2036  * region fd guest_phys_addr memory_size userspace_addr mmap_offset mmap_addr
2037  * ====== ===== ================== ================== ================== ================== ==================
2038  * 0 60 0x0000000000000000 0x00000000000a0000 0x00002aaaaac00000 0x0000000000000000 0x00002aab2b400000
2039  * 1 61 0x00000000000c0000 0x000000003ff40000 0x00002aaaaacc0000 0x00000000000c0000 0x00002aababcc0000
2040  *
2041  * Virtqueue 0 (TX)
2042  * qsz 256 last_avail_idx 0 last_used_idx 0
2043  * avail.flags 1 avail.idx 128 used.flags 1 used.idx 0
2044  * kickfd 62 callfd 64 errfd -1
2045  *
2046  * Virtqueue 1 (RX)
2047  * qsz 256 last_avail_idx 0 last_used_idx 0
2048  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
2049  * kickfd 65 callfd 66 errfd -1
2050  *
2051  * Virtqueue 2 (TX)
2052  * qsz 256 last_avail_idx 0 last_used_idx 0
2053  * avail.flags 1 avail.idx 128 used.flags 1 used.idx 0
2054  * kickfd 63 callfd 70 errfd -1
2055  *
2056  * Virtqueue 3 (RX)
2057  * qsz 256 last_avail_idx 0 last_used_idx 0
2058  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
2059  * kickfd 72 callfd 74 errfd -1
2060  *
2061  * Virtqueue 4 (TX disabled)
2062  * qsz 256 last_avail_idx 0 last_used_idx 0
2063  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
2064  * kickfd 76 callfd 78 errfd -1
2065  *
2066  * Virtqueue 5 (RX disabled)
2067  * qsz 256 last_avail_idx 0 last_used_idx 0
2068  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
2069  * kickfd 80 callfd 82 errfd -1
2070  *
2071  * Virtqueue 6 (TX disabled)
2072  * qsz 256 last_avail_idx 0 last_used_idx 0
2073  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
2074  * kickfd 84 callfd 86 errfd -1
2075  *
2076  * Virtqueue 7 (RX disabled)
2077  * qsz 256 last_avail_idx 0 last_used_idx 0
2078  * avail.flags 1 avail.idx 0 used.flags 1 used.idx 0
2079  * kickfd 88 callfd 90 errfd -1
2080  *
2081  * @cliexend
2082  *
2083  * The optional '<em>descriptors</em>' parameter will display the same output as
2084  * the previous example but will include the descriptor table for each queue.
2085  * The output is truncated below:
2086  * @cliexstart{show vhost-user VirtualEthernet0/0/0 descriptors}
2087  * Virtio vhost-user interfaces
2088  * Global:
2089  * coalesce frames 32 time 1e-3
2090  * Interface: VirtualEthernet0/0/0 (ifindex 1)
2091  * virtio_net_hdr_sz 12
2092  * features mask (0xffffffffffffffff):
2093  * features (0x50408000):
2094  * VIRTIO_NET_F_MRG_RXBUF (15)
2095  * VIRTIO_NET_F_MQ (22)
2096  * :
2097  * Virtqueue 0 (TX)
2098  * qsz 256 last_avail_idx 0 last_used_idx 0
2099  * avail.flags 1 avail.idx 128 used.flags 1 used.idx 0
2100  * kickfd 62 callfd 64 errfd -1
2101  *
2102  * descriptor table:
2103  * id addr len flags next user_addr
2104  * ===== ================== ===== ====== ===== ==================
2105  * 0 0x0000000010b6e974 2060 0x0002 1 0x00002aabbc76e974
2106  * 1 0x0000000010b6e034 2060 0x0002 2 0x00002aabbc76e034
2107  * 2 0x0000000010b6d6f4 2060 0x0002 3 0x00002aabbc76d6f4
2108  * 3 0x0000000010b6cdb4 2060 0x0002 4 0x00002aabbc76cdb4
2109  * 4 0x0000000010b6c474 2060 0x0002 5 0x00002aabbc76c474
2110  * 5 0x0000000010b6bb34 2060 0x0002 6 0x00002aabbc76bb34
2111  * 6 0x0000000010b6b1f4 2060 0x0002 7 0x00002aabbc76b1f4
2112  * 7 0x0000000010b6a8b4 2060 0x0002 8 0x00002aabbc76a8b4
2113  * 8 0x0000000010b69f74 2060 0x0002 9 0x00002aabbc769f74
2114  * 9 0x0000000010b69634 2060 0x0002 10 0x00002aabbc769634
2115  * 10 0x0000000010b68cf4 2060 0x0002 11 0x00002aabbc768cf4
2116  * :
2117  * 249 0x0000000000000000 0 0x0000 250 0x00002aab2b400000
2118  * 250 0x0000000000000000 0 0x0000 251 0x00002aab2b400000
2119  * 251 0x0000000000000000 0 0x0000 252 0x00002aab2b400000
2120  * 252 0x0000000000000000 0 0x0000 253 0x00002aab2b400000
2121  * 253 0x0000000000000000 0 0x0000 254 0x00002aab2b400000
2122  * 254 0x0000000000000000 0 0x0000 255 0x00002aab2b400000
2123  * 255 0x0000000000000000 0 0x0000 32768 0x00002aab2b400000
2124  *
2125  * Virtqueue 1 (RX)
2126  * qsz 256 last_avail_idx 0 last_used_idx 0
2127  * :
2128  * @cliexend
2129  * @endparblock
2130 ?*/
2131 /* *INDENT-OFF* */
2132 VLIB_CLI_COMMAND (show_vhost_user_command, static) = {
2133  .path = "show vhost-user",
2134  .short_help = "show vhost-user [<interface> [<interface> [..]]] [descriptors]",
2135  .function = show_vhost_user_command_fn,
2136 };
2137 /* *INDENT-ON* */
2138 
2139 clib_error_t *
2141  unformat_input_t * input,
2142  vlib_cli_command_t * cmd)
2143 {
2144  unformat_input_t _line_input, *line_input = &_line_input;
2145  clib_error_t *error = NULL;
2147  u8 onoff = 0;
2148  u8 input_found = 0;
2149 
2150  /* Get a line of input. */
2151  if (!unformat_user (input, unformat_line_input, line_input))
2152  return clib_error_return (0, "missing argument");
2153 
2154  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2155  {
2156  if (input_found)
2157  {
2158  error = clib_error_return (0, "unknown input `%U'",
2159  format_unformat_error, line_input);
2160  goto done;
2161  }
2162 
2163  if (unformat (line_input, "on"))
2164  {
2165  input_found = 1;
2166  onoff = 1;
2167  }
2168  else if (unformat (line_input, "off"))
2169  {
2170  input_found = 1;
2171  onoff = 0;
2172  }
2173  else
2174  {
2175  error = clib_error_return (0, "unknown input `%U'",
2176  format_unformat_error, line_input);
2177  goto done;
2178  }
2179  }
2180 
2181  vum->debug = onoff;
2182 
2183 done:
2184  unformat_free (line_input);
2185 
2186  return error;
2187 }
2188 
2189 /* *INDENT-OFF* */
2190 VLIB_CLI_COMMAND (debug_vhost_user_command, static) = {
2191  .path = "debug vhost-user",
2192  .short_help = "debug vhost-user <on | off>",
2193  .function = debug_vhost_user_command_fn,
2194 };
2195 /* *INDENT-ON* */
2196 
2197 static clib_error_t *
2199 {
2201 
2202  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2203  {
2204  if (unformat (input, "coalesce-frames %d", &vum->coalesce_frames))
2205  ;
2206  else if (unformat (input, "coalesce-time %f", &vum->coalesce_time))
2207  ;
2208  else if (unformat (input, "dont-dump-memory"))
2209  vum->dont_dump_vhost_user_memory = 1;
2210  else
2211  return clib_error_return (0, "unknown input `%U'",
2212  format_unformat_error, input);
2213  }
2214 
2215  return 0;
2216 }
2217 
2218 
2219 /* vhost-user { ... } configuration. */
2220 VLIB_CONFIG_FUNCTION (vhost_user_config, "vhost-user");
2221 
2222 void
2224 {
2226  vhost_user_intf_t *vui;
2227 
2228  if (vum->dont_dump_vhost_user_memory)
2229  {
2231  unmap_all_mem_regions (vui);
2232  );
2233  }
2234 }
2235 
2236 /*
2237  * fd.io coding-style-patch-verification: ON
2238  *
2239  * Local Variables:
2240  * eval: (c-set-style "gnu")
2241  * End:
2242  */
unformat_function_t unformat_vnet_hw_interface
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:437
vmrglw vmrglh hi
#define vec_foreach_index(var, v)
Iterate over vector indices.
static void clib_file_del(clib_file_main_t *um, clib_file_t *f)
Definition: file.h:109
vring_desc_t * desc
Definition: vhost_user.h:232
static uword random_default_seed(void)
Default random seed (unix/linux user-mode)
Definition: random.h:91
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:541
#define VHOST_USER_PROTOCOL_F_LOG_SHMFD
Definition: vhost_user.h:32
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:699
VNET_HW_INTERFACE_CLASS(vhost_interface_class, static)
vhost_cpu_t * cpus
Per-CPU data for vhost-user.
Definition: vhost_user.h:340
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:321
static void vhost_user_tx_thread_placement(vhost_user_intf_t *vui)
Definition: vhost_user.c:110
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
u64 region_guest_addr_hi[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost_user.h:275
static clib_error_t * vhost_user_socksvr_accept_ready(clib_file_t *uf)
Definition: vhost_user.c:898
unsigned long u64
Definition: types.h:89
#define VHOST_VRING_MAX_N
Definition: vhost_user.h:22
#define NULL
Definition: clib.h:55
uword mhash_unset(mhash_t *h, void *key, uword *old_value)
Definition: mhash.c:353
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:225
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
vring_avail_t * avail
Definition: vhost_user.h:233
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 file_descriptor
Definition: file.h:54
vnet_device_class_t vhost_user_device_class
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
int vnet_interface_name_renumber(u32 sw_if_index, u32 new_show_dev_instance)
Definition: interface.c:1393
static void vhost_user_vui_init(vnet_main_t *vnm, vhost_user_intf_t *vui, int server_sock_fd, const char *sock_filename, u64 feature_mask, u32 *sw_if_index)
Definition: vhost_user.c:1360
static clib_error_t * vhost_user_callfd_read_ready(clib_file_t *uf)
Definition: vhost_user.c:260
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:562
int i
#define VHOST_USER_EVENT_START_TIMER
Definition: vhost_user.h:250
uword unformat_user(unformat_input_t *input, unformat_function_t *func,...)
Definition: unformat.c:983
static_always_inline void vhost_user_vring_init(vhost_user_intf_t *vui, u32 qid)
Definition: vhost_user.c:296
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
unformat_function_t unformat_vnet_sw_interface
#define VNET_HW_INTERFACE_FLAG_LINK_UP
Definition: interface.h:458
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:228
vring_used_t * used
Definition: vhost_user.h:234
vhost_vring_addr_t addr
Definition: vhost_user.h:116
int vhost_user_modify_if(vnet_main_t *vnm, vlib_main_t *vm, const char *sock_filename, u8 is_server, u32 sw_if_index, u64 feature_mask, u8 renumber, u32 custom_dev_instance)
Definition: vhost_user.c:1478
format_function_t format_vnet_sw_if_index_name
#define VHOST_USER_VRING_NOFD_MASK
Definition: vhost_user.h:26
unsigned char u8
Definition: types.h:56
static uword vlib_process_suspend_time_is_zero(f64 dt)
Returns TRUE if a process suspend time is less than 10us.
Definition: node_funcs.h:436
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
double f64
Definition: types.h:142
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:212
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
clib_file_t * file_pool
Definition: file.h:88
vnet_hw_interface_rx_mode
Definition: interface.h:51
#define static_always_inline
Definition: clib.h:93
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:443
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:156
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:542
static clib_error_t * ip4_init(vlib_main_t *vm)
Definition: ip4_input.c:360
static clib_error_t * vhost_user_socket_read(clib_file_t *uf)
Definition: vhost_user.c:367
#define UNIX_GET_FD(unixfd_idx)
Definition: vhost_user.h:63
void * log_base_addr
Definition: vhost_user.h:285
static_always_inline void * map_guest_mem(vhost_user_intf_t *vui, uword addr, u32 *hint)
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
#define VHOST_VRING_IDX_TX(qid)
Definition: vhost_user.h:24
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define VRING_USED_F_NO_NOTIFY
Definition: vhost_user.h:45
#define clib_error_return(e, args...)
Definition: error.h:99
clib_file_main_t file_main
Definition: main.c:63
unsigned int u32
Definition: types.h:88
static vlib_node_registration_t vhost_user_process_node
(constructor) VLIB_REGISTER_NODE (vhost_user_process_node)
Definition: vhost_user.c:1163
#define vlib_call_init_function(vm, x)
Definition: init.h:227
static clib_error_t * vhost_user_kickfd_read_ready(clib_file_t *uf)
Definition: vhost_user.c:271
static clib_error_t * vhost_user_socket_error(clib_file_t *uf)
Definition: vhost_user.c:882
void vhost_user_unmap_all(void)
Definition: vhost_user.c:2223
static void vhost_user_rx_thread_placement()
Unassign existing interface/queue to thread mappings and re-assign new interface/queue to thread mapp...
Definition: vhost_user.c:150
unformat_function_t unformat_line_input
Definition: format.h:282
#define VHOST_USER_PROTOCOL_F_MQ
Definition: vhost_user.h:31
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:464
static_always_inline uword vnet_get_device_input_thread_index(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:127
static void vhost_user_term_if(vhost_user_intf_t *vui)
Disables and reset interface structure.
Definition: vhost_user.c:1175
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:952
static_always_inline int vhost_user_intf_ready(vhost_user_intf_t *vui)
Returns whether at least one TX and one RX vring are enabled.
Definition: vhost_user.c:214
u32 random
Pseudo random iterator.
Definition: vhost_user.h:343
uword mhash_set_mem(mhash_t *h, void *key, uword *new_value, uword *old_value)
Definition: mhash.c:271
int vhost_user_create_if(vnet_main_t *vnm, vlib_main_t *vm, const char *sock_filename, u8 is_server, u32 *sw_if_index, u64 feature_mask, u8 renumber, u32 custom_dev_instance, u8 *hwaddr)
Definition: vhost_user.c:1420
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
#define clib_error_return_unix(e, args...)
Definition: error.h:102
int vnet_hw_interface_get_rx_mode(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, vnet_hw_interface_rx_mode *mode)
Definition: devices.c:312
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
int vhost_user_delete_if(vnet_main_t *vnm, vlib_main_t *vm, u32 sw_if_index)
Definition: vhost_user.c:1204
format_function_t format_vnet_hw_interface_rx_mode
#define VLIB_CONFIG_FUNCTION(x, n,...)
Definition: init.h:164
char sock_filename[256]
Definition: vhost_user.h:260
vhost_user_main_t vhost_user_main
Definition: vhost_user.c:56
static void vhost_user_update_iface_state(vhost_user_intf_t *vui)
Definition: vhost_user.c:226
#define VHOST_USER_MSG_HDR_SZ
Definition: vhost_user.h:20
u32 region_mmap_fd[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost_user.h:276
int vhost_user_dump_ifs(vnet_main_t *vnm, vlib_main_t *vm, vhost_user_intf_details_t **out_vuids)
Definition: vhost_user.c:1641
#define VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE
Definition: interface.h:496
vhost_user_memory_region_t regions[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost_user.h:272
#define VHOST_VRING_IDX_RX(qid)
Definition: vhost_user.h:23
u32 * show_dev_instance_by_real_dev_instance
Definition: vhost_user.h:334
static_always_inline void vhost_user_vring_close(vhost_user_intf_t *vui, u32 qid)
Definition: vhost_user.c:318
vhost_user_intf_t * vhost_user_interfaces
Definition: vhost_user.h:333
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:153
static void mhash_init_c_string(mhash_t *h, uword n_value_bytes)
Definition: mhash.h:78
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
#define DBG_SOCK(args...)
Definition: vhost_user.h:48
vlib_main_t * vm
Definition: buffer.c:294
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
static void unmap_all_mem_regions(vhost_user_intf_t *vui)
Definition: vhost_user.c:74
static clib_error_t * vhost_user_config(vlib_main_t *vm, unformat_input_t *input)
Definition: vhost_user.c:2198
#define VLIB_MAIN_LOOP_EXIT_FUNCTION(x)
Definition: init.h:161
#define VHOST_USER_EVENT_STOP_TIMER
Definition: vhost_user.h:251
#define clib_warning(format, args...)
Definition: error.h:59
static uword vhost_user_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: vhost_user.c:1064
#define clib_memcpy(a, b, c)
Definition: string.h:75
#define ARRAY_LEN(x)
Definition: clib.h:59
clib_error_t * vhost_user_delete_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: vhost_user.c:1594
#define foreach_protocol_feature
static int vhost_user_init_server_sock(const char *sock_filename, int *sock_fd)
Open server unix socket on specified sock_filename.
Definition: vhost_user.c:1283
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
uword unformat_ethernet_address(unformat_input_t *input, va_list *args)
Definition: format.c:227
clib_error_t * show_vhost_user_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: vhost_user.c:1692
#define ASSERT(truth)
#define VHOST_VRING_F_LOG
Definition: vhost_user.h:33
static uword * mhash_get(mhash_t *h, const void *key)
Definition: mhash.h:110
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
static_always_inline void vhost_user_if_disconnect(vhost_user_intf_t *vui)
Definition: vhost_user.c:344
static void clib_mem_free(void *p)
Definition: mem.h:179
volatile u32 * vring_locks[VHOST_VRING_MAX_N]
Definition: vhost_user.h:280
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:275
mhash_t if_index_by_sock_name
Definition: vhost_user.h:331
static clib_error_t * vhost_user_exit(vlib_main_t *vm)
Definition: vhost_user.c:1261
#define clib_error_report(e)
Definition: error.h:113
vlib_node_registration_t vhost_user_send_interrupt_node
(constructor) VLIB_REGISTER_NODE (vhost_user_send_interrupt_node)
Definition: vhost_user.c:53
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static void vhost_user_create_ethernet(vnet_main_t *vnm, vlib_main_t *vm, vhost_user_intf_t *vui, u8 *hwaddress)
Create ethernet interface for vhost user interface.
Definition: vhost_user.c:1323
clib_error_t * debug_vhost_user_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: vhost_user.c:2140
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
struct _vlib_node_registration vlib_node_registration_t
void * region_mmap_addr[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost_user.h:273
static long get_huge_page_size(int fd)
Definition: vhost_user.c:66
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
vhost_user_vring_t vrings[VHOST_VRING_MAX_N]
Definition: vhost_user.h:279
static void vhost_user_set_interrupt_pending(vhost_user_intf_t *vui, u32 ifq)
Definition: vhost_user.c:244
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
#define VHOST_MEMORY_MAX_NREGIONS
Definition: vhost_user.h:19
clib_error_t * vhost_user_connect_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: vhost_user.c:1529
void vnet_sw_interface_set_mtu(vnet_main_t *vnm, u32 sw_if_index, u32 mtu)
Definition: interface.c:676
u32 rx_buffers_len
Definition: vhost_user.h:318
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:120
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1513
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
u64 region_guest_addr_lo[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost_user.h:274
#define vec_foreach(var, vec)
Vector iterator.
uword private_data
Definition: file.h:64
Definition: file.h:51
static clib_error_t * vhost_user_init(vlib_main_t *vm)
Definition: vhost_user.c:934
static_always_inline void * map_user_mem(vhost_user_intf_t *vui, uword addr)
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
static uword vhost_user_send_interrupt_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: vhost_user.c:967
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:681
vl_api_address_union_t un
Definition: ip_types.api:37
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
int dont_dump_vhost_user_memory
Definition: vhost_user.h:337