vhost-user.c

Go to the documentation of this file.

/*
 *------------------------------------------------------------------
 * vhost.c - vhost-user
 *
 * Copyright (c) 2014 Cisco and/or its affiliates.
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at:
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *------------------------------------------------------------------
 */

#include <fcntl.h>              /* for open */
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/uio.h>            /* for iovec */
#include <netinet/in.h>
#include <sys/vfs.h>

#include <linux/if_arp.h>
#include <linux/if_tun.h>

#include <vlib/vlib.h>
#include <vlib/unix/unix.h>

#include <vnet/ip/ip.h>

#include <vnet/ethernet/ethernet.h>
#include <vnet/devices/devices.h>
#include <vnet/feature/feature.h>

#include <vnet/devices/virtio/vhost-user.h>

#define VHOST_USER_DEBUG_SOCKET 0
#define VHOST_USER_DEBUG_VQ 0

/* Set to get virtio_net_hdr in buffer pre-data
   details will be shown in  packet trace */
#define VHOST_USER_COPY_TX_HDR 0

#if VHOST_USER_DEBUG_SOCKET == 1
#define DBG_SOCK(args...) clib_warning(args);
#else
#define DBG_SOCK(args...)
#endif

#if VHOST_USER_DEBUG_VQ == 1
#define DBG_VQ(args...) clib_warning(args);
#else
#define DBG_VQ(args...)
#endif

vlib_node_registration_t vhost_user_input_node;

#define foreach_vhost_user_tx_func_error      \
  _(NONE, "no error")  \
  _(NOT_READY, "vhost user state error")  \
  _(PKT_DROP_NOBUF, "tx packet drops (no available descriptors)")  \
  _(PKT_DROP_NOMRG, "tx packet drops (cannot merge descriptors)")  \
  _(MMAP_FAIL, "mmap failure") \
  _(INDIRECT_OVERFLOW, "indirect descriptor table overflow")

typedef enum
{
#define _(f,s) VHOST_USER_TX_FUNC_ERROR_##f,
  foreach_vhost_user_tx_func_error
#undef _
    VHOST_USER_TX_FUNC_N_ERROR,
} vhost_user_tx_func_error_t;

static char *vhost_user_tx_func_error_strings[] = {
#define _(n,s) s,
  foreach_vhost_user_tx_func_error
#undef _
};

#define foreach_vhost_user_input_func_error      \
  _(NO_ERROR, "no error")  \
  _(NO_BUFFER, "no available buffer")  \
  _(MMAP_FAIL, "mmap failure")  \
  _(INDIRECT_OVERFLOW, "indirect descriptor overflows table")  \
  _(UNDERSIZED_FRAME, "undersized ethernet frame received (< 14 bytes)") \
  _(FULL_RX_QUEUE, "full rx queue (possible driver tx drop)")

typedef enum
{
#define _(f,s) VHOST_USER_INPUT_FUNC_ERROR_##f,
  foreach_vhost_user_input_func_error
#undef _
    VHOST_USER_INPUT_FUNC_N_ERROR,
} vhost_user_input_func_error_t;

static char *vhost_user_input_func_error_strings[] = {
#define _(n,s) s,
  foreach_vhost_user_input_func_error
#undef _
};

/* *INDENT-OFF* */
static vhost_user_main_t vhost_user_main = {
  .mtu_bytes = 1518,
};

VNET_HW_INTERFACE_CLASS (vhost_interface_class, static) = {
  .name = "vhost-user",
};
/* *INDENT-ON* */

static u8 *
format_vhost_user_interface_name (u8 * s, va_list * args)
{
  u32 i = va_arg (*args, u32);
  u32 show_dev_instance = ~0;
  vhost_user_main_t *vum = &vhost_user_main;

  if (i < vec_len (vum->show_dev_instance_by_real_dev_instance))
    show_dev_instance = vum->show_dev_instance_by_real_dev_instance[i];

  if (show_dev_instance != ~0)
    i = show_dev_instance;

  s = format (s, "VirtualEthernet0/0/%d", i);
  return s;
}

static int
vhost_user_name_renumber (vnet_hw_interface_t * hi, u32 new_dev_instance)
{
  vhost_user_main_t *vum = &vhost_user_main;

  vec_validate_init_empty (vum->show_dev_instance_by_real_dev_instance,
                           hi->dev_instance, ~0);

  vum->show_dev_instance_by_real_dev_instance[hi->dev_instance] =
    new_dev_instance;

  DBG_SOCK ("renumbered vhost-user interface dev_instance %d to %d",
            hi->dev_instance, new_dev_instance);

  return 0;
}

static_always_inline void *
map_guest_mem (vhost_user_intf_t * vui, uword addr, u32 * hint)
{
  int i = *hint;
  if (PREDICT_TRUE ((vui->regions[i].guest_phys_addr <= addr) &&
                    ((vui->regions[i].guest_phys_addr +
                      vui->regions[i].memory_size) > addr)))
    {
      return (void *) (vui->region_mmap_addr[i] + addr -
                       vui->regions[i].guest_phys_addr);
    }
#if __SSE4_2__
  __m128i rl, rh, al, ah, r;
  al = _mm_set1_epi64x (addr + 1);
  ah = _mm_set1_epi64x (addr);

  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[0]);
  rl = _mm_cmpgt_epi64 (al, rl);
  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[0]);
  rh = _mm_cmpgt_epi64 (rh, ah);
  r = _mm_and_si128 (rl, rh);

  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[2]);
  rl = _mm_cmpgt_epi64 (al, rl);
  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[2]);
  rh = _mm_cmpgt_epi64 (rh, ah);
  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x22);

  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[4]);
  rl = _mm_cmpgt_epi64 (al, rl);
  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[4]);
  rh = _mm_cmpgt_epi64 (rh, ah);
  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x44);

  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[6]);
  rl = _mm_cmpgt_epi64 (al, rl);
  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[6]);
  rh = _mm_cmpgt_epi64 (rh, ah);
  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x88);

  r = _mm_shuffle_epi8 (r, _mm_set_epi64x (0, 0x0e060c040a020800));
  i = __builtin_ctzll (_mm_movemask_epi8 (r));

  if (i < vui->nregions)
    {
      *hint = i;
      return (void *) (vui->region_mmap_addr[i] + addr -
                       vui->regions[i].guest_phys_addr);
    }

#else
  for (i = 0; i < vui->nregions; i++)
    {
      if ((vui->regions[i].guest_phys_addr <= addr) &&
          ((vui->regions[i].guest_phys_addr + vui->regions[i].memory_size) >
           addr))
        {
          *hint = i;
          return (void *) (vui->region_mmap_addr[i] + addr -
                           vui->regions[i].guest_phys_addr);
        }
    }
#endif
  DBG_VQ ("failed to map guest mem addr %llx", addr);
  *hint = 0;
  return 0;
}

static inline void *
map_user_mem (vhost_user_intf_t * vui, uword addr)
{
  int i;
  for (i = 0; i < vui->nregions; i++)
    {
      if ((vui->regions[i].userspace_addr <= addr) &&
          ((vui->regions[i].userspace_addr + vui->regions[i].memory_size) >
           addr))
        {
          return (void *) (vui->region_mmap_addr[i] + addr -
                           vui->regions[i].userspace_addr);
        }
    }
  return 0;
}

static long
get_huge_page_size (int fd)
{
  struct statfs s;
  fstatfs (fd, &s);
  return s.f_bsize;
}

static void
unmap_all_mem_regions (vhost_user_intf_t * vui)
{
  int i, r;
  for (i = 0; i < vui->nregions; i++)
    {
      if (vui->region_mmap_addr[i] != (void *) -1)
        {

          long page_sz = get_huge_page_size (vui->region_mmap_fd[i]);

          ssize_t map_sz = (vui->regions[i].memory_size +
                            vui->regions[i].mmap_offset +
                            page_sz) & ~(page_sz - 1);

          r =
            munmap (vui->region_mmap_addr[i] - vui->regions[i].mmap_offset,
                    map_sz);

          DBG_SOCK
            ("unmap memory region %d addr 0x%lx len 0x%lx page_sz 0x%x", i,
             vui->region_mmap_addr[i], map_sz, page_sz);

          vui->region_mmap_addr[i] = (void *) -1;

          if (r == -1)
            {
              clib_warning ("failed to unmap memory region (errno %d)",
                            errno);
            }
          close (vui->region_mmap_fd[i]);
        }
    }
  vui->nregions = 0;
}


static clib_error_t *
vhost_user_callfd_read_ready (unix_file_t * uf)
{
  __attribute__ ((unused)) int n;
  u8 buff[8];
  n = read (uf->file_descriptor, ((char *) &buff), 8);
  return 0;
}

static inline void
vhost_user_if_disconnect (vhost_user_intf_t * vui)
{
  vhost_user_main_t *vum = &vhost_user_main;
  vnet_main_t *vnm = vnet_get_main ();
  int q;

  vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);

  if (vui->unix_file_index != ~0)
    {
      unix_file_del (&unix_main, unix_main.file_pool + vui->unix_file_index);
      vui->unix_file_index = ~0;
    }
  else
    close (vui->unix_fd);

  hash_unset (vum->vhost_user_interface_index_by_sock_fd, vui->unix_fd);
  hash_unset (vum->vhost_user_interface_index_by_listener_fd, vui->unix_fd);
  vui->unix_fd = -1;
  vui->is_up = 0;
  for (q = 0; q < vui->num_vrings; q++)
    {
      if (vui->vrings[q].callfd > -1)
        {
          unix_file_t *uf = pool_elt_at_index (unix_main.file_pool,
                                               vui->vrings[q].callfd_idx);
          unix_file_del (&unix_main, uf);
        }

      if (vui->vrings[q].kickfd > -1)
        close (vui->vrings[q].kickfd);

      vui->vrings[q].callfd = -1;
      vui->vrings[q].kickfd = -1;
      vui->vrings[q].desc = NULL;
      vui->vrings[q].avail = NULL;
      vui->vrings[q].used = NULL;
      vui->vrings[q].log_guest_addr = 0;
      vui->vrings[q].log_used = 0;
    }

  unmap_all_mem_regions (vui);
  DBG_SOCK ("interface ifindex %d disconnected", vui->sw_if_index);
}

#define VHOST_LOG_PAGE 0x1000
always_inline void
vhost_user_log_dirty_pages (vhost_user_intf_t * vui, u64 addr, u64 len)
{
  if (PREDICT_TRUE (vui->log_base_addr == 0
                    || !(vui->features & (1 << FEAT_VHOST_F_LOG_ALL))))
    {
      return;
    }
  if (PREDICT_FALSE ((addr + len - 1) / VHOST_LOG_PAGE / 8 >= vui->log_size))
    {
      DBG_SOCK ("vhost_user_log_dirty_pages(): out of range\n");
      return;
    }

  CLIB_MEMORY_BARRIER ();
  u64 page = addr / VHOST_LOG_PAGE;
  while (page * VHOST_LOG_PAGE < addr + len)
    {
      ((u8 *) vui->log_base_addr)[page / 8] |= 1 << page % 8;
      page++;
    }
}

#define vhost_user_log_dirty_ring(vui, vq, member) \
  if (PREDICT_FALSE(vq->log_used)) { \
    vhost_user_log_dirty_pages(vui, vq->log_guest_addr + STRUCT_OFFSET_OF(vring_used_t, member), \
                             sizeof(vq->used->member)); \
  }

static clib_error_t *
vhost_user_socket_read (unix_file_t * uf)
{
  int n, i;
  int fd, number_of_fds = 0;
  int fds[VHOST_MEMORY_MAX_NREGIONS];
  vhost_user_msg_t msg;
  struct msghdr mh;
  struct iovec iov[1];
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui;
  struct cmsghdr *cmsg;
  uword *p;
  u8 q;
  unix_file_t template = { 0 };
  vnet_main_t *vnm = vnet_get_main ();

  p = hash_get (vum->vhost_user_interface_index_by_sock_fd,
                uf->file_descriptor);
  if (p == 0)
    {
      DBG_SOCK ("FD %d doesn't belong to any interface", uf->file_descriptor);
      return 0;
    }
  else
    vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);

  char control[CMSG_SPACE (VHOST_MEMORY_MAX_NREGIONS * sizeof (int))];

  memset (&mh, 0, sizeof (mh));
  memset (control, 0, sizeof (control));

  for (i = 0; i < VHOST_MEMORY_MAX_NREGIONS; i++)
    fds[i] = -1;

  /* set the payload */
  iov[0].iov_base = (void *) &msg;
  iov[0].iov_len = VHOST_USER_MSG_HDR_SZ;

  mh.msg_iov = iov;
  mh.msg_iovlen = 1;
  mh.msg_control = control;
  mh.msg_controllen = sizeof (control);

  n = recvmsg (uf->file_descriptor, &mh, 0);

  if (n != VHOST_USER_MSG_HDR_SZ)
    goto close_socket;

  if (mh.msg_flags & MSG_CTRUNC)
    {
      goto close_socket;
    }

  cmsg = CMSG_FIRSTHDR (&mh);

  if (cmsg && (cmsg->cmsg_len > 0) && (cmsg->cmsg_level == SOL_SOCKET) &&
      (cmsg->cmsg_type == SCM_RIGHTS) &&
      (cmsg->cmsg_len - CMSG_LEN (0) <=
       VHOST_MEMORY_MAX_NREGIONS * sizeof (int)))
    {
      number_of_fds = (cmsg->cmsg_len - CMSG_LEN (0)) / sizeof (int);
      clib_memcpy (fds, CMSG_DATA (cmsg), number_of_fds * sizeof (int));
    }

  /* version 1, no reply bit set */
  if ((msg.flags & 7) != 1)
    {
      DBG_SOCK ("malformed message received. closing socket");
      goto close_socket;
    }

  {
    int rv __attribute__ ((unused));
    /* $$$$ pay attention to rv */
    rv = read (uf->file_descriptor, ((char *) &msg) + n, msg.size);
  }

  switch (msg.request)
    {
    case VHOST_USER_GET_FEATURES:
      DBG_SOCK ("if %d msg VHOST_USER_GET_FEATURES", vui->hw_if_index);

      msg.flags |= 4;
      msg.u64 = (1 << FEAT_VIRTIO_NET_F_MRG_RXBUF) |
        (1 << FEAT_VIRTIO_F_ANY_LAYOUT) |
        (1 << FEAT_VIRTIO_F_INDIRECT_DESC) |
        (1 << FEAT_VHOST_F_LOG_ALL) |
        (1 << FEAT_VIRTIO_NET_F_GUEST_ANNOUNCE) |
        (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES) |
        (1UL << FEAT_VIRTIO_F_VERSION_1);
      msg.u64 &= vui->feature_mask;

      msg.size = sizeof (msg.u64);
      break;

    case VHOST_USER_SET_FEATURES:
      DBG_SOCK ("if %d msg VHOST_USER_SET_FEATURES features 0x%016llx",
                vui->hw_if_index, msg.u64);

      vui->features = msg.u64;

      if (vui->features & (1 << FEAT_VIRTIO_NET_F_MRG_RXBUF))
        vui->virtio_net_hdr_sz = 12;
      else
        vui->virtio_net_hdr_sz = 10;

      vui->is_any_layout =
        (vui->features & (1 << FEAT_VIRTIO_F_ANY_LAYOUT)) ? 1 : 0;

      ASSERT (vui->virtio_net_hdr_sz < VLIB_BUFFER_PRE_DATA_SIZE);
      vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);
      vui->is_up = 0;

      for (q = 0; q < 2; q++)
        {
          vui->vrings[q].desc = 0;
          vui->vrings[q].avail = 0;
          vui->vrings[q].used = 0;
          vui->vrings[q].log_guest_addr = 0;
          vui->vrings[q].log_used = 0;
        }

      DBG_SOCK ("interface %d disconnected", vui->sw_if_index);

      break;

    case VHOST_USER_SET_MEM_TABLE:
      DBG_SOCK ("if %d msg VHOST_USER_SET_MEM_TABLE nregions %d",
                vui->hw_if_index, msg.memory.nregions);

      if ((msg.memory.nregions < 1) ||
          (msg.memory.nregions > VHOST_MEMORY_MAX_NREGIONS))
        {

          DBG_SOCK ("number of mem regions must be between 1 and %i",
                    VHOST_MEMORY_MAX_NREGIONS);

          goto close_socket;
        }

      if (msg.memory.nregions != number_of_fds)
        {
          DBG_SOCK ("each memory region must have FD");
          goto close_socket;
        }
      unmap_all_mem_regions (vui);
      for (i = 0; i < msg.memory.nregions; i++)
        {
          clib_memcpy (&(vui->regions[i]), &msg.memory.regions[i],
                       sizeof (vhost_user_memory_region_t));

          long page_sz = get_huge_page_size (fds[i]);

          /* align size to 2M page */
          ssize_t map_sz = (vui->regions[i].memory_size +
                            vui->regions[i].mmap_offset +
                            page_sz) & ~(page_sz - 1);

          vui->region_mmap_addr[i] = mmap (0, map_sz, PROT_READ | PROT_WRITE,
                                           MAP_SHARED, fds[i], 0);
          vui->region_guest_addr_lo[i] = vui->regions[i].guest_phys_addr;
          vui->region_guest_addr_hi[i] = vui->regions[i].guest_phys_addr +
            vui->regions[i].memory_size;

          DBG_SOCK
            ("map memory region %d addr 0 len 0x%lx fd %d mapped 0x%lx "
             "page_sz 0x%x", i, map_sz, fds[i], vui->region_mmap_addr[i],
             page_sz);

          if (vui->region_mmap_addr[i] == MAP_FAILED)
            {
              clib_warning ("failed to map memory. errno is %d", errno);
              goto close_socket;
            }
          vui->region_mmap_addr[i] += vui->regions[i].mmap_offset;
          vui->region_mmap_fd[i] = fds[i];
        }
      vui->nregions = msg.memory.nregions;
      break;

    case VHOST_USER_SET_VRING_NUM:
      DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_NUM idx %d num %d",
                vui->hw_if_index, msg.state.index, msg.state.num);

      if ((msg.state.num > 32768) ||    /* maximum ring size is 32768 */
          (msg.state.num == 0) ||       /* it cannot be zero */
          (msg.state.num % 2))  /* must be power of 2 */
        goto close_socket;
      vui->vrings[msg.state.index].qsz = msg.state.num;
      break;

    case VHOST_USER_SET_VRING_ADDR:
      DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ADDR idx %d",
                vui->hw_if_index, msg.state.index);

      vui->vrings[msg.state.index].desc = (vring_desc_t *)
        map_user_mem (vui, msg.addr.desc_user_addr);
      vui->vrings[msg.state.index].used = (vring_used_t *)
        map_user_mem (vui, msg.addr.used_user_addr);
      vui->vrings[msg.state.index].avail = (vring_avail_t *)
        map_user_mem (vui, msg.addr.avail_user_addr);

      if ((vui->vrings[msg.state.index].desc == NULL) ||
          (vui->vrings[msg.state.index].used == NULL) ||
          (vui->vrings[msg.state.index].avail == NULL))
        {
          DBG_SOCK ("failed to map user memory for hw_if_index %d",
                    vui->hw_if_index);
          goto close_socket;
        }

      vui->vrings[msg.state.index].log_guest_addr = msg.addr.log_guest_addr;
      vui->vrings[msg.state.index].log_used =
        (msg.addr.flags & (1 << VHOST_VRING_F_LOG)) ? 1 : 0;

      /* Spec says: If VHOST_USER_F_PROTOCOL_FEATURES has not been negotiated,
         the ring is initialized in an enabled state. */

      if (!(vui->features & (1 << FEAT_VHOST_USER_F_PROTOCOL_FEATURES)))
        {
          vui->vrings[msg.state.index].enabled = 1;
        }

      vui->vrings[msg.state.index].last_used_idx =
        vui->vrings[msg.state.index].last_avail_idx =
        vui->vrings[msg.state.index].used->idx;

      /* tell driver that we don't want interrupts */
      vui->vrings[msg.state.index].used->flags |= 1;
      break;

    case VHOST_USER_SET_OWNER:
      DBG_SOCK ("if %d msg VHOST_USER_SET_OWNER", vui->hw_if_index);
      break;

    case VHOST_USER_RESET_OWNER:
      DBG_SOCK ("if %d msg VHOST_USER_RESET_OWNER", vui->hw_if_index);
      break;

    case VHOST_USER_SET_VRING_CALL:
      DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_CALL u64 %d",
                vui->hw_if_index, msg.u64);

      q = (u8) (msg.u64 & 0xFF);

      if (!(msg.u64 & 0x100))
        {
          if (number_of_fds != 1)
            goto close_socket;

          /* if there is old fd, delete it */
          if (vui->vrings[q].callfd > -1)
            {
              unix_file_t *uf = pool_elt_at_index (unix_main.file_pool,
                                                   vui->vrings[q].callfd_idx);
              unix_file_del (&unix_main, uf);
            }
          vui->vrings[q].callfd = fds[0];
          template.read_function = vhost_user_callfd_read_ready;
          template.file_descriptor = fds[0];
          vui->vrings[q].callfd_idx = unix_file_add (&unix_main, &template);
        }
      else
        vui->vrings[q].callfd = -1;
      break;

    case VHOST_USER_SET_VRING_KICK:
      DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_KICK u64 %d",
                vui->hw_if_index, msg.u64);

      q = (u8) (msg.u64 & 0xFF);

      if (!(msg.u64 & 0x100))
        {
          if (number_of_fds != 1)
            goto close_socket;

          if (vui->vrings[q].kickfd > -1)
            close (vui->vrings[q].kickfd);

          vui->vrings[q].kickfd = fds[0];
        }
      else
        vui->vrings[q].kickfd = -1;
      break;

    case VHOST_USER_SET_VRING_ERR:
      DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_ERR u64 %d",
                vui->hw_if_index, msg.u64);

      q = (u8) (msg.u64 & 0xFF);

      if (!(msg.u64 & 0x100))
        {
          if (number_of_fds != 1)
            goto close_socket;

          fd = fds[0];
        }
      else
        fd = -1;

      vui->vrings[q].errfd = fd;
      break;

    case VHOST_USER_SET_VRING_BASE:
      DBG_SOCK ("if %d msg VHOST_USER_SET_VRING_BASE idx %d num %d",
                vui->hw_if_index, msg.state.index, msg.state.num);

      vui->vrings[msg.state.index].last_avail_idx = msg.state.num;
      break;

    case VHOST_USER_GET_VRING_BASE:
      DBG_SOCK ("if %d msg VHOST_USER_GET_VRING_BASE idx %d num %d",
                vui->hw_if_index, msg.state.index, msg.state.num);

      /* Spec says: Client must [...] stop ring upon receiving VHOST_USER_GET_VRING_BASE. */
      vui->vrings[msg.state.index].enabled = 0;

      msg.state.num = vui->vrings[msg.state.index].last_avail_idx;
      msg.flags |= 4;
      msg.size = sizeof (msg.state);
      break;

    case VHOST_USER_NONE:
      DBG_SOCK ("if %d msg VHOST_USER_NONE", vui->hw_if_index);

      break;

    case VHOST_USER_SET_LOG_BASE:
      {
        DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_BASE", vui->hw_if_index);

        if (msg.size != sizeof (msg.log))
          {
            DBG_SOCK
              ("invalid msg size for VHOST_USER_SET_LOG_BASE: %d instead of %d",
               msg.size, sizeof (msg.log));
            goto close_socket;
          }

        if (!
            (vui->protocol_features & (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD)))
          {
            DBG_SOCK
              ("VHOST_USER_PROTOCOL_F_LOG_SHMFD not set but VHOST_USER_SET_LOG_BASE received");
            goto close_socket;
          }

        fd = fds[0];
        /* align size to 2M page */
        long page_sz = get_huge_page_size (fd);
        ssize_t map_sz =
          (msg.log.size + msg.log.offset + page_sz) & ~(page_sz - 1);

        vui->log_base_addr = mmap (0, map_sz, PROT_READ | PROT_WRITE,
                                   MAP_SHARED, fd, 0);

        DBG_SOCK
          ("map log region addr 0 len 0x%lx off 0x%lx fd %d mapped 0x%lx",
           map_sz, msg.log.offset, fd, vui->log_base_addr);

        if (vui->log_base_addr == MAP_FAILED)
          {
            clib_warning ("failed to map memory. errno is %d", errno);
            goto close_socket;
          }

        vui->log_base_addr += msg.log.offset;
        vui->log_size = msg.log.size;

        msg.flags |= 4;
        msg.size = sizeof (msg.u64);

        break;
      }

    case VHOST_USER_SET_LOG_FD:
      DBG_SOCK ("if %d msg VHOST_USER_SET_LOG_FD", vui->hw_if_index);

      break;

    case VHOST_USER_GET_PROTOCOL_FEATURES:
      DBG_SOCK ("if %d msg VHOST_USER_GET_PROTOCOL_FEATURES",
                vui->hw_if_index);

      msg.flags |= 4;
      msg.u64 = (1 << VHOST_USER_PROTOCOL_F_LOG_SHMFD);
      msg.size = sizeof (msg.u64);
      break;

    case VHOST_USER_SET_PROTOCOL_FEATURES:
      DBG_SOCK ("if %d msg VHOST_USER_SET_PROTOCOL_FEATURES features 0x%lx",
                vui->hw_if_index, msg.u64);

      vui->protocol_features = msg.u64;

      break;

    case VHOST_USER_SET_VRING_ENABLE:
      DBG_SOCK ("if %d VHOST_USER_SET_VRING_ENABLE, enable: %d",
                vui->hw_if_index, msg.state.num);
      vui->vrings[msg.state.index].enabled = msg.state.num;
      break;

    default:
      DBG_SOCK ("unknown vhost-user message %d received. closing socket",
                msg.request);
      goto close_socket;
    }

  /* if we have pointers to descriptor table, go up */
  if (!vui->is_up &&
      vui->vrings[VHOST_NET_VRING_IDX_TX].desc &&
      vui->vrings[VHOST_NET_VRING_IDX_RX].desc)
    {

      DBG_SOCK ("interface %d connected", vui->sw_if_index);

      vnet_hw_interface_set_flags (vnm, vui->hw_if_index,
                                   VNET_HW_INTERFACE_FLAG_LINK_UP);
      vui->is_up = 1;

    }

  /* if we need to reply */
  if (msg.flags & 4)
    {
      n =
        send (uf->file_descriptor, &msg, VHOST_USER_MSG_HDR_SZ + msg.size, 0);
      if (n != (msg.size + VHOST_USER_MSG_HDR_SZ))
        goto close_socket;
    }

  return 0;

close_socket:
  vhost_user_if_disconnect (vui);
  return 0;
}

static clib_error_t *
vhost_user_socket_error (unix_file_t * uf)
{
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui;
  uword *p;

  p = hash_get (vum->vhost_user_interface_index_by_sock_fd,
                uf->file_descriptor);
  if (p == 0)
    {
      DBG_SOCK ("fd %d doesn't belong to any interface", uf->file_descriptor);
      return 0;
    }
  else
    vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);

  vhost_user_if_disconnect (vui);
  return 0;
}

static clib_error_t *
vhost_user_socksvr_accept_ready (unix_file_t * uf)
{
  int client_fd, client_len;
  struct sockaddr_un client;
  unix_file_t template = { 0 };
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui;
  uword *p;

  p = hash_get (vum->vhost_user_interface_index_by_listener_fd,
                uf->file_descriptor);
  if (p == 0)
    {
      DBG_SOCK ("fd %d doesn't belong to any interface", uf->file_descriptor);
      return 0;
    }
  else
    vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);

  client_len = sizeof (client);
  client_fd = accept (uf->file_descriptor,
                      (struct sockaddr *) &client,
                      (socklen_t *) & client_len);

  if (client_fd < 0)
    return clib_error_return_unix (0, "accept");

  template.read_function = vhost_user_socket_read;
  template.error_function = vhost_user_socket_error;
  template.file_descriptor = client_fd;
  vui->unix_file_index = unix_file_add (&unix_main, &template);

  vui->client_fd = client_fd;
  hash_set (vum->vhost_user_interface_index_by_sock_fd, vui->client_fd,
            vui - vum->vhost_user_interfaces);

  return 0;
}

static clib_error_t *
vhost_user_init (vlib_main_t * vm)
{
  clib_error_t *error;
  vhost_user_main_t *vum = &vhost_user_main;
  vlib_thread_main_t *tm = vlib_get_thread_main ();
  vlib_thread_registration_t *tr;
  uword *p;

  error = vlib_call_init_function (vm, ip4_init);
  if (error)
    return error;

  vum->vhost_user_interface_index_by_listener_fd =
    hash_create (0, sizeof (uword));
  vum->vhost_user_interface_index_by_sock_fd =
    hash_create (0, sizeof (uword));
  vum->vhost_user_interface_index_by_sw_if_index =
    hash_create (0, sizeof (uword));
  vum->coalesce_frames = 32;
  vum->coalesce_time = 1e-3;

  vec_validate_aligned (vum->rx_buffers, tm->n_vlib_mains - 1,
                        CLIB_CACHE_LINE_BYTES);

  /* find out which cpus will be used for input */
  vum->input_cpu_first_index = 0;
  vum->input_cpu_count = 1;
  p = hash_get_mem (tm->thread_registrations_by_name, "workers");
  tr = p ? (vlib_thread_registration_t *) p[0] : 0;

  if (tr && tr->count > 0)
    {
      vum->input_cpu_first_index = tr->first_index;
      vum->input_cpu_count = tr->count;
    }

  return 0;
}

VLIB_INIT_FUNCTION (vhost_user_init);

static clib_error_t *
vhost_user_exit (vlib_main_t * vm)
{
  /* TODO cleanup */
  return 0;
}

VLIB_MAIN_LOOP_EXIT_FUNCTION (vhost_user_exit);

typedef struct
{
  u16 virtqueue;
  u16 device_index;
#if VHOST_USER_COPY_TX_HDR == 1
  virtio_net_hdr_t hdr;
#endif
} vhost_user_input_trace_t;

static u8 *
format_vhost_user_input_trace (u8 * s, va_list * va)
{
  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
  CLIB_UNUSED (vnet_main_t * vnm) = vnet_get_main ();
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_input_trace_t *t = va_arg (*va, vhost_user_input_trace_t *);
  vhost_user_intf_t *vui = vec_elt_at_index (vum->vhost_user_interfaces,
                                             t->device_index);

  vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, vui->sw_if_index);

#if VHOST_USER_COPY_TX_HDR == 1
  uword indent = format_get_indent (s);
#endif

  s = format (s, "%U virtqueue %d",
              format_vnet_sw_interface_name, vnm, sw, t->virtqueue);

#if VHOST_USER_COPY_TX_HDR == 1
  s = format (s, "\n%Uvirtio_net_hdr flags 0x%02x gso_type %u hdr_len %u",
              format_white_space, indent,
              t->hdr.flags, t->hdr.gso_type, t->hdr.hdr_len);
#endif

  return s;
}

void
vhost_user_rx_trace (vlib_main_t * vm,
                     vlib_node_runtime_t * node,
                     vhost_user_intf_t * vui, i16 virtqueue)
{
  u32 *b, n_left;
  vhost_user_main_t *vum = &vhost_user_main;

  u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;

  n_left = vec_len (vui->d_trace_buffers);
  b = vui->d_trace_buffers;

  while (n_left >= 1)
    {
      u32 bi0;
      vlib_buffer_t *b0;
      vhost_user_input_trace_t *t0;

      bi0 = b[0];
      n_left -= 1;

      b0 = vlib_get_buffer (vm, bi0);
      vlib_trace_buffer (vm, node, next_index, b0, /* follow_chain */ 0);
      t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
      t0->virtqueue = virtqueue;
      t0->device_index = vui - vum->vhost_user_interfaces;
#if VHOST_USER_COPY_TX_HDR == 1
      clib_memcpy (&t0->hdr, b0->pre_data, sizeof (virtio_net_hdr_t));
#endif

      b += 1;
    }
}

static inline void
vhost_user_send_call (vlib_main_t * vm, vhost_user_vring_t * vq)
{
  vhost_user_main_t *vum = &vhost_user_main;
  u64 x = 1;
  int rv __attribute__ ((unused));
  /* $$$$ pay attention to rv */
  rv = write (vq->callfd, &x, sizeof (x));
  vq->n_since_last_int = 0;
  vq->int_deadline = vlib_time_now (vm) + vum->coalesce_time;
}


static u32
vhost_user_if_input (vlib_main_t * vm,
                     vhost_user_main_t * vum,
                     vhost_user_intf_t * vui, vlib_node_runtime_t * node)
{
  vhost_user_vring_t *txvq = &vui->vrings[VHOST_NET_VRING_IDX_TX];
  vhost_user_vring_t *rxvq = &vui->vrings[VHOST_NET_VRING_IDX_RX];
  uword n_rx_packets = 0, n_rx_bytes = 0;
  uword n_left;
  u32 n_left_to_next, *to_next;
  u32 next_index = 0;
  u32 next0;
  uword n_trace = vlib_get_trace_count (vm, node);
  u16 qsz_mask;
  u32 cpu_index, rx_len, drops, flush;
  f64 now = vlib_time_now (vm);
  u32 map_guest_hint_desc = 0;
  u32 map_guest_hint_indirect = 0;
  u32 *map_guest_hint_p = &map_guest_hint_desc;

  vec_reset_length (vui->d_trace_buffers);

  /* no descriptor ptr - bail out */
  if (PREDICT_FALSE (!txvq->desc || !txvq->avail || !txvq->enabled))
    return 0;

  /* do we have pending intterupts ? */
  if ((txvq->n_since_last_int) && (txvq->int_deadline < now))
    vhost_user_send_call (vm, txvq);

  if ((rxvq->n_since_last_int) && (rxvq->int_deadline < now))
    vhost_user_send_call (vm, rxvq);

  /* only bit 0 of avail.flags is used so we don't want to deal with this
     interface if any other bit is set */
  if (PREDICT_FALSE (txvq->avail->flags & 0xFFFE))
    return 0;

  n_left = (u16) (txvq->avail->idx - txvq->last_avail_idx);

  /* nothing to do */
  if (PREDICT_FALSE (n_left == 0))
    return 0;

  if (PREDICT_FALSE (n_left == txvq->qsz))
    {
      //Informational error logging when VPP is not receiving packets fast enough
      vlib_error_count (vm, node->node_index,
                        VHOST_USER_INPUT_FUNC_ERROR_FULL_RX_QUEUE, 1);
    }

  if (PREDICT_FALSE (!vui->admin_up))
    {
      /* if intf is admin down, just drop all packets waiting in the ring */
      txvq->last_avail_idx = txvq->last_used_idx = txvq->avail->idx;
      CLIB_MEMORY_BARRIER ();
      txvq->used->idx = txvq->last_used_idx;
      vhost_user_log_dirty_ring (vui, txvq, idx);
      vhost_user_send_call (vm, txvq);
      return 0;
    }

  qsz_mask = txvq->qsz - 1;
  cpu_index = os_get_cpu_number ();
  drops = 0;
  flush = 0;

  if (n_left > VLIB_FRAME_SIZE)
    n_left = VLIB_FRAME_SIZE;

  /* Allocate some buffers.
   * Note that buffers that are chained for jumbo
   * frames are allocated separately using a slower path.
   * The idea is to be certain to have enough buffers at least
   * to cycle through the descriptors without having to check for errors.
   * For jumbo frames, the bottleneck is memory copy anyway.
   */
  if (PREDICT_FALSE (!vum->rx_buffers[cpu_index]))
    {
      vec_alloc (vum->rx_buffers[cpu_index], 2 * VLIB_FRAME_SIZE);

      if (PREDICT_FALSE (!vum->rx_buffers[cpu_index]))
        flush = n_left;         //Drop all input
    }

  if (PREDICT_FALSE (_vec_len (vum->rx_buffers[cpu_index]) < n_left))
    {
      u32 curr_len = _vec_len (vum->rx_buffers[cpu_index]);
      _vec_len (vum->rx_buffers[cpu_index]) +=
        vlib_buffer_alloc_from_free_list (vm,
                                          vum->rx_buffers[cpu_index] +
                                          curr_len,
                                          2 * VLIB_FRAME_SIZE - curr_len,
                                          VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);

      if (PREDICT_FALSE (n_left > _vec_len (vum->rx_buffers[cpu_index])))
        flush = n_left - _vec_len (vum->rx_buffers[cpu_index]);
    }

  if (PREDICT_FALSE (flush))
    {
      //Remove some input buffers
      drops += flush;
      n_left -= flush;
      vlib_error_count (vm, vhost_user_input_node.index,
                        VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER, flush);
      while (flush)
        {
          u16 desc_chain_head =
            txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
          txvq->last_avail_idx++;
          txvq->used->ring[txvq->last_used_idx & qsz_mask].id =
            desc_chain_head;
          txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0;
          vhost_user_log_dirty_ring (vui, txvq,
                                     ring[txvq->last_used_idx & qsz_mask]);
          txvq->last_used_idx++;
          flush--;
        }
    }

  rx_len = vec_len (vum->rx_buffers[cpu_index]);        //vector might be null
  while (n_left > 0)
    {
      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);

      while (n_left > 0 && n_left_to_next > 0)
        {
          vlib_buffer_t *b_head, *b_current;
          u32 bi_head, bi_current;
          u16 desc_chain_head, desc_current;
          u8 error = VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR;

          if (PREDICT_TRUE (n_left > 1))
            {
              u32 next_desc =
                txvq->avail->ring[(txvq->last_avail_idx + 1) & qsz_mask];
              void *buffer_addr =
                map_guest_mem (vui, txvq->desc[next_desc].addr,
                               &map_guest_hint_desc);
              if (PREDICT_TRUE (buffer_addr != 0))
                CLIB_PREFETCH (buffer_addr, 64, STORE);

              u32 bi = vum->rx_buffers[cpu_index][rx_len - 2];
              vlib_prefetch_buffer_with_index (vm, bi, STORE);
              CLIB_PREFETCH (vlib_get_buffer (vm, bi)->data, 128, STORE);
            }

          desc_chain_head = desc_current =
            txvq->avail->ring[txvq->last_avail_idx & qsz_mask];
          bi_head = bi_current = vum->rx_buffers[cpu_index][--rx_len];
          b_head = b_current = vlib_get_buffer (vm, bi_head);
          vlib_buffer_chain_init (b_head);

          uword offset;
          if (PREDICT_TRUE (vui->is_any_layout) ||
              (!(txvq->desc[desc_current].flags & VIRTQ_DESC_F_NEXT) &&
               !(txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)))
            {
              /* ANYLAYOUT or single buffer */
              offset = vui->virtio_net_hdr_sz;
            }
          else
            {
              /* CSR case without ANYLAYOUT, skip 1st buffer */
              offset = txvq->desc[desc_current].len;
            }

          vring_desc_t *desc_table = txvq->desc;
          u32 desc_index = desc_current;
          map_guest_hint_p = &map_guest_hint_desc;

          if (txvq->desc[desc_current].flags & VIRTQ_DESC_F_INDIRECT)
            {
              desc_table = map_guest_mem (vui, txvq->desc[desc_current].addr,
                                          &map_guest_hint_desc);
              desc_index = 0;
              map_guest_hint_p = &map_guest_hint_indirect;
              if (PREDICT_FALSE (desc_table == 0))
                {
                  error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
                  goto out;
                }
            }

          while (1)
            {
              void *buffer_addr =
                map_guest_mem (vui, desc_table[desc_index].addr,
                               map_guest_hint_p);
              if (PREDICT_FALSE (buffer_addr == 0))
                {
                  error = VHOST_USER_INPUT_FUNC_ERROR_MMAP_FAIL;
                  goto out;
                }

              if (PREDICT_TRUE
                  (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT))
                {
                  CLIB_PREFETCH (&desc_table[desc_table[desc_index].next],
                                 sizeof (vring_desc_t), STORE);
                }

#if VHOST_USER_COPY_TX_HDR == 1
              if (PREDICT_TRUE (offset))
                clib_memcpy (b->pre_data, buffer_addr, sizeof (virtio_net_hdr_t));      /* 12 byte hdr is not used on tx */
#endif

              if (desc_table[desc_index].len > offset)
                {
                  u16 len = desc_table[desc_index].len - offset;
                  u16 copied = vlib_buffer_chain_append_data_with_alloc (vm,
                                                                         VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX,
                                                                         b_head,
                                                                         &b_current,
                                                                         buffer_addr
                                                                         +
                                                                         offset,
                                                                         len);
                  if (copied != len)
                    {
                      error = VHOST_USER_INPUT_FUNC_ERROR_NO_BUFFER;
                      break;
                    }
                }
              offset = 0;

              /* if next flag is set, take next desc in the chain */
              if ((desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT))
                desc_index = desc_table[desc_index].next;
              else
                goto out;
            }
        out:

          /* consume the descriptor and return it as used */
          txvq->last_avail_idx++;
          txvq->used->ring[txvq->last_used_idx & qsz_mask].id =
            desc_chain_head;
          txvq->used->ring[txvq->last_used_idx & qsz_mask].len = 0;
          vhost_user_log_dirty_ring (vui, txvq,
                                     ring[txvq->last_used_idx & qsz_mask]);
          txvq->last_used_idx++;

          //It is important to free RX as fast as possible such that the TX
          //process does not drop packets
          if ((txvq->last_used_idx & 0x3f) == 0)        // Every 64 packets
            txvq->used->idx = txvq->last_used_idx;

          if (PREDICT_FALSE (b_head->current_length < 14 &&
                             error == VHOST_USER_INPUT_FUNC_ERROR_NO_ERROR))
            error = VHOST_USER_INPUT_FUNC_ERROR_UNDERSIZED_FRAME;

          VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b_head);

          vnet_buffer (b_head)->sw_if_index[VLIB_RX] = vui->sw_if_index;
          vnet_buffer (b_head)->sw_if_index[VLIB_TX] = (u32) ~ 0;
          b_head->error = node->errors[error];

          if (PREDICT_FALSE (n_trace > n_rx_packets))
            vec_add1 (vui->d_trace_buffers, bi_head);

          if (PREDICT_FALSE (error))
            {
              drops++;
              next0 = VNET_DEVICE_INPUT_NEXT_DROP;
            }
          else
            {
              n_rx_bytes +=
                b_head->current_length +
                b_head->total_length_not_including_first_buffer;
              n_rx_packets++;
              next0 = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT;
            }

          to_next[0] = bi_head;
          to_next++;
          n_left_to_next--;

          /* redirect if feature path enabled */
          vnet_feature_device_input_redirect_x1 (node, vui->sw_if_index,
                                                 &next0, b_head, 0);

          vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
                                           to_next, n_left_to_next,
                                           bi_head, next0);
          n_left--;
        }

      vlib_put_next_frame (vm, node, next_index, n_left_to_next);

    }

  if (PREDICT_TRUE (vum->rx_buffers[cpu_index] != 0))
    _vec_len (vum->rx_buffers[cpu_index]) = rx_len;

  /* give buffers back to driver */
  CLIB_MEMORY_BARRIER ();
  txvq->used->idx = txvq->last_used_idx;
  vhost_user_log_dirty_ring (vui, txvq, idx);

  if (PREDICT_FALSE (vec_len (vui->d_trace_buffers) > 0))
    {
      vhost_user_rx_trace (vm, node, vui, VHOST_NET_VRING_IDX_TX);
      vlib_set_trace_count (vm, node,
                            n_trace - vec_len (vui->d_trace_buffers));
    }

  /* interrupt (call) handling */
  if ((txvq->callfd > -1) && !(txvq->avail->flags & 1))
    {
      txvq->n_since_last_int += n_rx_packets;

      if (txvq->n_since_last_int > vum->coalesce_frames)
        vhost_user_send_call (vm, txvq);
    }

  if (PREDICT_FALSE (drops))
    {
      vlib_increment_simple_counter
        (vnet_main.interface_main.sw_if_counters
         + VNET_INTERFACE_COUNTER_DROP, os_get_cpu_number (),
         vui->sw_if_index, drops);
    }

  /* increase rx counters */
  vlib_increment_combined_counter
    (vnet_main.interface_main.combined_sw_if_counters
     + VNET_INTERFACE_COUNTER_RX,
     os_get_cpu_number (), vui->sw_if_index, n_rx_packets, n_rx_bytes);

  return n_rx_packets;
}

static uword
vhost_user_input (vlib_main_t * vm,
                  vlib_node_runtime_t * node, vlib_frame_t * f)
{
  vhost_user_main_t *vum = &vhost_user_main;
  u32 cpu_index = os_get_cpu_number ();
  vhost_user_intf_t *vui;
  uword n_rx_packets = 0;
  int i;

  for (i = 0; i < vec_len (vum->vhost_user_interfaces); i++)
    {
      vui = vec_elt_at_index (vum->vhost_user_interfaces, i);
      if (vui->is_up)
        {
          if ((i % vum->input_cpu_count) ==
              (cpu_index - vum->input_cpu_first_index))
            n_rx_packets += vhost_user_if_input (vm, vum, vui, node);
        }
    }
  return n_rx_packets;
}

/* *INDENT-OFF* */
VLIB_REGISTER_NODE (vhost_user_input_node) = {
  .function = vhost_user_input,
  .type = VLIB_NODE_TYPE_INPUT,
  .name = "vhost-user-input",

  /* Will be enabled if/when hardware is detected. */
  .state = VLIB_NODE_STATE_DISABLED,

  .format_buffer = format_ethernet_header_with_length,
  .format_trace = format_vhost_user_input_trace,

  .n_errors = VHOST_USER_INPUT_FUNC_N_ERROR,
  .error_strings = vhost_user_input_func_error_strings,

  .n_next_nodes = VNET_DEVICE_INPUT_N_NEXT_NODES,
  .next_nodes = VNET_DEVICE_INPUT_NEXT_NODES,
};

VLIB_NODE_FUNCTION_MULTIARCH (vhost_user_input_node, vhost_user_input)
/* *INDENT-ON* */

static uword
vhost_user_intfc_tx (vlib_main_t * vm,
                     vlib_node_runtime_t * node, vlib_frame_t * frame)
{
  u32 *buffers = vlib_frame_args (frame);
  u32 n_left = 0;
  vhost_user_main_t *vum = &vhost_user_main;
  uword n_packets = 0;
  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
  vhost_user_intf_t *vui =
    vec_elt_at_index (vum->vhost_user_interfaces, rd->dev_instance);
  vhost_user_vring_t *rxvq = &vui->vrings[VHOST_NET_VRING_IDX_RX];
  u16 qsz_mask;
  u8 error = VHOST_USER_TX_FUNC_ERROR_NONE;

  n_left = n_packets = frame->n_vectors;
  u32 map_guest_hint_desc = 0;
  u32 map_guest_hint_indirect = 0;
  u32 *map_guest_hint_p = &map_guest_hint_desc;

  if (PREDICT_FALSE (!vui->is_up))
    goto done2;

  if (PREDICT_FALSE
      (!rxvq->desc || !rxvq->avail || vui->sock_errno != 0 || !rxvq->enabled))
    {
      error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
      goto done2;
    }

  if (PREDICT_FALSE (vui->lockp != 0))
    {
      while (__sync_lock_test_and_set (vui->lockp, 1))
        ;
    }

  /* only bit 0 of avail.flags is used so we don't want to deal with this
     interface if any other bit is set */
  if (PREDICT_FALSE (rxvq->avail->flags & 0xFFFE))
    {
      error = VHOST_USER_TX_FUNC_ERROR_NOT_READY;
      goto done2;
    }

  if (PREDICT_FALSE ((rxvq->avail->idx == rxvq->last_avail_idx)))
    {
      error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
      goto done2;
    }

  qsz_mask = rxvq->qsz - 1;     /* qsz is always power of 2 */

  while (n_left > 0)
    {
      vlib_buffer_t *b0, *current_b0;
      u16 desc_head, desc_index, desc_len;
      vring_desc_t *desc_table;
      void *buffer_addr;
      u32 buffer_len;

      b0 = vlib_get_buffer (vm, buffers[0]);
      buffers++;

      if (PREDICT_FALSE (rxvq->last_avail_idx == rxvq->avail->idx))
        {
          error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
          goto done;
        }

      desc_table = rxvq->desc;
      map_guest_hint_p = &map_guest_hint_desc;
      desc_head = desc_index =
        rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
      if (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT)
        {
          if (PREDICT_FALSE
              (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
            {
              error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
              goto done;
            }
          if (PREDICT_FALSE
              (!(desc_table =
                 map_guest_mem (vui, rxvq->desc[desc_index].addr,
                                &map_guest_hint_desc))))
            {
              error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
              goto done;
            }
          desc_index = 0;
          map_guest_hint_p = &map_guest_hint_indirect;
        }

      desc_len = vui->virtio_net_hdr_sz;

      if (PREDICT_FALSE
          (!(buffer_addr =
             map_guest_mem (vui, desc_table[desc_index].addr,
                            map_guest_hint_p))))
        {
          error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
          goto done;
        }
      buffer_len = desc_table[desc_index].len;

      CLIB_PREFETCH (buffer_addr, CLIB_CACHE_LINE_BYTES, STORE);

      virtio_net_hdr_mrg_rxbuf_t *hdr =
        (virtio_net_hdr_mrg_rxbuf_t *) buffer_addr;
      hdr->hdr.flags = 0;
      hdr->hdr.gso_type = 0;
      if (vui->virtio_net_hdr_sz == 12)
        hdr->num_buffers = 1;

      vhost_user_log_dirty_pages (vui, desc_table[desc_index].addr,
                                  vui->virtio_net_hdr_sz);

      u16 bytes_left = b0->current_length;
      buffer_addr += vui->virtio_net_hdr_sz;
      buffer_len -= vui->virtio_net_hdr_sz;
      current_b0 = b0;
      while (1)
        {
          if (!bytes_left)
            {                   //Get new input
              if (current_b0->flags & VLIB_BUFFER_NEXT_PRESENT)
                {
                  current_b0 = vlib_get_buffer (vm, current_b0->next_buffer);
                  bytes_left = current_b0->current_length;
                }
              else
                {
                  //End of packet
                  break;
                }
            }

          if (buffer_len == 0)
            {                   //Get new output
              if (desc_table[desc_index].flags & VIRTQ_DESC_F_NEXT)
                {
                  //Next one is chained
                  desc_index = desc_table[desc_index].next;
                  if (PREDICT_FALSE
                      (!(buffer_addr =
                         map_guest_mem (vui, desc_table[desc_index].addr,
                                        map_guest_hint_p))))
                    {
                      rxvq->last_used_idx -= hdr->num_buffers - 1;
                      rxvq->last_avail_idx -= hdr->num_buffers - 1;
                      error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
                      goto done;
                    }
                  buffer_len = desc_table[desc_index].len;
                }
              else if (vui->virtio_net_hdr_sz == 12)    //MRG is available
                {
                  //Move from available to used buffer
                  rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id =
                    desc_head;
                  rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len =
                    desc_len;
                  vhost_user_log_dirty_ring (vui, rxvq,
                                             ring[rxvq->last_used_idx &
                                                  qsz_mask]);
                  rxvq->last_avail_idx++;
                  rxvq->last_used_idx++;
                  hdr->num_buffers++;

                  if (PREDICT_FALSE
                      (rxvq->last_avail_idx == rxvq->avail->idx))
                    {
                      //Dequeue queued descriptors for this packet
                      rxvq->last_used_idx -= hdr->num_buffers - 1;
                      rxvq->last_avail_idx -= hdr->num_buffers - 1;
                      error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOBUF;
                      goto done;
                    }

                  desc_table = rxvq->desc;
                  map_guest_hint_p = &map_guest_hint_desc;
                  desc_head = desc_index =
                    rxvq->avail->ring[rxvq->last_avail_idx & qsz_mask];
                  if (PREDICT_FALSE
                      (rxvq->desc[desc_head].flags & VIRTQ_DESC_F_INDIRECT))
                    {
                      //It is seriously unlikely that a driver will put indirect descriptor
                      //after non-indirect descriptor.
                      if (PREDICT_FALSE
                          (rxvq->desc[desc_head].len < sizeof (vring_desc_t)))
                        {
                          error = VHOST_USER_TX_FUNC_ERROR_INDIRECT_OVERFLOW;
                          goto done;
                        }
                      if (PREDICT_FALSE
                          (!(desc_table =
                             map_guest_mem (vui,
                                            rxvq->desc[desc_index].addr,
                                            &map_guest_hint_desc))))
                        {
                          error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
                          goto done;
                        }
                      desc_index = 0;
                      map_guest_hint_p = &map_guest_hint_indirect;
                    }

                  if (PREDICT_FALSE
                      (!(buffer_addr =
                         map_guest_mem (vui, desc_table[desc_index].addr,
                                        map_guest_hint_p))))
                    {
                      error = VHOST_USER_TX_FUNC_ERROR_MMAP_FAIL;
                      goto done;
                    }
                  buffer_len = desc_table[desc_index].len;
                  CLIB_PREFETCH (buffer_addr, CLIB_CACHE_LINE_BYTES, STORE);
                }
              else
                {
                  error = VHOST_USER_TX_FUNC_ERROR_PKT_DROP_NOMRG;
                  goto done;
                }
            }

          u16 bytes_to_copy = bytes_left;
          bytes_to_copy =
            (bytes_to_copy > buffer_len) ? buffer_len : bytes_to_copy;
          clib_memcpy (buffer_addr,
                       vlib_buffer_get_current (current_b0) +
                       current_b0->current_length - bytes_left,
                       bytes_to_copy);

          vhost_user_log_dirty_pages (vui,
                                      desc_table[desc_index].addr +
                                      desc_table[desc_index].len -
                                      bytes_left - bytes_to_copy,
                                      bytes_to_copy);

          bytes_left -= bytes_to_copy;
          buffer_len -= bytes_to_copy;
          buffer_addr += bytes_to_copy;
          desc_len += bytes_to_copy;
        }

      //Move from available to used ring
      rxvq->used->ring[rxvq->last_used_idx & qsz_mask].id = desc_head;
      rxvq->used->ring[rxvq->last_used_idx & qsz_mask].len = desc_len;
      vhost_user_log_dirty_ring (vui, rxvq,
                                 ring[rxvq->last_used_idx & qsz_mask]);

      rxvq->last_avail_idx++;
      rxvq->last_used_idx++;

      n_left--;                 //At the end for error counting when 'goto done' is invoked
    }

done:
  CLIB_MEMORY_BARRIER ();
  rxvq->used->idx = rxvq->last_used_idx;
  vhost_user_log_dirty_ring (vui, rxvq, idx);

  /* interrupt (call) handling */
  if ((rxvq->callfd > -1) && !(rxvq->avail->flags & 1))
    {
      rxvq->n_since_last_int += n_packets - n_left;

      if (rxvq->n_since_last_int > vum->coalesce_frames)
        vhost_user_send_call (vm, rxvq);
    }

done2:

  if (PREDICT_FALSE (vui->lockp != 0))
    *vui->lockp = 0;

  if (PREDICT_FALSE (n_left && error != VHOST_USER_TX_FUNC_ERROR_NONE))
    {
      vlib_error_count (vm, node->node_index, error, n_left);
      vlib_increment_simple_counter
        (vnet_main.interface_main.sw_if_counters
         + VNET_INTERFACE_COUNTER_DROP,
         os_get_cpu_number (), vui->sw_if_index, n_left);
    }

  vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
  return frame->n_vectors;
}

static clib_error_t *
vhost_user_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index,
                                    u32 flags)
{
  vnet_hw_interface_t *hif = vnet_get_hw_interface (vnm, hw_if_index);
  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui =
    vec_elt_at_index (vum->vhost_user_interfaces, hif->dev_instance);

  vui->admin_up = is_up;

  if (is_up)
    vnet_hw_interface_set_flags (vnm, vui->hw_if_index,
                                 VNET_HW_INTERFACE_FLAG_LINK_UP);

  return /* no error */ 0;
}

/* *INDENT-OFF* */
VNET_DEVICE_CLASS (vhost_user_dev_class,static) = {
  .name = "vhost-user",
  .tx_function = vhost_user_intfc_tx,
  .tx_function_n_errors = VHOST_USER_TX_FUNC_N_ERROR,
  .tx_function_error_strings = vhost_user_tx_func_error_strings,
  .format_device_name = format_vhost_user_interface_name,
  .name_renumber = vhost_user_name_renumber,
  .admin_up_down_function = vhost_user_interface_admin_up_down,
  .no_flatten_output_chains = 1,
};

VLIB_DEVICE_TX_FUNCTION_MULTIARCH (vhost_user_dev_class,
                                   vhost_user_intfc_tx)
/* *INDENT-ON* */

static uword
vhost_user_process (vlib_main_t * vm,
                    vlib_node_runtime_t * rt, vlib_frame_t * f)
{
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui;
  struct sockaddr_un sun;
  int sockfd;
  unix_file_t template = { 0 };
  f64 timeout = 3153600000.0 /* 100 years */ ;
  uword *event_data = 0;

  sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
  sun.sun_family = AF_UNIX;
  template.read_function = vhost_user_socket_read;
  template.error_function = vhost_user_socket_error;


  if (sockfd < 0)
    return 0;

  while (1)
    {
      vlib_process_wait_for_event_or_clock (vm, timeout);
      vlib_process_get_events (vm, &event_data);
      vec_reset_length (event_data);

      timeout = 3.0;

      vec_foreach (vui, vum->vhost_user_interfaces)
      {

        if (vui->sock_is_server || !vui->active)
          continue;

        if (vui->unix_fd == -1)
          {
            /* try to connect */

            strncpy (sun.sun_path, (char *) vui->sock_filename,
                     sizeof (sun.sun_path) - 1);

            if (connect
                (sockfd, (struct sockaddr *) &sun,
                 sizeof (struct sockaddr_un)) == 0)
              {
                vui->sock_errno = 0;
                vui->unix_fd = sockfd;
                template.file_descriptor = sockfd;
                vui->unix_file_index = unix_file_add (&unix_main, &template);
                hash_set (vum->vhost_user_interface_index_by_sock_fd, sockfd,
                          vui - vum->vhost_user_interfaces);

                sockfd = socket (AF_UNIX, SOCK_STREAM, 0);
                if (sockfd < 0)
                  return 0;
              }
            else
              {
                vui->sock_errno = errno;
              }
          }
        else
          {
            /* check if socket is alive */
            int error = 0;
            socklen_t len = sizeof (error);
            int retval =
              getsockopt (vui->unix_fd, SOL_SOCKET, SO_ERROR, &error, &len);

            if (retval)
              vhost_user_if_disconnect (vui);
          }
      }
    }
  return 0;
}

/* *INDENT-OFF* */
VLIB_REGISTER_NODE (vhost_user_process_node,static) = {
    .function = vhost_user_process,
    .type = VLIB_NODE_TYPE_PROCESS,
    .name = "vhost-user-process",
};
/* *INDENT-ON* */

int
vhost_user_delete_if (vnet_main_t * vnm, vlib_main_t * vm, u32 sw_if_index)
{
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui;
  uword *p = NULL;
  int rv = 0;

  p = hash_get (vum->vhost_user_interface_index_by_sw_if_index, sw_if_index);
  if (p == 0)
    {
      return VNET_API_ERROR_INVALID_SW_IF_INDEX;
    }
  else
    {
      vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
    }

  // interface is inactive
  vui->active = 0;
  // disconnect interface sockets
  vhost_user_if_disconnect (vui);
  // add to inactive interface list
  vec_add1 (vum->vhost_user_inactive_interfaces_index, p[0]);

  // reset renumbered iface
  if (p[0] < vec_len (vum->show_dev_instance_by_real_dev_instance))
    vum->show_dev_instance_by_real_dev_instance[p[0]] = ~0;

  ethernet_delete_interface (vnm, vui->hw_if_index);
  DBG_SOCK ("deleted (deactivated) vhost-user interface instance %d", p[0]);

  return rv;
}

// init server socket on specified sock_filename
static int
vhost_user_init_server_sock (const char *sock_filename, int *sockfd)
{
  int rv = 0;
  struct sockaddr_un un = { };
  int fd;
  /* create listening socket */
  fd = socket (AF_UNIX, SOCK_STREAM, 0);

  if (fd < 0)
    {
      return VNET_API_ERROR_SYSCALL_ERROR_1;
    }

  un.sun_family = AF_UNIX;
  strncpy ((char *) un.sun_path, (char *) sock_filename,
           sizeof (un.sun_path) - 1);

  /* remove if exists */
  unlink ((char *) sock_filename);

  if (bind (fd, (struct sockaddr *) &un, sizeof (un)) == -1)
    {
      rv = VNET_API_ERROR_SYSCALL_ERROR_2;
      goto error;
    }

  if (listen (fd, 1) == -1)
    {
      rv = VNET_API_ERROR_SYSCALL_ERROR_3;
      goto error;
    }

  unix_file_t template = { 0 };
  template.read_function = vhost_user_socksvr_accept_ready;
  template.file_descriptor = fd;
  unix_file_add (&unix_main, &template);
  *sockfd = fd;
  return rv;

error:
  close (fd);
  return rv;
}

// get new vhost_user_intf_t from inactive interfaces or create new one
static vhost_user_intf_t *
vhost_user_vui_new ()
{
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui = NULL;
  int inactive_cnt = vec_len (vum->vhost_user_inactive_interfaces_index);
  // if there are any inactive ifaces
  if (inactive_cnt > 0)
    {
      // take last
      u32 vui_idx =
        vum->vhost_user_inactive_interfaces_index[inactive_cnt - 1];
      if (vec_len (vum->vhost_user_interfaces) > vui_idx)
        {
          vui = vec_elt_at_index (vum->vhost_user_interfaces, vui_idx);
          DBG_SOCK ("reusing inactive vhost-user interface index %d",
                    vui_idx);
        }
      // "remove" from inactive list
      _vec_len (vum->vhost_user_inactive_interfaces_index) -= 1;
    }

  // vui was not retrieved from inactive ifaces - create new
  if (!vui)
    vec_add2 (vum->vhost_user_interfaces, vui, 1);
  return vui;
}

// create ethernet interface for vhost user intf
static void
vhost_user_create_ethernet (vnet_main_t * vnm, vlib_main_t * vm,
                            vhost_user_intf_t * vui, u8 * hwaddress)
{
  vhost_user_main_t *vum = &vhost_user_main;
  u8 hwaddr[6];
  clib_error_t *error;

  /* create hw and sw interface */
  if (hwaddress)
    {
      clib_memcpy (hwaddr, hwaddress, 6);
    }
  else
    {
      f64 now = vlib_time_now (vm);
      u32 rnd;
      rnd = (u32) (now * 1e6);
      rnd = random_u32 (&rnd);

      clib_memcpy (hwaddr + 2, &rnd, sizeof (rnd));
      hwaddr[0] = 2;
      hwaddr[1] = 0xfe;
    }

  error = ethernet_register_interface
    (vnm,
     vhost_user_dev_class.index,
     vui - vum->vhost_user_interfaces /* device instance */ ,
     hwaddr /* ethernet address */ ,
     &vui->hw_if_index, 0 /* flag change */ );
  if (error)
    clib_error_report (error);

  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, vui->hw_if_index);
  hi->max_l3_packet_bytes[VLIB_RX] = hi->max_l3_packet_bytes[VLIB_TX] = 9000;
}

// initialize vui with specified attributes
static void
vhost_user_vui_init (vnet_main_t * vnm,
                     vhost_user_intf_t * vui, int sockfd,
                     const char *sock_filename,
                     u8 is_server, u64 feature_mask, u32 * sw_if_index)
{
  vnet_sw_interface_t *sw;
  sw = vnet_get_hw_sw_interface (vnm, vui->hw_if_index);
  vlib_thread_main_t *tm = vlib_get_thread_main ();
  int q;

  vui->unix_fd = sockfd;
  vui->sw_if_index = sw->sw_if_index;
  vui->num_vrings = 2;
  vui->sock_is_server = is_server;
  strncpy (vui->sock_filename, sock_filename,
           ARRAY_LEN (vui->sock_filename) - 1);
  vui->sock_errno = 0;
  vui->is_up = 0;
  vui->feature_mask = feature_mask;
  vui->active = 1;
  vui->unix_file_index = ~0;
  vui->log_base_addr = 0;

  for (q = 0; q < 2; q++)
    {
      vui->vrings[q].enabled = 0;
      vui->vrings[q].callfd = -1;
      vui->vrings[q].kickfd = -1;
    }

  vnet_hw_interface_set_flags (vnm, vui->hw_if_index, 0);

  if (sw_if_index)
    *sw_if_index = vui->sw_if_index;

  if (tm->n_vlib_mains > 1)
    {
      vui->lockp = clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES,
                                           CLIB_CACHE_LINE_BYTES);
      memset ((void *) vui->lockp, 0, CLIB_CACHE_LINE_BYTES);
    }
}

// register vui and start polling on it
static void
vhost_user_vui_register (vlib_main_t * vm, vhost_user_intf_t * vui)
{
  vhost_user_main_t *vum = &vhost_user_main;
  int cpu_index;
  vlib_thread_main_t *tm = vlib_get_thread_main ();

  hash_set (vum->vhost_user_interface_index_by_listener_fd, vui->unix_fd,
            vui - vum->vhost_user_interfaces);
  hash_set (vum->vhost_user_interface_index_by_sw_if_index, vui->sw_if_index,
            vui - vum->vhost_user_interfaces);

  /* start polling */
  cpu_index = vum->input_cpu_first_index +
    (vui - vum->vhost_user_interfaces) % vum->input_cpu_count;

  if (tm->n_vlib_mains == 1)
    vlib_node_set_state (vm, vhost_user_input_node.index,
                         VLIB_NODE_STATE_POLLING);
  else
    vlib_node_set_state (vlib_mains[cpu_index], vhost_user_input_node.index,
                         VLIB_NODE_STATE_POLLING);

  /* tell process to start polling for sockets */
  vlib_process_signal_event (vm, vhost_user_process_node.index, 0, 0);
}

int
vhost_user_create_if (vnet_main_t * vnm, vlib_main_t * vm,
                      const char *sock_filename,
                      u8 is_server,
                      u32 * sw_if_index,
                      u64 feature_mask,
                      u8 renumber, u32 custom_dev_instance, u8 * hwaddr)
{
  vhost_user_intf_t *vui = NULL;
  u32 sw_if_idx = ~0;
  int sockfd = -1;
  int rv = 0;

  if (is_server)
    {
      if ((rv = vhost_user_init_server_sock (sock_filename, &sockfd)) != 0)
        {
          return rv;
        }
    }

  vui = vhost_user_vui_new ();
  ASSERT (vui != NULL);

  vhost_user_create_ethernet (vnm, vm, vui, hwaddr);
  vhost_user_vui_init (vnm, vui, sockfd, sock_filename, is_server,
                       feature_mask, &sw_if_idx);

  if (renumber)
    {
      vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
    }

  vhost_user_vui_register (vm, vui);

  if (sw_if_index)
    *sw_if_index = sw_if_idx;

  return rv;
}

int
vhost_user_modify_if (vnet_main_t * vnm, vlib_main_t * vm,
                      const char *sock_filename,
                      u8 is_server,
                      u32 sw_if_index,
                      u64 feature_mask, u8 renumber, u32 custom_dev_instance)
{
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui = NULL;
  u32 sw_if_idx = ~0;
  int sockfd = -1;
  int rv = 0;
  uword *p = NULL;

  p = hash_get (vum->vhost_user_interface_index_by_sw_if_index, sw_if_index);
  if (p == 0)
    {
      return VNET_API_ERROR_INVALID_SW_IF_INDEX;
    }
  else
    {
      vui = vec_elt_at_index (vum->vhost_user_interfaces, p[0]);
    }

  // interface is inactive
  vui->active = 0;
  // disconnect interface sockets
  vhost_user_if_disconnect (vui);

  if (is_server)
    {
      if ((rv = vhost_user_init_server_sock (sock_filename, &sockfd)) != 0)
        {
          return rv;
        }
    }

  vhost_user_vui_init (vnm, vui, sockfd, sock_filename, is_server,
                       feature_mask, &sw_if_idx);

  if (renumber)
    {
      vnet_interface_name_renumber (sw_if_idx, custom_dev_instance);
    }

  vhost_user_vui_register (vm, vui);

  return rv;
}

clib_error_t *
vhost_user_connect_command_fn (vlib_main_t * vm,
                               unformat_input_t * input,
                               vlib_cli_command_t * cmd)
{
  unformat_input_t _line_input, *line_input = &_line_input;
  u8 *sock_filename = NULL;
  u32 sw_if_index;
  u8 is_server = 0;
  u64 feature_mask = (u64) ~ 0;
  u8 renumber = 0;
  u32 custom_dev_instance = ~0;
  u8 hwaddr[6];
  u8 *hw = NULL;

  /* Get a line of input. */
  if (!unformat_user (input, unformat_line_input, line_input))
    return 0;

  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
    {
      if (unformat (line_input, "socket %s", &sock_filename))
        ;
      else if (unformat (line_input, "server"))
        is_server = 1;
      else if (unformat (line_input, "feature-mask 0x%llx", &feature_mask))
        ;
      else
        if (unformat
            (line_input, "hwaddr %U", unformat_ethernet_address, hwaddr))
        hw = hwaddr;
      else if (unformat (line_input, "renumber %d", &custom_dev_instance))
        {
          renumber = 1;
        }
      else
        return clib_error_return (0, "unknown input `%U'",
                                  format_unformat_error, input);
    }
  unformat_free (line_input);

  vnet_main_t *vnm = vnet_get_main ();

  int rv;
  if ((rv = vhost_user_create_if (vnm, vm, (char *) sock_filename,
                                  is_server, &sw_if_index, feature_mask,
                                  renumber, custom_dev_instance, hw)))
    {
      vec_free (sock_filename);
      return clib_error_return (0, "vhost_user_create_if returned %d", rv);
    }

  vec_free (sock_filename);
  vlib_cli_output (vm, "%U\n", format_vnet_sw_if_index_name, vnet_get_main (),
                   sw_if_index);
  return 0;
}

clib_error_t *
vhost_user_delete_command_fn (vlib_main_t * vm,
                              unformat_input_t * input,
                              vlib_cli_command_t * cmd)
{
  unformat_input_t _line_input, *line_input = &_line_input;
  u32 sw_if_index = ~0;

  /* Get a line of input. */
  if (!unformat_user (input, unformat_line_input, line_input))
    return 0;

  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
    {
      if (unformat (line_input, "sw_if_index %d", &sw_if_index))
        ;
      else
        return clib_error_return (0, "unknown input `%U'",
                                  format_unformat_error, input);
    }
  unformat_free (line_input);

  vnet_main_t *vnm = vnet_get_main ();

  vhost_user_delete_if (vnm, vm, sw_if_index);

  return 0;
}

int
vhost_user_dump_ifs (vnet_main_t * vnm, vlib_main_t * vm,
                     vhost_user_intf_details_t ** out_vuids)
{
  int rv = 0;
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui;
  vhost_user_intf_details_t *r_vuids = NULL;
  vhost_user_intf_details_t *vuid = NULL;
  u32 *hw_if_indices = 0;
  vnet_hw_interface_t *hi;
  u8 *s = NULL;
  int i;

  if (!out_vuids)
    return -1;

  vec_foreach (vui, vum->vhost_user_interfaces)
  {
    if (vui->active)
      vec_add1 (hw_if_indices, vui->hw_if_index);
  }

  for (i = 0; i < vec_len (hw_if_indices); i++)
    {
      hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
      vui = vec_elt_at_index (vum->vhost_user_interfaces, hi->dev_instance);

      vec_add2 (r_vuids, vuid, 1);
      vuid->sw_if_index = vui->sw_if_index;
      vuid->virtio_net_hdr_sz = vui->virtio_net_hdr_sz;
      vuid->features = vui->features;
      vuid->is_server = vui->sock_is_server;
      vuid->num_regions = vui->nregions;
      vuid->sock_errno = vui->sock_errno;
      strncpy ((char *) vuid->sock_filename, (char *) vui->sock_filename,
               ARRAY_LEN (vuid->sock_filename) - 1);

      s = format (s, "%v%c", hi->name, 0);

      strncpy ((char *) vuid->if_name, (char *) s,
               ARRAY_LEN (vuid->if_name) - 1);
      _vec_len (s) = 0;
    }

  vec_free (s);
  vec_free (hw_if_indices);

  *out_vuids = r_vuids;

  return rv;
}

clib_error_t *
show_vhost_user_command_fn (vlib_main_t * vm,
                            unformat_input_t * input,
                            vlib_cli_command_t * cmd)
{
  clib_error_t *error = 0;
  vnet_main_t *vnm = vnet_get_main ();
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui;
  u32 hw_if_index, *hw_if_indices = 0;
  vnet_hw_interface_t *hi;
  int i, j, q;
  int show_descr = 0;
  struct feat_struct
  {
    u8 bit;
    char *str;
  };
  struct feat_struct *feat_entry;

  static struct feat_struct feat_array[] = {
#define _(s,b) { .str = #s, .bit = b, },
    foreach_virtio_net_feature
#undef _
    {.str = NULL}
  };

  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
    {
      if (unformat
          (input, "%U", unformat_vnet_hw_interface, vnm, &hw_if_index))
        {
          vec_add1 (hw_if_indices, hw_if_index);
          vlib_cli_output (vm, "add %d", hw_if_index);
        }
      else if (unformat (input, "descriptors") || unformat (input, "desc"))
        show_descr = 1;
      else
        {
          error = clib_error_return (0, "unknown input `%U'",
                                     format_unformat_error, input);
          goto done;
        }
    }
  if (vec_len (hw_if_indices) == 0)
    {
      vec_foreach (vui, vum->vhost_user_interfaces)
      {
        if (vui->active)
          vec_add1 (hw_if_indices, vui->hw_if_index);
      }
    }
  vlib_cli_output (vm, "Virtio vhost-user interfaces");
  vlib_cli_output (vm, "Global:\n  coalesce frames %d time %e\n\n",
                   vum->coalesce_frames, vum->coalesce_time);

  for (i = 0; i < vec_len (hw_if_indices); i++)
    {
      hi = vnet_get_hw_interface (vnm, hw_if_indices[i]);
      vui = vec_elt_at_index (vum->vhost_user_interfaces, hi->dev_instance);
      vlib_cli_output (vm, "Interface: %s (ifindex %d)",
                       hi->name, hw_if_indices[i]);

      vlib_cli_output (vm, "virtio_net_hdr_sz %d\n features (0x%llx): \n",
                       vui->virtio_net_hdr_sz, vui->features);

      feat_entry = (struct feat_struct *) &feat_array;
      while (feat_entry->str)
        {
          if (vui->features & (1 << feat_entry->bit))
            vlib_cli_output (vm, "   %s (%d)", feat_entry->str,
                             feat_entry->bit);
          feat_entry++;
        }

      vlib_cli_output (vm, "\n");


      vlib_cli_output (vm, " socket filename %s type %s errno \"%s\"\n\n",
                       vui->sock_filename,
                       vui->sock_is_server ? "server" : "client",
                       strerror (vui->sock_errno));

      vlib_cli_output (vm, " Memory regions (total %d)\n", vui->nregions);

      if (vui->nregions)
        {
          vlib_cli_output (vm,
                           " region fd    guest_phys_addr    memory_size        userspace_addr     mmap_offset        mmap_addr\n");
          vlib_cli_output (vm,
                           " ====== ===== ================== ================== ================== ================== ==================\n");
        }
      for (j = 0; j < vui->nregions; j++)
        {
          vlib_cli_output (vm,
                           "  %d     %-5d 0x%016lx 0x%016lx 0x%016lx 0x%016lx 0x%016lx\n",
                           j, vui->region_mmap_fd[j],
                           vui->regions[j].guest_phys_addr,
                           vui->regions[j].memory_size,
                           vui->regions[j].userspace_addr,
                           vui->regions[j].mmap_offset,
                           pointer_to_uword (vui->region_mmap_addr[j]));
        }
      for (q = 0; q < vui->num_vrings; q++)
        {
          vlib_cli_output (vm, "\n Virtqueue %d\n", q);

          vlib_cli_output (vm,
                           "  qsz %d last_avail_idx %d last_used_idx %d\n",
                           vui->vrings[q].qsz, vui->vrings[q].last_avail_idx,
                           vui->vrings[q].last_used_idx);

          if (vui->vrings[q].avail && vui->vrings[q].used)
            vlib_cli_output (vm,
                             "  avail.flags %x avail.idx %d used.flags %x used.idx %d\n",
                             vui->vrings[q].avail->flags,
                             vui->vrings[q].avail->idx,
                             vui->vrings[q].used->flags,
                             vui->vrings[q].used->idx);

          vlib_cli_output (vm, "  kickfd %d callfd %d errfd %d\n",
                           vui->vrings[q].kickfd,
                           vui->vrings[q].callfd, vui->vrings[q].errfd);

          if (show_descr)
            {
              vlib_cli_output (vm, "\n  descriptor table:\n");
              vlib_cli_output (vm,
                               "   id          addr         len  flags  next      user_addr\n");
              vlib_cli_output (vm,
                               "  ===== ================== ===== ====== ===== ==================\n");
              for (j = 0; j < vui->vrings[q].qsz; j++)
                {
                  u32 mem_hint = 0;
                  vlib_cli_output (vm,
                                   "  %-5d 0x%016lx %-5d 0x%04x %-5d 0x%016lx\n",
                                   j, vui->vrings[q].desc[j].addr,
                                   vui->vrings[q].desc[j].len,
                                   vui->vrings[q].desc[j].flags,
                                   vui->vrings[q].desc[j].next,
                                   pointer_to_uword (map_guest_mem
                                                     (vui,
                                                      vui->vrings[q].desc[j].
                                                      addr, &mem_hint)));
                }
            }
        }
      vlib_cli_output (vm, "\n");
    }
done:
  vec_free (hw_if_indices);
  return error;
}

/*
 * CLI functions
 */

/* *INDENT-OFF* */
VLIB_CLI_COMMAND (vhost_user_connect_command, static) = {
    .path = "create vhost-user",
    .short_help = "create vhost-user socket <socket-filename> [server] [feature-mask <hex>] [renumber <dev_instance>]",
    .function = vhost_user_connect_command_fn,
};

VLIB_CLI_COMMAND (vhost_user_delete_command, static) = {
    .path = "delete vhost-user",
    .short_help = "delete vhost-user sw_if_index <nn>",
    .function = vhost_user_delete_command_fn,
};

VLIB_CLI_COMMAND (show_vhost_user_command, static) = {
    .path = "show vhost-user",
    .short_help = "show vhost-user interface",
    .function = show_vhost_user_command_fn,
};
/* *INDENT-ON* */

static clib_error_t *
vhost_user_config (vlib_main_t * vm, unformat_input_t * input)
{
  vhost_user_main_t *vum = &vhost_user_main;

  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
    {
      if (unformat (input, "coalesce-frames %d", &vum->coalesce_frames))
        ;
      else if (unformat (input, "coalesce-time %f", &vum->coalesce_time))
        ;
      else if (unformat (input, "dont-dump-memory"))
        vum->dont_dump_vhost_user_memory = 1;
      else
        return clib_error_return (0, "unknown input `%U'",
                                  format_unformat_error, input);
    }

  return 0;
}

/* vhost-user { ... } configuration. */
VLIB_CONFIG_FUNCTION (vhost_user_config, "vhost-user");

void
vhost_user_unmap_all (void)
{
  vhost_user_main_t *vum = &vhost_user_main;
  vhost_user_intf_t *vui;

  if (vum->dont_dump_vhost_user_memory)
    {
      vec_foreach (vui, vum->vhost_user_interfaces)
      {
        unmap_all_mem_regions (vui);
      }
    }
}

/*
 * fd.io coding-style-patch-verification: ON
 *
 * Local Variables:
 * eval: (c-set-style "gnu")
 * End:
 */