FD.io VPP  v19.04.4-rc0-5-ge88582fac
Vector Packet Processing
vhost_user_inline.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef __VIRTIO_VHOST_USER_INLINE_H__
16 #define __VIRTIO_VHOST_USER_INLINE_H__
17 /* vhost-user inline functions */
18 #include <vppinfra/elog.h>
19 
22 {
23  int i = *hint;
24  if (PREDICT_TRUE ((vui->regions[i].guest_phys_addr <= addr) &&
25  ((vui->regions[i].guest_phys_addr +
26  vui->regions[i].memory_size) > addr)))
27  {
28  return (void *) (vui->region_mmap_addr[i] + addr -
29  vui->regions[i].guest_phys_addr);
30  }
31 #if __SSE4_2__
32  __m128i rl, rh, al, ah, r;
33  al = _mm_set1_epi64x (addr + 1);
34  ah = _mm_set1_epi64x (addr);
35 
36  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[0]);
37  rl = _mm_cmpgt_epi64 (al, rl);
38  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[0]);
39  rh = _mm_cmpgt_epi64 (rh, ah);
40  r = _mm_and_si128 (rl, rh);
41 
42  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[2]);
43  rl = _mm_cmpgt_epi64 (al, rl);
44  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[2]);
45  rh = _mm_cmpgt_epi64 (rh, ah);
46  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x22);
47 
48  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[4]);
49  rl = _mm_cmpgt_epi64 (al, rl);
50  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[4]);
51  rh = _mm_cmpgt_epi64 (rh, ah);
52  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x44);
53 
54  rl = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_lo[6]);
55  rl = _mm_cmpgt_epi64 (al, rl);
56  rh = _mm_loadu_si128 ((__m128i *) & vui->region_guest_addr_hi[6]);
57  rh = _mm_cmpgt_epi64 (rh, ah);
58  r = _mm_blend_epi16 (r, _mm_and_si128 (rl, rh), 0x88);
59 
60  r = _mm_shuffle_epi8 (r, _mm_set_epi64x (0, 0x0e060c040a020800));
61  i = count_trailing_zeros (_mm_movemask_epi8 (r) |
63 
64  if (i < vui->nregions)
65  {
66  *hint = i;
67  return (void *) (vui->region_mmap_addr[i] + addr -
68  vui->regions[i].guest_phys_addr);
69  }
70 #elif __aarch64__ && __ARM_NEON
71  uint64x2_t al, ah, rl, rh, r;
72  uint32_t u32 = 0;
73 
74  al = vdupq_n_u64 (addr + 1);
75  ah = vdupq_n_u64 (addr);
76 
77  /*First Iteration */
78  rl = vld1q_u64 (&vui->region_guest_addr_lo[0]);
79  rl = vcgtq_u64 (al, rl);
80  rh = vld1q_u64 (&vui->region_guest_addr_hi[0]);
81  rh = vcgtq_u64 (rh, ah);
82  r = vandq_u64 (rl, rh);
83  u32 |= (vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 0) & 0x1);
84  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 8) & 0x1) << 1);
85 
86  if (u32)
87  {
88  i = count_trailing_zeros (u32);
89  goto vhost_map_guest_mem_done;
90  }
91 
92  /*Second Iteration */
93  rl = vld1q_u64 (&vui->region_guest_addr_lo[2]);
94  rl = vcgtq_u64 (al, rl);
95  rh = vld1q_u64 (&vui->region_guest_addr_hi[2]);
96  rh = vcgtq_u64 (rh, ah);
97  r = vandq_u64 (rl, rh);
98  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 0) & 0x1) << 2);
99  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 8) & 0x1) << 3);
100 
101  if (u32)
102  {
103  i = count_trailing_zeros (u32);
104  goto vhost_map_guest_mem_done;
105  }
106 
107  /*Third Iteration */
108  rl = vld1q_u64 (&vui->region_guest_addr_lo[4]);
109  rl = vcgtq_u64 (al, rl);
110  rh = vld1q_u64 (&vui->region_guest_addr_hi[4]);
111  rh = vcgtq_u64 (rh, ah);
112  r = vandq_u64 (rl, rh);
113  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 0) & 0x1) << 6);
114  u32 |= ((vgetq_lane_u8 (vreinterpretq_u8_u64 (r), 8) & 0x1) << 7);
115 
117 
118 vhost_map_guest_mem_done:
119  if (i < vui->nregions)
120  {
121  *hint = i;
122  return (void *) (vui->region_mmap_addr[i] + addr -
123  vui->regions[i].guest_phys_addr);
124  }
125 #else
126  for (i = 0; i < vui->nregions; i++)
127  {
128  if ((vui->regions[i].guest_phys_addr <= addr) &&
129  ((vui->regions[i].guest_phys_addr + vui->regions[i].memory_size) >
130  addr))
131  {
132  *hint = i;
133  return (void *) (vui->region_mmap_addr[i] + addr -
134  vui->regions[i].guest_phys_addr);
135  }
136  }
137 #endif
138  /* *INDENT-OFF* */
139  ELOG_TYPE_DECLARE (el) =
140  {
141  .format = "failed to map guest mem addr %lx",
142  .format_args = "i8",
143  };
144  /* *INDENT-ON* */
145  struct
146  {
147  uword addr;
148  } *ed;
150  ed->addr = addr;
151  *hint = 0;
152  return 0;
153 }
154 
157 {
158  int i;
159  for (i = 0; i < vui->nregions; i++)
160  {
161  if ((vui->regions[i].userspace_addr <= addr) &&
162  ((vui->regions[i].userspace_addr + vui->regions[i].memory_size) >
163  addr))
164  {
165  return (void *) (vui->region_mmap_addr[i] + addr -
166  vui->regions[i].userspace_addr);
167  }
168  }
169  return 0;
170 }
171 
172 #define VHOST_LOG_PAGE 0x1000
173 
176  u64 addr, u64 len, u8 is_host_address)
177 {
178  if (PREDICT_TRUE (vui->log_base_addr == 0
179  || !(vui->features & (1 << FEAT_VHOST_F_LOG_ALL))))
180  {
181  return;
182  }
183  if (is_host_address)
184  {
185  addr = pointer_to_uword (map_user_mem (vui, (uword) addr));
186  }
187  if (PREDICT_FALSE ((addr + len - 1) / VHOST_LOG_PAGE / 8 >= vui->log_size))
188  {
189  vu_log_debug (vui, "vhost_user_log_dirty_pages(): out of range\n");
190  return;
191  }
192 
194  u64 page = addr / VHOST_LOG_PAGE;
195  while (page * VHOST_LOG_PAGE < addr + len)
196  {
197  ((u8 *) vui->log_base_addr)[page / 8] |= 1 << page % 8;
198  page++;
199  }
200 }
201 
202 
203 #define vhost_user_log_dirty_ring(vui, vq, member) \
204  if (PREDICT_FALSE(vq->log_used)) { \
205  vhost_user_log_dirty_pages_2(vui, vq->log_guest_addr + STRUCT_OFFSET_OF(vring_used_t, member), \
206  sizeof(vq->used->member), 0); \
207  }
208 
210 format_vhost_trace (u8 * s, va_list * va)
211 {
212  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
213  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
216  vhost_trace_t *t = va_arg (*va, vhost_trace_t *);
218  t->device_index);
219 
221 
222  u32 indent = format_get_indent (s);
223 
224  s = format (s, "%U %U queue %d\n", format_white_space, indent,
225  format_vnet_sw_interface_name, vnm, sw, t->qid);
226 
227  s = format (s, "%U virtio flags:\n", format_white_space, indent);
228 #define _(n,i,st) \
229  if (t->virtio_ring_flags & (1 << VIRTIO_TRACE_F_##n)) \
230  s = format (s, "%U %s %s\n", format_white_space, indent, #n, st);
232 #undef _
233  s = format (s, "%U virtio_net_hdr first_desc_len %u\n",
234  format_white_space, indent, t->first_desc_len);
235 
236  s = format (s, "%U flags 0x%02x gso_type %u\n",
237  format_white_space, indent,
238  t->hdr.hdr.flags, t->hdr.hdr.gso_type);
239 
240  if (vui->virtio_net_hdr_sz == 12)
241  s = format (s, "%U num_buff %u",
242  format_white_space, indent, t->hdr.num_buffers);
243 
244  return s;
245 }
246 
249 {
251  u64 x = 1;
252  int fd = UNIX_GET_FD (vq->callfd_idx);
253  int rv;
254 
255  rv = write (fd, &x, sizeof (x));
256  if (rv <= 0)
257  {
259  ("Error: Could not write to unix socket for callfd %d", fd);
260  return;
261  }
262 
263  vq->n_since_last_int = 0;
264  vq->int_deadline = vlib_time_now (vm) + vum->coalesce_time;
265 }
266 
269 {
270  return vui->admin_up && vui->is_ready;
271 }
272 
273 #endif
274 
275 /*
276  * fd.io coding-style-patch-verification: ON
277  *
278  * Local Variables:
279  * eval: (c-set-style "gnu")
280  * End:
281  */
u32 nregions
Definition: vhost_user.h:114
vlib_main_t vlib_global_main
Definition: main.c:1924
#define CLIB_UNUSED(x)
Definition: clib.h:82
virtio_net_hdr_mrg_rxbuf_t hdr
Length of the first data descriptor.
Definition: vhost_user.h:322
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
u64 region_guest_addr_hi[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost_user.h:291
#define PREDICT_TRUE(x)
Definition: clib.h:112
unsigned long u64
Definition: types.h:89
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:255
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
vhost_vring_addr_t addr
Definition: vhost_user.h:121
unsigned char u8
Definition: types.h:56
#define count_trailing_zeros(x)
Definition: clib.h:139
#define vu_log_debug(dev, f,...)
Definition: vhost_user.h:48
#define static_always_inline
Definition: clib.h:99
#define UNIX_GET_FD(unixfd_idx)
Definition: vhost_user.h:68
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
void * log_base_addr
Definition: vhost_user.h:301
static_always_inline void * map_guest_mem(vhost_user_intf_t *vui, uword addr, u32 *hint)
static_always_inline u8 * format_vhost_trace(u8 *s, va_list *va)
unsigned int u32
Definition: types.h:88
format_function_t format_vnet_sw_interface_name
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
#define VHOST_LOG_PAGE
#define ELOG_DATA(em, f)
Definition: elog.h:484
#define PREDICT_FALSE(x)
Definition: clib.h:111
vhost_user_main_t vhost_user_main
Definition: vhost_user.c:56
vhost_user_memory_region_t regions[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost_user.h:288
static_always_inline u8 vui_is_link_up(vhost_user_intf_t *vui)
u8 len
Definition: ip_types.api:49
The fine-grained event logger allows lightweight, thread-safe event logging at minimum cost...
u16 device_index
The interface queue index (Not the virtio vring idx)
Definition: vhost_user.h:319
vhost_user_intf_t * vhost_user_interfaces
Definition: vhost_user.h:346
vlib_main_t * vm
Definition: buffer.c:312
elog_main_t elog_main
Definition: main.h:172
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:442
u16 first_desc_len
Runtime queue flags.
Definition: vhost_user.h:321
static_always_inline void vhost_user_send_call(vlib_main_t *vm, vhost_user_vring_t *vq)
static uword pointer_to_uword(const void *p)
Definition: types.h:131
void * region_mmap_addr[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost_user.h:289
u64 uword
Definition: types.h:112
#define clib_unix_warning(format, args...)
Definition: error.h:68
#define VHOST_MEMORY_MAX_NREGIONS
Definition: vhost_user.h:19
static_always_inline void vhost_user_log_dirty_pages_2(vhost_user_intf_t *vui, u64 addr, u64 len, u8 is_host_address)
u64 region_guest_addr_lo[VHOST_MEMORY_MAX_NREGIONS]
Definition: vhost_user.h:290
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:115
static_always_inline void * map_user_mem(vhost_user_intf_t *vui, uword addr)