FD.io VPP  v19.04.4-rc0-5-ge88582fac
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2016 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <fcntl.h>
21 
22 #include <vlib/vlib.h>
23 #include <vlib/unix/unix.h>
24 #include <vnet/ethernet/ethernet.h>
25 #include <vnet/ip/ip4_packet.h>
26 #include <vnet/ip/ip6_packet.h>
28 
29 #define foreach_virtio_tx_func_error \
30 _(NO_FREE_SLOTS, "no free tx slots") \
31 _(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \
32 _(PENDING_MSGS, "pending msgs in tx ring") \
33 _(NO_TX_QUEUES, "no tx queues")
34 
35 typedef enum
36 {
37 #define _(f,s) VIRTIO_TX_ERROR_##f,
39 #undef _
42 
43 static char *virtio_tx_func_error_strings[] = {
44 #define _(n,s) s,
46 #undef _
47 };
48 
49 #ifndef CLIB_MARCH_VARIANT
50 u8 *
51 format_virtio_device_name (u8 * s, va_list * args)
52 {
53  u32 dev_instance = va_arg (*args, u32);
55  virtio_if_t *vif = pool_elt_at_index (mm->interfaces, dev_instance);
56 
57  if (vif->type == VIRTIO_IF_TYPE_TAP)
58  s = format (s, "tap%u", vif->id);
59  else if (vif->type == VIRTIO_IF_TYPE_PCI)
60  s = format (s, "virtio-%x/%x/%x/%x", vif->pci_addr.domain,
61  vif->pci_addr.bus, vif->pci_addr.slot,
62  vif->pci_addr.function);
63  else
64  s = format (s, "virtio-%lu", vif->dev_instance);
65 
66  return s;
67 }
68 #endif /* CLIB_MARCH_VARIANT */
69 
70 static u8 *
71 format_virtio_device (u8 * s, va_list * args)
72 {
73  u32 dev_instance = va_arg (*args, u32);
74  int verbose = va_arg (*args, int);
75  u32 indent = format_get_indent (s);
76 
77  s = format (s, "VIRTIO interface");
78  if (verbose)
79  {
80  s = format (s, "\n%U instance %u", format_white_space, indent + 2,
81  dev_instance);
82  }
83  return s;
84 }
85 
86 static u8 *
87 format_virtio_tx_trace (u8 * s, va_list * args)
88 {
89  s = format (s, "Unimplemented...");
90  return s;
91 }
92 
95 {
96  u16 used = vring->desc_in_use;
97  u16 sz = vring->size;
98  u16 mask = sz - 1;
99  u16 last = vring->last_used_idx;
100  u16 n_left = vring->used->idx - last;
101 
102  if (n_left == 0)
103  return;
104 
105  while (n_left)
106  {
107  struct vring_used_elem *e = &vring->used->ring[last & mask];
108  u16 slot, n_buffers;
109  slot = n_buffers = e->id;
110 
111  while (e->id == n_buffers)
112  {
113  n_left--;
114  last++;
115  n_buffers++;
116  if (n_left == 0)
117  break;
118  e = &vring->used->ring[last & mask];
119  }
120  vlib_buffer_free_from_ring (vm, vring->buffers, slot,
121  sz, (n_buffers - slot));
122  used -= (n_buffers - slot);
123 
124  if (n_left > 0)
125  {
126  slot = e->id;
127 
128  vlib_buffer_free (vm, &vring->buffers[slot], 1);
129  used--;
130  last++;
131  n_left--;
132  }
133  }
134  vring->desc_in_use = used;
135  vring->last_used_idx = last;
136 }
137 
140  virtio_vring_t * vring, u32 bi, u16 avail, u16 next,
141  u16 mask, int do_gso)
142 {
143  u16 n_added = 0;
144  int hdr_sz = vif->virtio_net_hdr_sz;
145  struct vring_desc *d;
146  d = &vring->desc[next];
147  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
148  struct virtio_net_hdr_v1 *hdr = vlib_buffer_get_current (b) - hdr_sz;
149 
150  clib_memset (hdr, 0, hdr_sz);
151  if (do_gso && (b->flags & VNET_BUFFER_F_GSO))
152  {
153  if (b->flags & VNET_BUFFER_F_IS_IP4)
154  {
155  hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
156  hdr->gso_size = vnet_buffer2 (b)->gso_size;
157  hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
158  hdr->csum_start = vnet_buffer (b)->l4_hdr_offset; // 0x22;
159  hdr->csum_offset = 0x10;
160  }
161  else
162  {
163  hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
164  hdr->gso_size = vnet_buffer2 (b)->gso_size;
165  hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
166  hdr->csum_start = vnet_buffer (b)->l4_hdr_offset; // 0x36;
167  hdr->csum_offset = 0x10;
168  }
169  }
170 
171  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
172  {
173  d->addr =
175  b) :
177  d->len = b->current_length + hdr_sz;
178  d->flags = 0;
179  }
180  else
181  {
182  /*
183  * We are using single vlib_buffer_t for indirect descriptor(s)
184  * chain. Single descriptor is 16 bytes and vlib_buffer_t
185  * has 2048 bytes space. So maximum long chain can have 128
186  * (=2048/16) indirect descriptors.
187  * It can easily support 65535 bytes of Jumbo frames with
188  * each data buffer size of 512 bytes minimum.
189  */
190  u32 indirect_buffer = 0;
191  if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
192  return n_added;
193 
194  vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
195  indirect_desc->current_data = 0;
196  indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
197  indirect_desc->next_buffer = bi;
198  bi = indirect_buffer;
199 
200  struct vring_desc *id =
201  (struct vring_desc *) vlib_buffer_get_current (indirect_desc);
202  u32 count = 1;
203  if (vif->type == VIRTIO_IF_TYPE_PCI)
204  {
205  d->addr = vlib_physmem_get_pa (vm, id);
206  id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
207 
208  /*
209  * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
210  * should be presented in separate descriptor and data will start
211  * from next descriptor.
212  */
213  if (PREDICT_TRUE
214  (vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)))
215  id->len = b->current_length + hdr_sz;
216  else
217  {
218  id->len = hdr_sz;
219  id->flags = VRING_DESC_F_NEXT;
220  id->next = count;
221  count++;
222  id++;
223  id->addr = vlib_buffer_get_current_pa (vm, b);
224  id->len = b->current_length;
225  }
226  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
227  {
228  id->flags = VRING_DESC_F_NEXT;
229  id->next = count;
230  count++;
231  id++;
232  b = vlib_get_buffer (vm, b->next_buffer);
233  id->addr = vlib_buffer_get_current_pa (vm, b);
234  id->len = b->current_length;
235  }
236  }
237  else /* VIRTIO_IF_TYPE_TAP */
238  {
239  d->addr = pointer_to_uword (id);
240  /* first buffer in chain */
241  id->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz;
242  id->len = b->current_length + hdr_sz;
243 
244  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
245  {
246  id->flags = VRING_DESC_F_NEXT;
247  id->next = count;
248  count++;
249  id++;
250  b = vlib_get_buffer (vm, b->next_buffer);
251  id->addr = pointer_to_uword (vlib_buffer_get_current (b));
252  id->len = b->current_length;
253  }
254  }
255  id->flags = 0;
256  id->next = 0;
257  d->len = count * sizeof (struct vring_desc);
258  d->flags = VRING_DESC_F_INDIRECT;
259  }
260  vring->buffers[next] = bi;
261  vring->avail->ring[avail & mask] = next;
262  n_added++;
263  return n_added;
264 }
265 
268  vlib_frame_t * frame, virtio_if_t * vif,
269  int do_gso)
270 {
271  u16 n_left = frame->n_vectors;
272  virtio_vring_t *vring;
273  u16 qid = vm->thread_index % vif->num_txqs;
274  vring = vec_elt_at_index (vif->txq_vrings, qid);
275  u16 used, next, avail;
276  u16 sz = vring->size;
277  u16 mask = sz - 1;
278  u32 *buffers = vlib_frame_vector_args (frame);
279 
281 
282  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 &&
283  (vring->last_kick_avail_idx != vring->avail->idx))
284  virtio_kick (vm, vring, vif);
285 
286  /* free consumed buffers */
287  virtio_free_used_device_desc (vm, vring);
288 
289  used = vring->desc_in_use;
290  next = vring->desc_next;
291  avail = vring->avail->idx;
292 
293  while (n_left && used < sz)
294  {
295  u16 n_added = 0;
296  n_added =
297  add_buffer_to_slot (vm, vif, vring, buffers[0], avail, next, mask,
298  do_gso);
299  if (!n_added)
300  break;
301  avail += n_added;
302  next = (next + n_added) & mask;
303  used += n_added;
304  buffers++;
305  n_left--;
306  }
307 
308  if (n_left != frame->n_vectors)
309  {
311  vring->avail->idx = avail;
312  vring->desc_next = next;
313  vring->desc_in_use = used;
314  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
315  virtio_kick (vm, vring, vif);
316  }
317 
318  if (n_left)
319  {
320  vlib_error_count (vm, node->node_index, VIRTIO_TX_ERROR_NO_FREE_SLOTS,
321  n_left);
322  vlib_buffer_free (vm, buffers, n_left);
323  }
324 
326 
327  return frame->n_vectors - n_left;
328 }
329 
331  vlib_node_runtime_t * node,
332  vlib_frame_t * frame)
333 {
334  virtio_main_t *nm = &virtio_main;
335  vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
337  vnet_main_t *vnm = vnet_get_main ();
338 
339  if (vnm->interface_main.gso_interface_count > 0)
340  return virtio_interface_tx_inline (vm, node, frame, vif, 1 /* do_gso */ );
341  else
342  return virtio_interface_tx_inline (vm, node, frame, vif,
343  0 /* no do_gso */ );
344 }
345 
346 static void
348  u32 node_index)
349 {
350  virtio_main_t *apm = &virtio_main;
351  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
353 
354  /* Shut off redirection */
355  if (node_index == ~0)
356  {
357  vif->per_interface_next_index = node_index;
358  return;
359  }
360 
363  node_index);
364 }
365 
366 static void
368 {
369  /* Nothing for now */
370 }
371 
372 static clib_error_t *
375 {
376  virtio_main_t *mm = &virtio_main;
377  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
379  virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid);
380 
381  if (vif->type == VIRTIO_IF_TYPE_PCI && !(vif->support_int_mode))
382  {
383  vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT;
384  return clib_error_return (0, "interrupt mode is not supported");
385  }
386 
388  vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT;
389  else
390  vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT;
391 
392  return 0;
393 }
394 
395 static clib_error_t *
397 {
398  virtio_main_t *mm = &virtio_main;
399  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
401 
403  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
404  else
405  vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
406 
407  return 0;
408 }
409 
410 static clib_error_t *
412  u32 hw_if_index,
413  struct vnet_sw_interface_t *st, int is_add)
414 {
415  /* Nothing for now */
416  return 0;
417 }
418 
419 /* *INDENT-OFF* */
421  .name = "virtio",
422  .format_device_name = format_virtio_device_name,
423  .format_device = format_virtio_device,
424  .format_tx_trace = format_virtio_tx_trace,
425  .tx_function_n_errors = VIRTIO_TX_N_ERROR,
426  .tx_function_error_strings = virtio_tx_func_error_strings,
427  .rx_redirect_to_node = virtio_set_interface_next_node,
428  .clear_counters = virtio_clear_hw_interface_counters,
429  .admin_up_down_function = virtio_interface_admin_up_down,
430  .subif_add_del_function = virtio_subif_add_del_function,
431  .rx_mode_change_function = virtio_interface_rx_mode_change,
432 };
433 /* *INDENT-ON* */
434 
435 /*
436  * fd.io coding-style-patch-verification: ON
437  *
438  * Local Variables:
439  * eval: (c-set-style "gnu")
440  * End:
441  */
u32 per_interface_next_index
Definition: virtio.h:150
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
struct vring_used * used
Definition: virtio.h:105
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:427
vlib_node_registration_t virtio_input_node
(constructor) VLIB_REGISTER_NODE (virtio_input_node)
Definition: node.c:384
virtio_if_t * interfaces
Definition: virtio.h:191
u32 flags
Definition: vhost_user.h:115
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
#define foreach_virtio_tx_func_error
Definition: device.c:29
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
#define vnet_buffer2(b)
Definition: buffer.h:428
vnet_interface_main_t interface_main
Definition: vnet.h:56
#define PREDICT_TRUE(x)
Definition: clib.h:112
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:118
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
u32 dev_instance
Definition: virtio.h:139
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:197
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
static u32 format_get_indent(u8 *s)
Definition: format.h:72
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
u16 domain
Definition: virtio.h:126
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1092
unsigned char u8
Definition: types.h:56
static char * virtio_tx_func_error_strings[]
Definition: device.c:43
virtio_tx_func_error_t
Definition: device.c:35
vnet_hw_interface_rx_mode
Definition: interface.h:52
struct vring_avail * avail
Definition: virtio.h:106
u64 features
Definition: virtio.h:163
#define static_always_inline
Definition: clib.h:99
static_always_inline uword virtio_interface_tx_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, virtio_if_t *vif, int do_gso)
Definition: device.c:267
VNET_DEVICE_CLASS_TX_FN() virtio_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: device.c:330
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
u8 bus
Definition: virtio.h:127
u8 support_int_mode
Definition: virtio.h:167
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
clib_spinlock_t lockp
Definition: virtio.h:107
static void virtio_clear_hw_interface_counters(u32 instance)
Definition: device.c:367
#define clib_error_return(e, args...)
Definition: error.h:99
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:287
unsigned int u32
Definition: types.h:88
u32 id
Definition: virtio.h:147
static clib_error_t * virtio_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
Definition: device.c:373
u16 num_txqs
Definition: virtio.h:170
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:912
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
unsigned short u16
Definition: types.h:57
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
pci_addr_t pci_addr
Definition: virtio.h:148
#define PREDICT_FALSE(x)
Definition: clib.h:111
u32 node_index
Node index.
Definition: node.h:494
static u8 * format_virtio_device(u8 *s, va_list *args)
Definition: device.c:71
u16 desc_next
Definition: virtio.h:109
u16 virtio_net_hdr_sz
Definition: virtio.h:143
static void virtio_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:347
virtio_vring_t * rxq_vrings
Definition: virtio.h:161
u16 last_used_idx
Definition: virtio.h:118
u16 n_vectors
Definition: node.h:395
static u8 * format_virtio_tx_trace(u8 *s, va_list *args)
Definition: device.c:87
vlib_main_t * vm
Definition: buffer.c:312
#define VIRTIO_RING_FLAG_MASK_INT
Definition: virtio.h:99
static clib_error_t * virtio_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
Definition: device.c:411
static clib_error_t * virtio_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:396
u32 flags
Definition: virtio.h:137
u16 last_kick_avail_idx
Definition: virtio.h:119
vl_api_vxlan_gbp_api_tunnel_mode_t mode
Definition: vxlan_gbp.api:44
virtio_if_type_t type
Definition: virtio.h:144
u8 is_add
Definition: ipsec_gre.api:36
#define VIRTIO_FEATURE(X)
Definition: virtio.h:76
VNET_DEVICE_CLASS(bond_dev_class)
size_t count
Definition: vapi.c:47
u8 function
Definition: virtio.h:129
static uword pointer_to_uword(const void *p)
Definition: types.h:131
virtio_main_t virtio_main
Definition: virtio.c:37
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static_always_inline u16 add_buffer_to_slot(vlib_main_t *vm, virtio_if_t *vif, virtio_vring_t *vring, u32 bi, u16 avail, u16 next, u16 mask, int do_gso)
Definition: device.c:139
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
static_always_inline void virtio_free_used_device_desc(vlib_main_t *vm, virtio_vring_t *vring)
Definition: device.c:94
u32 * buffers
Definition: virtio.h:117
#define vnet_buffer(b)
Definition: buffer.h:369
struct vring_desc * desc
Definition: virtio.h:104
u8 * format_virtio_device_name(u8 *s, va_list *args)
Definition: device.c:51
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
Definition: buffer_funcs.h:612
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
u16 desc_in_use
Definition: virtio.h:108
virtio_vring_t * txq_vrings
Definition: virtio.h:162
static_always_inline void virtio_kick(vlib_main_t *vm, virtio_vring_t *vring, virtio_if_t *vif)
Definition: virtio.h:216
u8 slot
Definition: virtio.h:128