FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2016 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <fcntl.h>
21 #include <linux/virtio_net.h>
22 #include <linux/vhost.h>
23 
24 #include <vlib/vlib.h>
25 #include <vlib/unix/unix.h>
26 #include <vnet/ethernet/ethernet.h>
27 #include <vnet/ip/ip4_packet.h>
28 #include <vnet/ip/ip6_packet.h>
30 
31 #define foreach_virtio_tx_func_error \
32 _(NO_FREE_SLOTS, "no free tx slots") \
33 _(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \
34 _(PENDING_MSGS, "pending msgs in tx ring") \
35 _(NO_TX_QUEUES, "no tx queues")
36 
37 typedef enum
38 {
39 #define _(f,s) TAP_TX_ERROR_##f,
41 #undef _
44 
45 static char *virtio_tx_func_error_strings[] = {
46 #define _(n,s) s,
48 #undef _
49 };
50 
51 u8 *
52 format_virtio_device_name (u8 * s, va_list * args)
53 {
54  u32 dev_instance = va_arg (*args, u32);
56  virtio_if_t *vif = pool_elt_at_index (mm->interfaces, dev_instance);
57 
58  if (vif->type == VIRTIO_IF_TYPE_TAP)
59  {
60  s = format (s, "tap%u", vif->id);
61  }
62  else
63  s = format (s, "virtio%lu", vif->dev_instance);
64 
65  return s;
66 }
67 
68 static u8 *
69 format_virtio_device (u8 * s, va_list * args)
70 {
71  u32 dev_instance = va_arg (*args, u32);
72  int verbose = va_arg (*args, int);
73  u32 indent = format_get_indent (s);
74 
75  s = format (s, "VIRTIO interface");
76  if (verbose)
77  {
78  s = format (s, "\n%U instance %u", format_white_space, indent + 2,
79  dev_instance);
80  }
81  return s;
82 }
83 
84 static u8 *
85 format_virtio_tx_trace (u8 * s, va_list * args)
86 {
87  s = format (s, "Unimplemented...");
88  return s;
89 }
90 
91 inline void
93 {
94  u16 used = vring->desc_in_use;
95  u16 sz = vring->size;
96  u16 mask = sz - 1;
97  u16 last = vring->last_used_idx;
98  u16 n_left = vring->used->idx - last;
99 
100  if (n_left == 0)
101  return;
102 
103  while (n_left)
104  {
105  struct vring_used_elem *e = &vring->used->ring[last & mask];
106  u16 slot = e->id;
107  struct vring_desc *d = &vring->desc[slot];
108 
109  if (PREDICT_FALSE (d->flags & VRING_DESC_F_INDIRECT))
110  {
111  d = uword_to_pointer (d->addr, struct vring_desc *);
112  vec_free (d);
113  }
114 
115  vlib_buffer_free (vm, &vring->buffers[slot], 1);
116  used--;
117  last++;
118  n_left--;
119  }
120  vring->desc_in_use = used;
121  vring->last_used_idx = last;
122 }
123 
126  u16 avail, u16 next, u16 mask)
127 {
128  u16 n_added = 0;
129  const int hdr_sz = sizeof (struct virtio_net_hdr_v1);
130  struct vring_desc *d;
131  d = &vring->desc[next];
132  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
133  struct virtio_net_hdr_v1 *hdr = vlib_buffer_get_current (b) - hdr_sz;
134 
135  clib_memset (hdr, 0, hdr_sz);
136 
137  if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
138  {
139  d->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz;
140  d->len = b->current_length + hdr_sz;
141  d->flags = 0;
142  }
143  else
144  {
145  struct vring_desc *id, *descs = 0;
146 
147  /* first buffer in chain */
148  vec_add2_aligned (descs, id, 1, CLIB_CACHE_LINE_BYTES);
149  id->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz;
150  id->len = b->current_length + hdr_sz;
151 
152  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
153  {
154  id->flags = VRING_DESC_F_NEXT;
155  id->next = vec_len (descs);
156  vec_add2_aligned (descs, id, 1, CLIB_CACHE_LINE_BYTES);
157  b = vlib_get_buffer (vm, b->next_buffer);
158  id->addr = pointer_to_uword (vlib_buffer_get_current (b));
159  id->len = b->current_length;
160  }
161 
162  d->addr = pointer_to_uword (descs);
163  d->len = vec_len (descs) * sizeof (struct vring_desc);
164  d->flags = VRING_DESC_F_INDIRECT;
165  }
166  vring->buffers[next] = bi;
167  vring->avail->ring[avail & mask] = next;
168  n_added++;
169  return n_added;
170 }
171 
174  vlib_frame_t * frame, virtio_if_t * vif)
175 {
176  u8 qid = 0;
177  u16 n_left = frame->n_vectors;
178  virtio_vring_t *vring = vec_elt_at_index (vif->vrings, (qid << 1) + 1);
179  u16 used, next, avail;
180  u16 sz = vring->size;
181  u16 mask = sz - 1;
182  u32 *buffers = vlib_frame_vector_args (frame);
183 
185 
186  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 &&
187  vring->last_kick_avail_idx != vring->avail->idx)
188  virtio_kick (vring);
189 
190  /* free consumed buffers */
191  virtio_free_used_desc (vm, vring);
192 
193  used = vring->desc_in_use;
194  next = vring->desc_next;
195  avail = vring->avail->idx;
196 
197  while (n_left && used < sz)
198  {
199  u16 n_added;
200  n_added = add_buffer_to_slot (vm, vring, buffers[0], avail, next, mask);
201  avail += n_added;
202  next = (next + n_added) & mask;
203  used += n_added;
204  buffers++;
205  n_left--;
206  }
207 
208  if (n_left != frame->n_vectors)
209  {
211  vring->avail->idx = avail;
212  vring->desc_next = next;
213  vring->desc_in_use = used;
214  if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0)
215  virtio_kick (vring);
216  }
217 
218 
219  if (n_left)
220  {
221  vlib_error_count (vm, node->node_index, TAP_TX_ERROR_NO_FREE_SLOTS,
222  n_left);
223  vlib_buffer_free (vm, buffers, n_left);
224  }
225 
227 
228  return frame->n_vectors - n_left;
229 }
230 
231 static uword
233  vlib_node_runtime_t * node, vlib_frame_t * frame)
234 {
235  virtio_main_t *nm = &virtio_main;
236  vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
238 
239  return virtio_interface_tx_inline (vm, node, frame, vif);
240 }
241 
242 static void
244  u32 node_index)
245 {
246  virtio_main_t *apm = &virtio_main;
247  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
249 
250  /* Shut off redirection */
251  if (node_index == ~0)
252  {
253  vif->per_interface_next_index = node_index;
254  return;
255  }
256 
259  node_index);
260 }
261 
262 static void
264 {
265  /* Nothing for now */
266 }
267 
268 static clib_error_t *
271 {
272  virtio_main_t *mm = &virtio_main;
273  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
275  virtio_vring_t *vring = vec_elt_at_index (vif->vrings, qid);
276 
278  vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT;
279  else
280  vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT;
281 
282  return 0;
283 }
284 
285 static clib_error_t *
287 {
288  virtio_main_t *mm = &virtio_main;
289  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
291 
293  vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
294  else
295  vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
296 
297  return 0;
298 }
299 
300 static clib_error_t *
302  u32 hw_if_index,
303  struct vnet_sw_interface_t *st, int is_add)
304 {
305  /* Nothing for now */
306  return 0;
307 }
308 
309 /* *INDENT-OFF* */
311  .name = "virtio",
312  .tx_function = virtio_interface_tx,
313  .format_device_name = format_virtio_device_name,
314  .format_device = format_virtio_device,
315  .format_tx_trace = format_virtio_tx_trace,
316  .tx_function_n_errors = TAP_TX_N_ERROR,
317  .tx_function_error_strings = virtio_tx_func_error_strings,
318  .rx_redirect_to_node = virtio_set_interface_next_node,
319  .clear_counters = virtio_clear_hw_interface_counters,
320  .admin_up_down_function = virtio_interface_admin_up_down,
321  .subif_add_del_function = virtio_subif_add_del_function,
322  .rx_mode_change_function = virtio_interface_rx_mode_change,
323 };
324 
327 /* *INDENT-ON* */
328 
329 /*
330  * fd.io coding-style-patch-verification: ON
331  *
332  * Local Variables:
333  * eval: (c-set-style "gnu")
334  * End:
335  */
u32 per_interface_next_index
Definition: virtio.h:102
struct vring_used * used
Definition: virtio.h:78
vlib_node_registration_t virtio_input_node
(constructor) VLIB_REGISTER_NODE (virtio_input_node)
Definition: node.c:285
virtio_if_t * interfaces
Definition: virtio.h:126
u32 flags
Definition: vhost_user.h:115
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:529
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:545
#define foreach_virtio_tx_func_error
Definition: device.c:31
#define PREDICT_TRUE(x)
Definition: clib.h:112
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:118
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
u32 dev_instance
Definition: virtio.h:99
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:576
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
vnet_device_class_t virtio_device_class
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
static u32 format_get_indent(u8 *s)
Definition: format.h:72
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
virtio_vring_t * vrings
Definition: virtio.h:105
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1092
unsigned char u8
Definition: types.h:56
static char * virtio_tx_func_error_strings[]
Definition: device.c:45
virtio_tx_func_error_t
Definition: device.c:37
vnet_hw_interface_rx_mode
Definition: interface.h:52
struct vring_avail * avail
Definition: virtio.h:79
#define static_always_inline
Definition: clib.h:99
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
static_always_inline uword virtio_interface_tx_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, virtio_if_t *vif)
Definition: device.c:173
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
static void virtio_clear_hw_interface_counters(u32 instance)
Definition: device.c:263
unsigned int u32
Definition: types.h:88
u32 id
Definition: virtio.h:98
static clib_error_t * virtio_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
Definition: device.c:269
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:114
unsigned short u16
Definition: types.h:57
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:214
#define PREDICT_FALSE(x)
Definition: clib.h:111
clib_spinlock_t lockp
Definition: virtio.h:96
u32 node_index
Node index.
Definition: node.h:518
static u8 * format_virtio_device(u8 *s, va_list *args)
Definition: device.c:69
u16 desc_next
Definition: virtio.h:81
static void virtio_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:243
u16 last_used_idx
Definition: virtio.h:89
static_always_inline void virtio_kick(virtio_vring_t *vring)
Definition: virtio.h:142
u16 n_vectors
Definition: node.h:420
static u8 * format_virtio_tx_trace(u8 *s, va_list *args)
Definition: device.c:85
vlib_main_t * vm
Definition: buffer.c:301
#define VIRTIO_RING_FLAG_MASK_INT
Definition: virtio.h:85
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
static clib_error_t * virtio_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
Definition: device.c:301
static clib_error_t * virtio_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:286
static_always_inline u16 add_buffer_to_slot(vlib_main_t *vm, virtio_vring_t *vring, u32 bi, u16 avail, u16 next, u16 mask)
Definition: device.c:125
u32 flags
Definition: virtio.h:95
u16 last_kick_avail_idx
Definition: virtio.h:90
virtio_if_type_t type
Definition: virtio.h:109
#define uword_to_pointer(u, type)
Definition: types.h:136
void virtio_free_used_desc(vlib_main_t *vm, virtio_vring_t *vring)
Definition: device.c:92
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:130
VNET_DEVICE_CLASS(bond_dev_class)
static uword pointer_to_uword(const void *p)
Definition: types.h:131
virtio_main_t virtio_main
Definition: virtio.c:35
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
u32 * buffers
Definition: virtio.h:88
vl_api_gbp_vxlan_tunnel_mode_t mode
Definition: gbp.api:349
struct vring_desc * desc
Definition: virtio.h:77
u8 * format_virtio_device_name(u8 *s, va_list *args)
Definition: device.c:52
u32 id
Definition: udp.api:45
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:117
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:62
#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH(dev, fn)
Definition: interface.h:320
u16 desc_in_use
Definition: virtio.h:80
static uword virtio_interface_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: device.c:232