FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
input.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <poll.h>
19 #include <vlib/vlib.h>
20 #include <vlib/unix/unix.h>
21 #include <vlib/pci/pci.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/devices/devices.h>
24 #include "af_xdp.h"
25 
26 #define foreach_af_xdp_input_error \
27  _(POLL_REQUIRED, "poll required") \
28  _(POLL_FAILURES, "poll failures")
29 
30 typedef enum
31 {
32 #define _(f,s) AF_XDP_INPUT_ERROR_##f,
34 #undef _
37 
38 static __clib_unused char *af_xdp_input_error_strings[] = {
39 #define _(n,s) s,
41 #undef _
42 };
43 
46  u32 n_left, const u32 * bi, u32 next_index,
47  u32 hw_if_index)
48 {
49  u32 n_trace;
50 
51  if (PREDICT_TRUE (0 == (n_trace = vlib_get_trace_count (vm, node))))
52  return;
53 
54  while (n_trace && n_left)
55  {
56  vlib_buffer_t *b;
58  b = vlib_get_buffer (vm, bi[0]);
59  vlib_trace_buffer (vm, node, next_index, b,
60  /* follow_chain */ 0);
61  tr = vlib_add_trace (vm, node, b, sizeof (*tr));
62  tr->next_index = next_index;
63  tr->hw_if_index = hw_if_index;
64 
65  n_trace--;
66  n_left--;
67  bi++;
68  }
69 
70  vlib_set_trace_count (vm, node, n_trace);
71 }
72 
75  const vlib_node_runtime_t * node,
76  af_xdp_device_t * ad, af_xdp_rxq_t * rxq,
77  const u32 n_alloc)
78 {
79  int ret;
80 
81  xsk_ring_prod__submit (&rxq->fq, n_alloc);
82 
83  if (!xsk_ring_prod__needs_wakeup (&rxq->fq))
84  return;
85 
86  vlib_error_count (vm, node->node_index, AF_XDP_INPUT_ERROR_POLL_REQUIRED,
87  1);
88 
89  struct pollfd fd = {.fd = rxq->xsk_fd,.events = POLLIN };
90  ret = poll (&fd, 1, 0);
91  if (PREDICT_TRUE (ret >= 0))
92  return;
93 
94  /* something bad is happening */
95  vlib_error_count (vm, node->node_index, AF_XDP_INPUT_ERROR_POLL_FAILURES,
96  1);
97  af_xdp_device_error (ad, "poll() failed");
98 }
99 
102  const vlib_node_runtime_t * node,
103  af_xdp_device_t * ad, af_xdp_rxq_t * rxq,
104  const int copy)
105 {
106  __u64 *fill;
107  const u32 size = rxq->fq.size;
108  const u32 mask = size - 1;
109  u32 bis[VLIB_FRAME_SIZE], *bi = bis;
110  u32 n_alloc, n, n_wrap;
111  u32 idx = 0;
112 
113  ASSERT (mask == rxq->fq.mask);
114 
115  /* do not enqueue more packet than ring space */
116  n_alloc = xsk_prod_nb_free (&rxq->fq, 16);
117  /* do not bother to allocate if too small */
118  if (n_alloc < 16)
119  return;
120 
121  n_alloc = clib_min (n_alloc, ARRAY_LEN (bis));
122  n_alloc = vlib_buffer_alloc_from_pool (vm, bis, n_alloc, ad->pool);
123  n = xsk_ring_prod__reserve (&rxq->fq, n_alloc, &idx);
124  ASSERT (n == n_alloc);
125 
126  fill = xsk_ring_prod__fill_addr (&rxq->fq, idx);
127  n = clib_min (n_alloc, size - (idx & mask));
128  n_wrap = n_alloc - n;
129 
130  /*
131  * Note about headroom: for some reasons, there seem to be a discrepency
132  * between 0-copy and copy mode:
133  * - 0-copy: XDP_PACKET_HEADROOM will be added to the user headroom
134  * - copy: nothing is added to the user headroom
135  * We privileged 0-copy and set headroom to 0. As XDP_PACKET_HEADROOM ==
136  * sizeof(vlib_buffer_t), data will correctly point to vlib_buffer_t->data.
137  * In copy mode, we have to add sizeof(vlib_buffer_t) to desc offset during
138  * refill.
139  */
140  STATIC_ASSERT (sizeof (vlib_buffer_t) == XDP_PACKET_HEADROOM, "wrong size");
141 #define bi2addr(bi) \
142  (((bi) << CLIB_LOG2_CACHE_LINE_BYTES) + (copy ? sizeof(vlib_buffer_t) : 0))
143 
144 wrap_around:
145 
146  while (n >= 8)
147  {
148 #ifdef CLIB_HAVE_VEC256
149  u64x4 b0 = u64x4_from_u32x4 (*(u32x4u *) (bi + 0));
150  u64x4 b1 = u64x4_from_u32x4 (*(u32x4u *) (bi + 4));
151  *(u64x4u *) (fill + 0) = bi2addr (b0);
152  *(u64x4u *) (fill + 4) = bi2addr (b1);
153 #else
154  fill[0] = bi2addr (bi[0]);
155  fill[1] = bi2addr (bi[1]);
156  fill[2] = bi2addr (bi[2]);
157  fill[3] = bi2addr (bi[3]);
158  fill[4] = bi2addr (bi[4]);
159  fill[5] = bi2addr (bi[5]);
160  fill[6] = bi2addr (bi[6]);
161  fill[7] = bi2addr (bi[7]);
162 #endif
163  fill += 8;
164  bi += 8;
165  n -= 8;
166  }
167 
168  while (n >= 1)
169  {
170  fill[0] = bi2addr (bi[0]);
171  fill += 1;
172  bi += 1;
173  n -= 1;
174  }
175 
176  if (n_wrap)
177  {
178  fill = xsk_ring_prod__fill_addr (&rxq->fq, 0);
179  n = n_wrap;
180  n_wrap = 0;
181  goto wrap_around;
182  }
183 
184  af_xdp_device_input_refill_db (vm, node, ad, rxq, n_alloc);
185 }
186 
189  const u32 next_index, const u32 sw_if_index,
190  const u32 hw_if_index)
191 {
192  vlib_next_frame_t *nf;
193  vlib_frame_t *f;
195 
197  return;
198 
199  nf =
202  f = vlib_get_frame (vm, nf->frame);
204 
205  ef = vlib_frame_scalar_args (f);
206  ef->sw_if_index = sw_if_index;
207  ef->hw_if_index = hw_if_index;
208 }
209 
212  af_xdp_rxq_t * rxq, u32 * bis, const u32 n_rx,
213  vlib_buffer_t * bt, u32 idx, const int copy)
214 {
215  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
216  u16 lens[VLIB_FRAME_SIZE], *len = lens;
217  const u32 mask = rxq->rx.mask;
218  u32 n = n_rx, *bi = bis, bytes = 0;
219 
220 #define addr2bi(addr) \
221  (((addr) - (copy ? sizeof(vlib_buffer_t) : 0)) >> CLIB_LOG2_CACHE_LINE_BYTES)
222 
223  while (n >= 1)
224  {
225  const struct xdp_desc *desc = xsk_ring_cons__rx_desc (&rxq->rx, idx);
226  bi[0] = addr2bi (xsk_umem__extract_addr (desc->addr));
227  ASSERT (vlib_buffer_is_known (vm, bi[0]) ==
229  len[0] = desc->len;
230  idx = (idx + 1) & mask;
231  bi += 1;
232  len += 1;
233  n -= 1;
234  }
235 
236  vlib_get_buffers (vm, bis, bufs, n_rx);
237 
238  n = n_rx;
239  len = lens;
240 
241  while (n >= 8)
242  {
243  vlib_prefetch_buffer_header (b[4], LOAD);
244  vlib_buffer_copy_template (b[0], bt);
245  bytes += b[0]->current_length = len[0];
246 
247  vlib_prefetch_buffer_header (b[5], LOAD);
248  vlib_buffer_copy_template (b[1], bt);
249  bytes += b[1]->current_length = len[1];
250 
251  vlib_prefetch_buffer_header (b[6], LOAD);
252  vlib_buffer_copy_template (b[2], bt);
253  bytes += b[2]->current_length = len[2];
254 
255  vlib_prefetch_buffer_header (b[7], LOAD);
256  vlib_buffer_copy_template (b[3], bt);
257  bytes += b[3]->current_length = len[3];
258 
259  b += 4;
260  len += 4;
261  n -= 4;
262  }
263 
264  while (n >= 1)
265  {
266  vlib_buffer_copy_template (b[0], bt);
267  bytes += b[0]->current_length = len[0];
268  b += 1;
269  len += 1;
270  n -= 1;
271  }
272 
273  xsk_ring_cons__release (&rxq->rx, n_rx);
274  return bytes;
275 }
276 
280  u16 qid, const int copy)
281 {
282  vnet_main_t *vnm = vnet_get_main ();
283  af_xdp_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
284  vlib_buffer_t bt;
285  u32 next_index, *to_next, n_left_to_next;
286  u32 n_rx_packets, n_rx_bytes;
287  u32 idx;
288 
289  n_rx_packets = xsk_ring_cons__peek (&rxq->rx, VLIB_FRAME_SIZE, &idx);
290 
291  if (PREDICT_FALSE (0 == n_rx_packets))
292  goto refill;
293 
295  next_index = ad->per_interface_next_index;
297  vnet_feature_start_device_input_x1 (ad->sw_if_index, &next_index, &bt);
298 
299  vlib_get_new_next_frame (vm, node, next_index, to_next, n_left_to_next);
300 
301  n_rx_bytes =
302  af_xdp_device_input_bufs (vm, ad, rxq, to_next, n_rx_packets, &bt, idx,
303  copy);
304  af_xdp_device_input_ethernet (vm, node, next_index, ad->sw_if_index,
305  ad->hw_if_index);
306 
307  vlib_put_next_frame (vm, node, next_index, n_left_to_next - n_rx_packets);
308 
309  af_xdp_device_input_trace (vm, node, n_rx_packets, to_next, next_index,
310  ad->hw_if_index);
311 
315  ad->hw_if_index, n_rx_packets, n_rx_bytes);
316 
317 refill:
318  af_xdp_device_input_refill (vm, node, ad, rxq, copy);
319 
320  return n_rx_packets;
321 }
322 
326 {
327  u32 n_rx = 0;
328  af_xdp_main_t *am = &af_xdp_main;
329  vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
331 
333  {
334  af_xdp_device_t *ad;
335  ad = vec_elt_at_index (am->devices, dq->dev_instance);
336  if ((ad->flags & AF_XDP_DEVICE_F_ADMIN_UP) == 0)
337  continue;
338  if (PREDICT_TRUE (ad->flags & AF_XDP_DEVICE_F_ZEROCOPY))
339  n_rx += af_xdp_device_input_inline (vm, node, frame, ad, dq->queue_id,
340  /* copy */ 0);
341  else
342  n_rx += af_xdp_device_input_inline (vm, node, frame, ad, dq->queue_id,
343  /* copy */ 1);
344  }
345  return n_rx;
346 }
347 
348 /* *INDENT-OFF* */
350  .name = "af_xdp-input",
351  .sibling_of = "device-input",
352  .format_trace = format_af_xdp_input_trace,
353  .type = VLIB_NODE_TYPE_INPUT,
354  .state = VLIB_NODE_STATE_DISABLED,
355  .n_errors = AF_XDP_INPUT_N_ERROR,
356  .error_strings = af_xdp_input_error_strings,
358 };
359 /* *INDENT-ON* */
360 
361 /*
362  * fd.io coding-style-patch-verification: ON
363  *
364  * Local Variables:
365  * eval: (c-set-style "gnu")
366  * End:
367  */
vnet_device_and_queue_t * devices_and_queues
Definition: devices.h:69
#define clib_min(x, y)
Definition: clib.h:327
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:193
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
struct xsk_ring_cons rx
Definition: af_xdp.h:58
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
vlib_node_registration_t af_xdp_input_node
(constructor) VLIB_REGISTER_NODE (af_xdp_input_node)
Definition: input.c:349
vnet_interface_main_t interface_main
Definition: vnet.h:59
u32 per_interface_next_index
Definition: af_xdp.h:88
#define PREDICT_TRUE(x)
Definition: clib.h:121
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define VLIB_NODE_FLAG_TRACE_SUPPORTED
Definition: node.h:305
u32 thread_index
Definition: main.h:249
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:269
static_always_inline void af_xdp_device_input_trace(vlib_main_t *vm, vlib_node_runtime_t *node, u32 n_left, const u32 *bi, u32 next_index, u32 hw_if_index)
Definition: input.c:45
struct xsk_ring_prod fq
Definition: af_xdp.h:59
vlib_main_t * vm
Definition: in2out_ed.c:1582
#define af_xdp_device_error(dev, fmt,...)
Definition: af_xdp.h:44
#define VLIB_NODE_FN(node)
Definition: node.h:202
static vlib_buffer_known_state_t vlib_buffer_is_known(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:498
u16 mask
Definition: flow_types.api:52
af_xdp_device_t * devices
Definition: af_xdp.h:114
af_xdp_rxq_t * rxqs
Definition: af_xdp.h:85
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:136
vlib_buffer_t * buffer_template
Definition: af_xdp.h:87
#define static_always_inline
Definition: clib.h:108
#define ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX
Definition: ethernet.h:52
af_xdp_main_t af_xdp_main
Definition: device.c:30
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:881
static_always_inline int vnet_device_input_have_features(u32 sw_if_index)
Definition: feature.h:336
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define vlib_get_new_next_frame(vm, node, next_index, vectors, n_vectors_left)
Definition: node_funcs.h:396
unsigned int u32
Definition: types.h:88
static_always_inline void af_xdp_device_input_ethernet(vlib_main_t *vm, vlib_node_runtime_t *node, const u32 next_index, const u32 sw_if_index, const u32 hw_if_index)
Definition: input.c:188
#define VLIB_FRAME_SIZE
Definition: node.h:377
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:317
static __clib_unused char * af_xdp_input_error_strings[]
Definition: input.c:38
static_always_inline void af_xdp_device_input_refill_db(vlib_main_t *vm, const vlib_node_runtime_t *node, af_xdp_device_t *ad, af_xdp_rxq_t *rxq, const u32 n_alloc)
Definition: input.c:74
format_function_t format_af_xdp_input_trace
Definition: af_xdp.h:153
#define bi2addr(bi)
int xsk_fd
Definition: af_xdp.h:60
static_always_inline void vlib_buffer_copy_template(vlib_buffer_t *b, vlib_buffer_t *bt)
Definition: buffer_funcs.h:181
unsigned short u16
Definition: types.h:57
u32 size
Definition: vhost_user.h:106
#define PREDICT_FALSE(x)
Definition: clib.h:120
u32 node_index
Node index.
Definition: node.h:487
#define foreach_af_xdp_input_error
Definition: input.c:26
u8 len
Definition: ip_types.api:92
u32 dev_instance
Definition: af_xdp.h:99
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
#define addr2bi(addr)
static void * vlib_frame_scalar_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:311
#define ARRAY_LEN(x)
Definition: clib.h:67
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:483
af_xdp_input_error_t
Definition: input.c:30
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1582
#define ASSERT(truth)
vlib_frame_t * frame
Definition: node.h:405
u16 flags
Definition: node.h:387
u32 hw_if_index
Definition: af_xdp.h:90
static_always_inline u32 af_xdp_device_input_bufs(vlib_main_t *vm, const af_xdp_device_t *ad, af_xdp_rxq_t *rxq, u32 *bis, const u32 n_rx, vlib_buffer_t *bt, u32 idx, const int copy)
Definition: input.c:211
#define foreach_device_and_queue(var, vec)
Definition: devices.h:161
#define STATIC_ASSERT(truth,...)
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1583
static_always_inline uword af_xdp_device_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, af_xdp_device_t *ad, u16 qid, const int copy)
Definition: input.c:278
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
u32 sw_if_index
Definition: af_xdp.h:89
static_always_inline void af_xdp_device_input_refill(vlib_main_t *vm, const vlib_node_runtime_t *node, af_xdp_device_t *ad, af_xdp_rxq_t *rxq, const int copy)
Definition: input.c:101
u64x4
Definition: vector_avx2.h:121
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:343
void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace.c:577
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:209
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:280
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
vl_api_interface_index_t sw_if_index
Definition: wireguard.api:33
static __clib_warn_unused_result u32 vlib_buffer_alloc_from_pool(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers from specific pool into supplied array.
Definition: buffer_funcs.h:566