FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
output.c
Go to the documentation of this file.
1 #include <errno.h>
2 #include <string.h>
3 #include <vlib/vlib.h>
4 #include <vlib/unix/unix.h>
6 #include <vnet/devices/devices.h>
7 #include <af_xdp/af_xdp.h>
8 
9 #define AF_XDP_TX_RETRIES 5
10 
13  af_xdp_txq_t * txq)
14 {
15  const __u64 *compl;
16  const u32 size = txq->cq.size;
17  const u32 mask = size - 1;
18  u32 bis[VLIB_FRAME_SIZE], *bi = bis;
19  u32 n_wrap, idx;
20  u32 n = xsk_ring_cons__peek (&txq->cq, ARRAY_LEN (bis), &idx);
21  const u32 n_free = n;
22 
23  /* we rely on on casting addr (u64) -> bi (u32) to discard XSK offset below */
25  XSK_UNALIGNED_BUF_OFFSET_SHIFT, "wrong size");
26  ASSERT (mask == txq->cq.mask);
27 
28  if (!n_free)
29  return;
30 
31  compl = xsk_ring_cons__comp_addr (&txq->cq, idx);
32  n = clib_min (n_free, size - (idx & mask));
33  n_wrap = n_free - n;
34 
35 wrap_around:
36 
37  while (n >= 8)
38  {
39 #ifdef CLIB_HAVE_VEC256
40  u64x4 b0 = (*(u64x4u *) (compl + 0)) >> CLIB_LOG2_CACHE_LINE_BYTES;
41  u64x4 b1 = (*(u64x4u *) (compl + 4)) >> CLIB_LOG2_CACHE_LINE_BYTES;
42  /* permute 256-bit register so lower u32s of each buffer index are
43  * placed into lower 128-bits */
44  const u32x8 mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
45  u32x8 b2 = u32x8_permute ((u32x8) b0, mask);
46  u32x8 b3 = u32x8_permute ((u32x8) b1, mask);
47  /* extract lower 128-bits and save them to the array of buffer indices */
48  *(u32x4u *) (bi + 0) = u32x8_extract_lo (b2);
49  *(u32x4u *) (bi + 4) = u32x8_extract_lo (b3);
50 #else
51  bi[0] = compl[0] >> CLIB_LOG2_CACHE_LINE_BYTES;
52  bi[1] = compl[1] >> CLIB_LOG2_CACHE_LINE_BYTES;
53  bi[2] = compl[2] >> CLIB_LOG2_CACHE_LINE_BYTES;
54  bi[3] = compl[3] >> CLIB_LOG2_CACHE_LINE_BYTES;
55  bi[4] = compl[4] >> CLIB_LOG2_CACHE_LINE_BYTES;
56  bi[5] = compl[5] >> CLIB_LOG2_CACHE_LINE_BYTES;
57  bi[6] = compl[6] >> CLIB_LOG2_CACHE_LINE_BYTES;
58  bi[7] = compl[7] >> CLIB_LOG2_CACHE_LINE_BYTES;
59 #endif
60  compl += 8;
61  bi += 8;
62  n -= 8;
63  }
64 
65  while (n >= 1)
66  {
67  bi[0] = compl[0] >> CLIB_LOG2_CACHE_LINE_BYTES;
68  ASSERT (vlib_buffer_is_known (vm, bi[0]) ==
70  compl += 1;
71  bi += 1;
72  n -= 1;
73  }
74 
75  if (n_wrap)
76  {
77  compl = xsk_ring_cons__comp_addr (&txq->cq, 0);
78  n = n_wrap;
79  n_wrap = 0;
80  goto wrap_around;
81  }
82 
83  xsk_ring_cons__release (&txq->cq, n_free);
84  vlib_buffer_free (vm, bis, n_free);
85 }
86 
89  const vlib_node_runtime_t * node,
90  af_xdp_device_t * ad,
91  af_xdp_txq_t * txq, const u32 n_tx)
92 {
93  int ret;
94 
95  xsk_ring_prod__submit (&txq->tx, n_tx);
96 
97  if (!xsk_ring_prod__needs_wakeup (&txq->tx))
98  return;
99 
100  vlib_error_count (vm, node->node_index, AF_XDP_TX_ERROR_SENDTO_REQUIRED, 1);
101 
102  ret = sendto (txq->xsk_fd, NULL, 0, MSG_DONTWAIT, NULL, 0);
103  if (PREDICT_TRUE (ret >= 0))
104  return;
105 
106  /* those errors are fine */
107  switch (errno)
108  {
109  case ENOBUFS:
110  case EAGAIN:
111  case EBUSY:
112  return;
113  }
114 
115  /* something bad is happening */
116  vlib_error_count (vm, node->node_index, AF_XDP_TX_ERROR_SENDTO_FAILURES, 1);
117  af_xdp_device_error (ad, "sendto() failed");
118 }
119 
122  const vlib_node_runtime_t * node,
123  af_xdp_device_t * ad, af_xdp_txq_t * txq,
124  u32 n_tx, u32 * bi)
125 {
126  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b = bufs;
127  const uword start = vm->buffer_main->buffer_mem_start;
128  const u32 size = txq->tx.size;
129  const u32 mask = size - 1;
130  struct xdp_desc *desc;
131  u64 offset, addr;
132  u32 idx, n, n_wrap;
133 
134  ASSERT (mask == txq->cq.mask);
135 
136  n_tx = xsk_ring_prod__reserve (&txq->tx, n_tx, &idx);
137 
138  /* if ring is full, do nothing */
139  if (PREDICT_FALSE (0 == n_tx))
140  return 0;
141 
142  vlib_get_buffers (vm, bi, bufs, n_tx);
143 
144  desc = xsk_ring_prod__tx_desc (&txq->tx, idx);
145  n = clib_min (n_tx, size - (idx & mask));
146  n_wrap = n_tx - n;
147 
148 wrap_around:
149 
150  while (n >= 8)
151  {
152  vlib_prefetch_buffer_header (b[4], LOAD);
153  offset =
154  (sizeof (vlib_buffer_t) +
155  b[0]->current_data) << XSK_UNALIGNED_BUF_OFFSET_SHIFT;
156  addr = pointer_to_uword (b[0]) - start;
157  desc[0].addr = offset | addr;
158  desc[0].len = b[0]->current_length;
159 
160  vlib_prefetch_buffer_header (b[5], LOAD);
161  offset =
162  (sizeof (vlib_buffer_t) +
163  b[1]->current_data) << XSK_UNALIGNED_BUF_OFFSET_SHIFT;
164  addr = pointer_to_uword (b[1]) - start;
165  desc[1].addr = offset | addr;
166  desc[1].len = b[1]->current_length;
167 
168  vlib_prefetch_buffer_header (b[6], LOAD);
169  offset =
170  (sizeof (vlib_buffer_t) +
171  b[2]->current_data) << XSK_UNALIGNED_BUF_OFFSET_SHIFT;
172  addr = pointer_to_uword (b[2]) - start;
173  desc[2].addr = offset | addr;
174  desc[2].len = b[2]->current_length;
175 
176  vlib_prefetch_buffer_header (b[7], LOAD);
177  offset =
178  (sizeof (vlib_buffer_t) +
179  b[3]->current_data) << XSK_UNALIGNED_BUF_OFFSET_SHIFT;
180  addr = pointer_to_uword (b[3]) - start;
181  desc[3].addr = offset | addr;
182  desc[3].len = b[3]->current_length;
183 
184  desc += 4;
185  b += 4;
186  n -= 4;
187  }
188 
189  while (n >= 1)
190  {
191  offset =
192  (sizeof (vlib_buffer_t) +
193  b[0]->current_data) << XSK_UNALIGNED_BUF_OFFSET_SHIFT;
194  addr = pointer_to_uword (b[0]) - start;
195  desc[0].addr = offset | addr;
196  desc[0].len = b[0]->current_length;
197  desc += 1;
198  b += 1;
199  n -= 1;
200  }
201 
202  if (n_wrap)
203  {
204  desc = xsk_ring_prod__tx_desc (&txq->tx, 0);
205  n = n_wrap;
206  n_wrap = 0;
207  goto wrap_around;
208  }
209 
210  return n_tx;
211 }
212 
216 {
217  af_xdp_main_t *rm = &af_xdp_main;
218  vnet_interface_output_runtime_t *ord = (void *) node->runtime_data;
220  u32 thread_index = vm->thread_index;
221  af_xdp_txq_t *txq = vec_elt_at_index (ad->txqs, thread_index % ad->txq_num);
222  u32 *from;
223  u32 n, n_tx;
224  int i;
225 
226  from = vlib_frame_vector_args (frame);
227  n_tx = frame->n_vectors;
228 
230 
231  for (i = 0, n = 0; i < AF_XDP_TX_RETRIES && n < n_tx; i++)
232  {
233  u32 n_enq;
234  af_xdp_device_output_free (vm, node, txq);
235  n_enq = af_xdp_device_output_tx_try (vm, node, ad, txq, n_tx - n, from);
236  n += n_enq;
237  from += n_enq;
238  }
239 
240  af_xdp_device_output_tx_db (vm, node, ad, txq, n);
241 
243 
244  if (PREDICT_FALSE (n != n_tx))
245  {
246  vlib_buffer_free (vm, from + n, n_tx - n);
247  vlib_error_count (vm, node->node_index,
248  AF_XDP_TX_ERROR_NO_FREE_SLOTS, n_tx - n);
249  }
250 
251  return n;
252 }
253 
254 /*
255  * fd.io coding-style-patch-verification: ON
256  *
257  * Local Variables:
258  * eval: (c-set-style "gnu")
259  * End:
260  */
#define clib_min(x, y)
Definition: clib.h:327
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
Optimized string handling code, including c11-compliant "safe C library" variants.
#define PREDICT_TRUE(x)
Definition: clib.h:121
unsigned long u64
Definition: types.h:89
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:127
u32 thread_index
Definition: main.h:249
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
#define CLIB_LOG2_CACHE_LINE_BYTES
Definition: cache.h:50
vlib_buffer_main_t * buffer_main
Definition: main.h:183
foreach_avx2_vec256i static foreach_avx2_vec256u u32x8 u32x8_permute(u32x8 v, u32x8 idx)
Definition: vector_avx2.h:73
vlib_main_t * vm
Definition: in2out_ed.c:1582
#define af_xdp_device_error(dev, fmt,...)
Definition: af_xdp.h:44
VNET_DEVICE_CLASS_TX_FN() af_xdp_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: output.c:213
static vlib_buffer_known_state_t vlib_buffer_is_known(vlib_main_t *vm, u32 buffer_index)
Definition: buffer_funcs.h:498
u16 mask
Definition: flow_types.api:52
vhost_vring_addr_t addr
Definition: vhost_user.h:111
af_xdp_device_t * devices
Definition: af_xdp.h:114
#define static_always_inline
Definition: clib.h:108
af_xdp_main_t af_xdp_main
Definition: device.c:30
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:203
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:316
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:377
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
u32 size
Definition: vhost_user.h:106
static_always_inline void af_xdp_device_output_free(vlib_main_t *vm, const vlib_node_runtime_t *node, af_xdp_txq_t *txq)
Definition: output.c:12
#define PREDICT_FALSE(x)
Definition: clib.h:120
u32 node_index
Node index.
Definition: node.h:487
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
static_always_inline void af_xdp_device_output_tx_db(vlib_main_t *vm, const vlib_node_runtime_t *node, af_xdp_device_t *ad, af_xdp_txq_t *txq, const u32 n_tx)
Definition: output.c:88
struct xsk_ring_prod tx
Definition: af_xdp.h:74
#define ARRAY_LEN(x)
Definition: clib.h:67
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1582
#define AF_XDP_TX_RETRIES
Definition: output.c:9
#define ASSERT(truth)
clib_spinlock_t lock
Definition: af_xdp.h:73
static uword pointer_to_uword(const void *p)
Definition: types.h:131
int xsk_fd
Definition: af_xdp.h:76
static_always_inline u32 af_xdp_device_output_tx_try(vlib_main_t *vm, const vlib_node_runtime_t *node, af_xdp_device_t *ad, af_xdp_txq_t *txq, u32 n_tx, u32 *bi)
Definition: output.c:121
#define STATIC_ASSERT(truth,...)
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1583
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:297
uword buffer_mem_start
Definition: buffer.h:452
struct clib_bihash_value offset
template key/value backing page structure
u64x4
Definition: vector_avx2.h:121
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:280
#define BITS(x)
Definition: clib.h:66
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:104
struct xsk_ring_cons cq
Definition: af_xdp.h:75