FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
output.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vppinfra/ring.h>
22 #include <vnet/ethernet/ethernet.h>
23 #include <vnet/devices/devices.h>
24 
25 #include <avf/avf.h>
26 
29 {
30  return d->qword[1] & 0x0f;
31 }
32 
34 avf_tx_enqueue (vlib_main_t * vm, avf_txq_t * txq, u32 * buffers,
35  u32 n_packets, int use_va_dma)
36 {
37  u16 next = txq->next;
39  u16 n_desc = 0;
40  u16 *slot, n_desc_left, n_packets_left = n_packets;
41  u16 mask = txq->size - 1;
42  vlib_buffer_t *b[4];
43  avf_tx_desc_t *d = txq->descs + next;
44 
45  /* avoid ring wrap */
46  n_desc_left = txq->size - clib_max (txq->next, txq->n_enqueued + 8);
47 
48  if (n_desc_left == 0)
49  return 0;
50 
51  while (n_packets_left && n_desc_left)
52  {
53  u32 or_flags;
54  if (n_packets_left < 8 || n_desc_left < 4)
55  goto one_by_one;
56 
57  vlib_prefetch_buffer_with_index (vm, buffers[4], LOAD);
58  vlib_prefetch_buffer_with_index (vm, buffers[5], LOAD);
59  vlib_prefetch_buffer_with_index (vm, buffers[6], LOAD);
60  vlib_prefetch_buffer_with_index (vm, buffers[7], LOAD);
61 
62  b[0] = vlib_get_buffer (vm, buffers[0]);
63  b[1] = vlib_get_buffer (vm, buffers[1]);
64  b[2] = vlib_get_buffer (vm, buffers[2]);
65  b[3] = vlib_get_buffer (vm, buffers[3]);
66 
67  or_flags = b[0]->flags | b[1]->flags | b[2]->flags | b[3]->flags;
68 
69  if (or_flags & VLIB_BUFFER_NEXT_PRESENT)
70  goto one_by_one;
71 
72  clib_memcpy_fast (txq->bufs + next, buffers, sizeof (u32) * 4);
73 
74  if (use_va_dma)
75  {
76  d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
77  d[1].qword[0] = vlib_buffer_get_current_va (b[1]);
78  d[2].qword[0] = vlib_buffer_get_current_va (b[2]);
79  d[3].qword[0] = vlib_buffer_get_current_va (b[3]);
80  }
81  else
82  {
83  d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
84  d[1].qword[0] = vlib_buffer_get_current_pa (vm, b[1]);
85  d[2].qword[0] = vlib_buffer_get_current_pa (vm, b[2]);
86  d[3].qword[0] = vlib_buffer_get_current_pa (vm, b[3]);
87  }
88 
89  d[0].qword[1] = ((u64) b[0]->current_length) << 34 | bits;
90  d[1].qword[1] = ((u64) b[1]->current_length) << 34 | bits;
91  d[2].qword[1] = ((u64) b[2]->current_length) << 34 | bits;
92  d[3].qword[1] = ((u64) b[3]->current_length) << 34 | bits;
93 
94  next += 4;
95  n_desc += 4;
96  buffers += 4;
97  n_packets_left -= 4;
98  n_desc_left -= 4;
99  d += 4;
100  continue;
101 
102  one_by_one:
103  txq->bufs[next] = buffers[0];
104  b[0] = vlib_get_buffer (vm, buffers[0]);
105 
106  if (use_va_dma)
107  d[0].qword[0] = vlib_buffer_get_current_va (b[0]);
108  else
109  d[0].qword[0] = vlib_buffer_get_current_pa (vm, b[0]);
110 
111  d[0].qword[1] = (((u64) b[0]->current_length) << 34) | bits;
112 
113  next += 1;
114  n_desc += 1;
115  buffers += 1;
116  n_packets_left -= 1;
117  n_desc_left -= 1;
118  d += 1;
119  }
120 
121  if ((slot = clib_ring_enq (txq->rs_slots)))
122  {
123  u16 rs_slot = slot[0] = (next - 1) & mask;
124  d = txq->descs + rs_slot;
125  d[0].qword[1] |= AVF_TXD_CMD_RS;
126  }
127 
129  *(txq->qtx_tail) = txq->next = next & mask;
130  txq->n_enqueued += n_desc;
131  return n_packets - n_packets_left;
132 }
133 
135  vlib_node_runtime_t * node,
136  vlib_frame_t * frame)
137 {
138  avf_main_t *am = &avf_main;
139  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
141  u32 thread_index = vm->thread_index;
142  u8 qid = thread_index;
143  avf_txq_t *txq = vec_elt_at_index (ad->txqs, qid % ad->num_queue_pairs);
144  u32 *buffers = vlib_frame_vector_args (frame);
145  u16 n_enq, n_left;
146  u16 n_retry = 2;
147 
149 
150  n_left = frame->n_vectors;
151 
152 retry:
153  /* release consumed bufs */
154  if (txq->n_enqueued)
155  {
156  i32 complete_slot = -1;
157  while (1)
158  {
159  u16 *slot = clib_ring_get_first (txq->rs_slots);
160 
161  if (slot == 0)
162  break;
163 
164  if (avf_tx_desc_get_dtyp (txq->descs + slot[0]) != 0x0F)
165  break;
166 
167  complete_slot = slot[0];
168 
169  clib_ring_deq (txq->rs_slots);
170  }
171 
172  if (complete_slot >= 0)
173  {
174  u16 first, mask, n_free;
175  mask = txq->size - 1;
176  first = (txq->next - txq->n_enqueued) & mask;
177  n_free = (complete_slot + 1 - first) & mask;
178 
179  txq->n_enqueued -= n_free;
180  vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size,
181  n_free);
182  }
183  }
184 
185  if (ad->flags & AVF_DEVICE_F_VA_DMA)
186  n_enq = avf_tx_enqueue (vm, txq, buffers, n_left, 1);
187  else
188  n_enq = avf_tx_enqueue (vm, txq, buffers, n_left, 0);
189 
190  n_left -= n_enq;
191 
192  if (n_left)
193  {
194  buffers += n_enq;
195 
196  if (n_retry--)
197  goto retry;
198 
199  vlib_buffer_free (vm, buffers, n_left);
200  vlib_error_count (vm, node->node_index,
201  AVF_TX_ERROR_NO_FREE_SLOTS, n_left);
202  }
203 
205 
206  return frame->n_vectors - n_left;
207 }
208 
209 /*
210  * fd.io coding-style-patch-verification: ON
211  *
212  * Local Variables:
213  * eval: (c-set-style "gnu")
214  * End:
215  */
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:328
static uword vlib_buffer_get_current_va(vlib_buffer_t *b)
Definition: buffer.h:222
static_always_inline u16 avf_tx_enqueue(vlib_main_t *vm, avf_txq_t *txq, u32 *buffers, u32 n_packets, int use_va_dma)
Definition: output.c:34
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:529
unsigned long u64
Definition: types.h:89
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
u32 thread_index
Definition: main.h:179
avf_device_t * devices
Definition: avf.h:196
volatile u32 * qtx_tail
Definition: avf.h:110
unsigned char u8
Definition: types.h:56
u16 * rs_slots
Definition: avf.h:117
#define static_always_inline
Definition: clib.h:99
#define AVF_TXD_CMD_RSV
Definition: avf.h:39
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
Definition: buffer_funcs.h:341
clib_spinlock_t lock
Definition: avf.h:113
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
VNET_DEVICE_CLASS_TX_FN() avf_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: output.c:134
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:287
unsigned int u32
Definition: types.h:88
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:584
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
#define clib_ring_deq(ring)
Definition: ring.h:120
#define AVF_TXD_CMD_EOP
Definition: avf.h:37
unsigned short u16
Definition: types.h:57
#define clib_ring_get_first(ring)
Definition: ring.h:123
vlib_main_t * vm
Definition: buffer.c:301
#define clib_ring_enq(ring)
Definition: ring.h:94
avf_tx_desc_t * descs
Definition: avf.h:114
static_always_inline u8 avf_tx_desc_get_dtyp(avf_tx_desc_t *d)
Definition: output.c:28
u32 * bufs
Definition: avf.h:115
signed int i32
Definition: types.h:77
Definition: avf.h:107
u16 n_enqueued
Definition: avf.h:116
avf_main_t avf_main
Definition: device.c:37
#define clib_max(x, y)
Definition: clib.h:288
#define AVF_TXD_CMD_RS
Definition: avf.h:38
u16 next
Definition: avf.h:111
u64 qword[2]
Definition: avf.h:86
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
u16 size
Definition: avf.h:112
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:115
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:117
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:62