FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
output.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/devices.h>
23 #include <vnet/ip/ip6_packet.h>
24 #include <vnet/ip/ip4_packet.h>
25 
26 #include <vmxnet3/vmxnet3.h>
27 
30 {
31  vmxnet3_tx_comp_ring *comp_ring = &txq->tx_comp_ring;
32 
33  comp_ring->next++;
34  if (PREDICT_FALSE (comp_ring->next == txq->size))
35  {
36  comp_ring->next = 0;
37  comp_ring->gen ^= VMXNET3_TXCF_GEN;
38  }
39 }
40 
43 {
44  txq->tx_ring.produce++;
45  if (PREDICT_FALSE (txq->tx_ring.produce == txq->size))
46  {
47  txq->tx_ring.produce = 0;
48  txq->tx_ring.gen ^= VMXNET3_TXF_GEN;
49  }
50 }
51 
54 {
55  txq->tx_ring.consume++;
56  txq->tx_ring.consume &= txq->size - 1;
57 }
58 
61  vmxnet3_txq_t * txq)
62 {
63  vmxnet3_tx_comp *tx_comp;
64  vmxnet3_tx_comp_ring *comp_ring;
65 
66  comp_ring = &txq->tx_comp_ring;
67  tx_comp = &txq->tx_comp[comp_ring->next];
68 
69  while ((tx_comp->flags & VMXNET3_TXCF_GEN) == comp_ring->gen)
70  {
71  u16 eop_idx = tx_comp->index & VMXNET3_TXC_INDEX;
72  u32 bi0 = txq->tx_ring.bufs[txq->tx_ring.consume];
73 
74  vlib_buffer_free_one (vm, bi0);
75  while (txq->tx_ring.consume != eop_idx)
76  {
78  }
80 
82  tx_comp = &txq->tx_comp[comp_ring->next];
83  }
84 }
85 
88 {
89  u16 count;
90 
91  count = (txq->tx_ring.consume - txq->tx_ring.produce - 1);
92  /* Wrapped? */
93  if (txq->tx_ring.produce >= txq->tx_ring.consume)
94  count += txq->size;
95  return count;
96 }
97 
101 {
102  vmxnet3_main_t *vmxm = &vmxnet3_main;
103  vnet_interface_output_runtime_t *rd = (void *) node->runtime_data;
105  u32 *buffers = vlib_frame_vector_args (frame);
106  u32 bi0;
107  vlib_buffer_t *b0;
108  vmxnet3_tx_desc *txd = 0;
109  u32 desc_idx, generation, first_idx;
110  u16 space_left;
111  u16 n_left = frame->n_vectors;
112  vmxnet3_txq_t *txq;
113  u16 qid = vm->thread_index % vd->num_tx_queues, produce;
114 
115  if (PREDICT_FALSE (!(vd->flags & VMXNET3_DEVICE_F_LINK_UP)))
116  {
117  vlib_buffer_free (vm, buffers, n_left);
118  vlib_error_count (vm, node->node_index, VMXNET3_TX_ERROR_LINK_DOWN,
119  n_left);
120  return (0);
121  }
122 
123  txq = vec_elt_at_index (vd->txqs, qid);
125 
126  vmxnet3_txq_release (vm, vd, txq);
127 
128  produce = txq->tx_ring.produce;
129  while (PREDICT_TRUE (n_left))
130  {
131  u16 space_needed = 1, i;
132  u32 gso_size = 0;
133  u32 l4_hdr_sz;
134  vlib_buffer_t *b;
135  u32 hdr_len = 0;
136 
137  bi0 = buffers[0];
138  b0 = vlib_get_buffer (vm, bi0);
139  b = b0;
140 
141  space_left = vmxnet3_tx_ring_space_left (txq);
142  while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
143  {
144  u32 next_buffer = b->next_buffer;
145 
146  b = vlib_get_buffer (vm, next_buffer);
147  space_needed++;
148  }
149  if (PREDICT_FALSE (space_left < space_needed))
150  {
151  vmxnet3_txq_release (vm, vd, txq);
152  space_left = vmxnet3_tx_ring_space_left (txq);
153 
154  if (PREDICT_FALSE (space_left < space_needed))
155  {
156  vlib_buffer_free_one (vm, bi0);
157  vlib_error_count (vm, node->node_index,
158  VMXNET3_TX_ERROR_NO_FREE_SLOTS, 1);
159  buffers++;
160  n_left--;
161  /*
162  * Drop this packet. But we may have enough room for the next
163  * packet
164  */
165  continue;
166  }
167  }
168 
169  /*
170  * Toggle the generation bit for SOP fragment to avoid device starts
171  * reading incomplete packet
172  */
173  generation = txq->tx_ring.gen ^ VMXNET3_TXF_GEN;
174  first_idx = txq->tx_ring.produce;
175  for (i = 0; i < space_needed; i++)
176  {
177  b0 = vlib_get_buffer (vm, bi0);
179 
180  desc_idx = txq->tx_ring.produce;
181 
183  txq->tx_ring.bufs[desc_idx] = bi0;
184 
185  txd = &txq->tx_desc[desc_idx];
186 
187  txd->address = vlib_buffer_get_current_pa (vm, b0);
188 
189  txd->flags[0] = generation | b0->current_length;
190  txd->flags[1] = 0;
191  if (PREDICT_FALSE (b0->flags & VNET_BUFFER_F_GSO))
192  {
193  /*
194  * We should not be getting GSO outbound traffic unless it is
195  * lro is enable
196  */
197  ASSERT (vd->gso_enable == 1);
198  gso_size = vnet_buffer2 (b0)->gso_size;
199  l4_hdr_sz = vnet_buffer2 (b0)->gso_l4_hdr_sz;
200  if (b0->flags & VNET_BUFFER_F_IS_IP6)
201  hdr_len = sizeof (ethernet_header_t) + sizeof (ip6_header_t) +
202  l4_hdr_sz;
203  else
204  hdr_len = sizeof (ethernet_header_t) + sizeof (ip4_header_t) +
205  l4_hdr_sz;
206  }
207 
208  generation = txq->tx_ring.gen;
209  bi0 = b0->next_buffer;
210  }
211  if (PREDICT_FALSE (gso_size != 0))
212  {
213  txq->tx_desc[first_idx].flags[1] = hdr_len;
214  txq->tx_desc[first_idx].flags[1] |= VMXNET3_TXF_OM (VMXNET3_OM_TSO);
215  txq->tx_desc[first_idx].flags[0] |= VMXNET3_TXF_MSSCOF (gso_size);
216  }
217  txd->flags[1] |= VMXNET3_TXF_CQ | VMXNET3_TXF_EOP;
218  asm volatile ("":::"memory");
219  /*
220  * Now toggle back the generation bit for the first segment.
221  * Device can start reading the packet
222  */
223  txq->tx_desc[first_idx].flags[0] ^= VMXNET3_TXF_GEN;
224 
225  buffers++;
226  n_left--;
227  }
228 
229  if (PREDICT_TRUE (produce != txq->tx_ring.produce))
231 
233 
234  return (frame->n_vectors - n_left);
235 }
236 
237 /*
238  * fd.io coding-style-patch-verification: ON
239  *
240  * Local Variables:
241  * eval: (c-set-style "gnu")
242  * End:
243  */
static_always_inline void vmxnet3_tx_ring_advance_consume(vmxnet3_txq_t *txq)
Definition: output.c:53
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
u8 count
Definition: dhcp.api:208
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:463
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:937
#define vnet_buffer2(b)
Definition: buffer.h:482
#define PREDICT_TRUE(x)
Definition: clib.h:121
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
clib_spinlock_t lock
Definition: vmxnet3.h:546
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:127
u32 thread_index
Definition: main.h:249
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
vmxnet3_main_t vmxnet3_main
Definition: vmxnet3.c:28
vlib_main_t * vm
Definition: in2out_ed.c:1582
#define VMXNET3_TXF_OM(x)
Definition: vmxnet3.h:149
VNET_DEVICE_CLASS_TX_FN() vmxnet3_device_class(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: output.c:98
#define static_always_inline
Definition: clib.h:108
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define VNET_DEVICE_CLASS_TX_FN(devclass)
Definition: interface.h:316
unsigned int u32
Definition: types.h:88
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
#define VMXNET3_TXC_INDEX
Definition: vmxnet3.h:156
static_always_inline u16 vmxnet3_tx_ring_space_left(vmxnet3_txq_t *txq)
Definition: output.c:87
unsigned short u16
Definition: types.h:57
static_always_inline void vmxnet3_tx_ring_advance_produce(vmxnet3_txq_t *txq)
Definition: output.c:42
vmxnet3_tx_comp_ring tx_comp_ring
Definition: vmxnet3.h:551
vmxnet3_tx_comp * tx_comp
Definition: vmxnet3.h:549
#define PREDICT_FALSE(x)
Definition: clib.h:120
#define VMXNET3_TXF_CQ
Definition: vmxnet3.h:152
#define VMXNET3_TXF_GEN
Definition: vmxnet3.h:146
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1582
#define ASSERT(truth)
static_always_inline void vmxnet3_txq_release(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_txq_t *txq)
Definition: output.c:60
#define VMXNET3_TXCF_GEN
Definition: vmxnet3.h:155
#define VMXNET3_TXF_EOP
Definition: vmxnet3.h:151
static_always_inline void vmxnet3_tx_comp_ring_advance_next(vmxnet3_txq_t *txq)
Definition: output.c:29
u32 reg_txprod
Definition: vmxnet3.h:545
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1583
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
Definition: buffer.h:492
VLIB buffer representation.
Definition: buffer.h:102
vmxnet3_tx_desc * tx_desc
Definition: vmxnet3.h:548
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:297
vmxnet3_device_t * devices
Definition: vmxnet3.h:592
#define VMXNET3_TXF_MSSCOF(x)
Definition: vmxnet3.h:150
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:970
vmxnet3_tx_ring tx_ring
Definition: vmxnet3.h:550
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:104
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static_always_inline void vmxnet3_reg_write_inline(vmxnet3_device_t *vd, u8 bar, u32 addr, u32 val)
Definition: vmxnet3.h:646