FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
input.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vlib/unix/unix.h>
20 #include <vlib/pci/pci.h>
21 #include <vnet/ethernet/ethernet.h>
22 #include <vnet/devices/devices.h>
23 
24 #include <vmxnet3/vmxnet3.h>
25 
26 #define foreach_vmxnet3_input_error \
27  _(BUFFER_ALLOC, "buffer alloc error") \
28  _(RX_PACKET_NO_SOP, "Rx packet error - no SOP") \
29  _(RX_PACKET, "Rx packet error") \
30  _(RX_PACKET_EOP, "Rx packet error found on EOP") \
31  _(NO_BUFFER, "Rx no buffer error")
32 
33 typedef enum
34 {
35 #define _(f,s) VMXNET3_INPUT_ERROR_##f,
37 #undef _
40 
41 static __clib_unused char *vmxnet3_input_error_strings[] = {
42 #define _(n,s) s,
44 #undef _
45 };
46 
48 vmxnet3_find_rid (vmxnet3_device_t * vd, vmxnet3_rx_comp * rx_comp)
49 {
50  u32 rid;
51 
52  // rid is bits 16-25 (10 bits number)
53  rid = rx_comp->index & (0xffffffff >> 6);
54  rid >>= 16;
55  if ((rid >= vd->num_rx_queues) && (rid < (vd->num_rx_queues << 1)))
56  return 1;
57  else
58  return 0;
59 }
60 
63 {
64  vmxnet3_rx_comp_ring *comp_ring = &rxq->rx_comp_ring;
65 
66  comp_ring->next++;
67  if (PREDICT_FALSE (comp_ring->next == rxq->size))
68  {
69  comp_ring->next = 0;
70  comp_ring->gen ^= VMXNET3_RXCF_GEN;
71  }
72 }
73 
76  vlib_frame_t * frame, vmxnet3_device_t * vd,
77  u16 qid)
78 {
79  vnet_main_t *vnm = vnet_get_main ();
80  uword n_trace = vlib_get_trace_count (vm, node);
81  u32 n_rx_packets = 0, n_rx_bytes = 0;
82  vmxnet3_rx_comp *rx_comp;
83  u32 desc_idx;
84  vmxnet3_rxq_t *rxq;
85  u32 thread_index = vm->thread_index;
86  u32 buffer_indices[VLIB_FRAME_SIZE], *bi;
87  u16 nexts[VLIB_FRAME_SIZE], *next;
88  vmxnet3_rx_ring *ring;
89  vmxnet3_rx_comp_ring *comp_ring;
90  u16 rid;
91  vlib_buffer_t *prev_b0 = 0, *hb = 0;
93  u8 known_next = 0, got_packet = 0;
94  vmxnet3_rx_desc *rxd;
95  clib_error_t *error;
96 
97  rxq = vec_elt_at_index (vd->rxqs, qid);
98  comp_ring = &rxq->rx_comp_ring;
99  bi = buffer_indices;
100  next = nexts;
101  rx_comp = &rxq->rx_comp[comp_ring->next];
102 
103  while (PREDICT_TRUE ((n_rx_packets < VLIB_FRAME_SIZE) &&
104  (comp_ring->gen ==
105  (rx_comp->flags & VMXNET3_RXCF_GEN))))
106  {
107  vlib_buffer_t *b0;
108  u32 bi0;
109 
110  rid = vmxnet3_find_rid (vd, rx_comp);
111  ring = &rxq->rx_ring[rid];
112 
113  if (PREDICT_TRUE (ring->fill >= 1))
114  ring->fill--;
115  else
116  {
117  vlib_error_count (vm, node->node_index,
118  VMXNET3_INPUT_ERROR_NO_BUFFER, 1);
119  if (hb)
120  {
122  hb = 0;
123  }
124  prev_b0 = 0;
125  break;
126  }
127 
128  desc_idx = rx_comp->index & VMXNET3_RXC_INDEX;
129  ring->consume = desc_idx;
130  rxd = &rxq->rx_desc[rid][desc_idx];
131 
132  bi0 = ring->bufs[desc_idx];
133  ring->bufs[desc_idx] = ~0;
134 
135  b0 = vlib_get_buffer (vm, bi0);
136  vnet_buffer (b0)->sw_if_index[VLIB_RX] = vd->sw_if_index;
137  vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
138  vnet_buffer (b0)->feature_arc_index = 0;
139  b0->current_length = rx_comp->len & VMXNET3_RXCL_LEN_MASK;
140  b0->current_data = 0;
142  b0->next_buffer = 0;
143  b0->flags = 0;
144  b0->error = 0;
145  b0->current_config_index = 0;
146  ASSERT (b0->current_length != 0);
147 
148  if (PREDICT_FALSE ((rx_comp->index & VMXNET3_RXCI_EOP) &&
149  (rx_comp->len & VMXNET3_RXCL_ERROR)))
150  {
151  vlib_buffer_free_one (vm, bi0);
152  vlib_error_count (vm, node->node_index,
153  VMXNET3_INPUT_ERROR_RX_PACKET_EOP, 1);
154  if (hb && vlib_get_buffer_index (vm, hb) != bi0)
155  {
157  hb = 0;
158  }
159  prev_b0 = 0;
160  goto next;
161  }
162 
163  if (rx_comp->index & VMXNET3_RXCI_SOP)
164  {
165  ASSERT (!(rxd->flags & VMXNET3_RXF_BTYPE));
166  /* start segment */
167  hb = b0;
168  bi[0] = bi0;
169  if (!(rx_comp->index & VMXNET3_RXCI_EOP))
170  {
171  hb->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID;
172  prev_b0 = b0;
173  }
174  else
175  {
176  /*
177  * Both start and end of packet is set. It is a complete packet
178  */
179  prev_b0 = 0;
180  got_packet = 1;
181  }
182  }
183  else if (rx_comp->index & VMXNET3_RXCI_EOP)
184  {
185  /* end of segment */
186  if (prev_b0)
187  {
188  prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
189  prev_b0->next_buffer = bi0;
190  hb->total_length_not_including_first_buffer +=
191  b0->current_length;
192  prev_b0 = 0;
193  got_packet = 1;
194  }
195  else
196  {
197  /* EOP without SOP, error */
198  vlib_error_count (vm, node->node_index,
199  VMXNET3_INPUT_ERROR_RX_PACKET_NO_SOP, 1);
200  vlib_buffer_free_one (vm, bi0);
201  if (hb && vlib_get_buffer_index (vm, hb) != bi0)
202  {
204  hb = 0;
205  }
206  goto next;
207  }
208  }
209  else if (prev_b0) // !sop && !eop
210  {
211  /* mid chain */
212  ASSERT (rxd->flags & VMXNET3_RXF_BTYPE);
213  prev_b0->flags |= VLIB_BUFFER_NEXT_PRESENT;
214  prev_b0->next_buffer = bi0;
215  prev_b0 = b0;
217  }
218  else
219  {
220  vlib_error_count (vm, node->node_index,
221  VMXNET3_INPUT_ERROR_RX_PACKET, 1);
222  vlib_buffer_free_one (vm, bi0);
223  if (hb && vlib_get_buffer_index (vm, hb) != bi0)
224  {
226  hb = 0;
227  }
228  goto next;
229  }
230 
231  n_rx_bytes += b0->current_length;
232 
233  if (got_packet)
234  {
235  ethernet_header_t *e = (ethernet_header_t *) hb->data;
236 
237  if (PREDICT_FALSE (vd->per_interface_next_index != ~0))
238  {
239  next_index = vd->per_interface_next_index;
240  known_next = 1;
241  }
242 
243  if (PREDICT_FALSE
245  {
247  &next_index, hb);
248  known_next = 1;
249  }
250 
251  if (PREDICT_FALSE (known_next))
252  {
253  next[0] = next_index;
254  }
255  else
256  {
259  else
260  {
261  if (rx_comp->flags & VMXNET3_RXCF_IP4)
262  {
264  hb->flags |= VNET_BUFFER_F_IS_IP4;
267  [next[0]]);
268  }
269  else if (rx_comp->flags & VMXNET3_RXCF_IP6)
270  {
272  hb->flags |= VNET_BUFFER_F_IS_IP6;
275  [next[0]]);
276  }
277  else
278  {
280  }
281  }
282  }
283 
284  n_rx_packets++;
285  next++;
286  bi++;
287  hb = 0;
288  got_packet = 0;
289  }
290 
291  next:
293  rx_comp = &rxq->rx_comp[comp_ring->next];
294  }
295 
296  if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node))))
297  {
298  u32 n_left = n_rx_packets;
299 
300  bi = buffer_indices;
301  next = nexts;
302  while (n_trace && n_left)
303  {
304  vlib_buffer_t *b;
306 
307  b = vlib_get_buffer (vm, bi[0]);
308  vlib_trace_buffer (vm, node, next[0], b, /* follow_chain */ 0);
309  tr = vlib_add_trace (vm, node, b, sizeof (*tr));
310  tr->next_index = next[0];
311  tr->hw_if_index = vd->hw_if_index;
312  tr->buffer = *b;
313 
314  n_trace--;
315  n_left--;
316  bi++;
317  next++;
318  }
319  vlib_set_trace_count (vm, node, n_trace);
320  }
321 
322  if (PREDICT_TRUE (n_rx_packets))
323  {
324  vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts,
325  n_rx_packets);
328  VNET_INTERFACE_COUNTER_RX, thread_index,
329  vd->hw_if_index, n_rx_packets, n_rx_bytes);
330  }
331 
332  error = vmxnet3_rxq_refill_ring0 (vm, vd, rxq);
333  if (PREDICT_FALSE (error != 0))
334  {
335  vlib_error_count (vm, node->node_index,
336  VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1);
337  }
338  error = vmxnet3_rxq_refill_ring1 (vm, vd, rxq);
339  if (PREDICT_FALSE (error != 0))
340  {
341  vlib_error_count (vm, node->node_index,
342  VMXNET3_INPUT_ERROR_BUFFER_ALLOC, 1);
343  }
344 
345  return n_rx_packets;
346 }
347 
349  vlib_node_runtime_t * node,
350  vlib_frame_t * frame)
351 {
352  u32 n_rx = 0;
353  vmxnet3_main_t *vmxm = &vmxnet3_main;
354  vnet_device_input_runtime_t *rt = (void *) node->runtime_data;
356 
358  {
359  vmxnet3_device_t *vd;
360  vd = vec_elt_at_index (vmxm->devices, dq->dev_instance);
361  if ((vd->flags & VMXNET3_DEVICE_F_ADMIN_UP) == 0)
362  continue;
363  n_rx += vmxnet3_device_input_inline (vm, node, frame, vd, dq->queue_id);
364  }
365  return n_rx;
366 }
367 
368 #ifndef CLIB_MARCH_VARIANT
369 /* *INDENT-OFF* */
371  .name = "vmxnet3-input",
372  .sibling_of = "device-input",
373  .format_trace = format_vmxnet3_input_trace,
374  .type = VLIB_NODE_TYPE_INPUT,
375  .state = VLIB_NODE_STATE_DISABLED,
376  .n_errors = VMXNET3_INPUT_N_ERROR,
377  .error_strings = vmxnet3_input_error_strings,
378 };
379 #endif
380 
381 /* *INDENT-ON* */
382 
383 /*
384  * fd.io coding-style-patch-verification: ON
385  *
386  * Local Variables:
387  * eval: (c-set-style "gnu")
388  * End:
389  */
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring0(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:590
vnet_device_and_queue_t * devices_and_queues
Definition: devices.h:69
static u32 vlib_get_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt)
Definition: trace_funcs.h:156
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
const u32 device_input_next_node_advance[((VNET_DEVICE_INPUT_N_NEXT_NODES/CLIB_CACHE_LINE_BYTES)+1) *CLIB_CACHE_LINE_BYTES]
Definition: devices.c:47
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:136
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
vmxnet3_rx_desc * rx_desc[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:423
vnet_interface_main_t interface_main
Definition: vnet.h:56
#define PREDICT_TRUE(x)
Definition: clib.h:112
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
#define VMXNET3_RXCF_IP4
Definition: vmxnet3.h:85
u32 thread_index
Definition: main.h:179
vmxnet3_main_t vmxnet3_main
Definition: vmxnet3.c:28
#define VLIB_NODE_FN(node)
Definition: node.h:201
unsigned char u8
Definition: types.h:56
#define VMXNET3_RXCF_GEN
Definition: vmxnet3.h:86
static void vlib_trace_buffer(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, vlib_buffer_t *b, int follow_chain)
Definition: trace_funcs.h:114
vmxnet3_rxq_t * rxqs
Definition: vmxnet3.h:476
static __clib_unused char * vmxnet3_input_error_strings[]
Definition: input.c:41
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
#define static_always_inline
Definition: clib.h:99
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:839
static_always_inline int vnet_device_input_have_features(u32 sw_if_index)
Definition: feature.h:301
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define VMXNET3_RXC_INDEX
Definition: vmxnet3.h:87
#define VMXNET3_RXCL_LEN_MASK
Definition: vmxnet3.h:112
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:401
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
Definition: buffer_funcs.h:158
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:114
static_always_inline void vmxnet3_rx_comp_ring_advance_next(vmxnet3_rxq_t *rxq)
Definition: input.c:62
unsigned short u16
Definition: types.h:57
vmxnet3_rx_comp_ring rx_comp_ring
Definition: vmxnet3.h:425
#define PREDICT_FALSE(x)
Definition: clib.h:111
u32 node_index
Node index.
Definition: node.h:518
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:139
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
vlib_main_t * vm
Definition: buffer.c:301
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:332
vlib_buffer_t buffer
Definition: vmxnet3.h:521
u32 per_interface_next_index
Definition: vmxnet3.h:465
#define ASSERT(truth)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:130
static_always_inline int ethernet_frame_is_tagged(u16 type)
Definition: ethernet.h:111
#define VMXNET3_RXCI_EOP
Definition: vmxnet3.h:114
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:233
#define foreach_vmxnet3_input_error
Definition: input.c:26
static_always_inline uword vmxnet3_device_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vmxnet3_device_t *vd, u16 qid)
Definition: input.c:75
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:57
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:156
vlib_node_registration_t vmxnet3_input_node
(constructor) VLIB_REGISTER_NODE (vmxnet3_input_node)
Definition: input.c:370
#define foreach_device_and_queue(var, vec)
Definition: devices.h:161
Definition: defs.h:47
#define VMXNET3_RXCL_ERROR
Definition: vmxnet3.h:113
u64 uword
Definition: types.h:112
#define VMXNET3_RXF_BTYPE
Definition: vmxnet3.h:82
vmxnet3_rx_comp * rx_comp
Definition: vmxnet3.h:424
vmxnet3_rx_ring rx_ring[VMXNET3_RX_RING_SIZE]
Definition: vmxnet3.h:422
vmxnet3_input_error_t
Definition: input.c:33
format_function_t format_vmxnet3_input_trace
Definition: vmxnet3.h:534
#define vnet_buffer(b)
Definition: buffer.h:368
#define VMXNET3_RXCF_IP6
Definition: vmxnet3.h:84
static_always_inline void vnet_feature_start_device_input_x1(u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:308
vmxnet3_device_t * devices
Definition: vmxnet3.h:498
static_always_inline clib_error_t * vmxnet3_rxq_refill_ring1(vlib_main_t *vm, vmxnet3_device_t *vd, vmxnet3_rxq_t *rxq)
Definition: vmxnet3.h:635
#define VMXNET3_RXCI_SOP
Definition: vmxnet3.h:115
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
Definition: buffer_funcs.h:570
static void vlib_set_trace_count(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 count)
Definition: trace_funcs.h:172
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:117
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:62
Definition: defs.h:46
static_always_inline u16 vmxnet3_find_rid(vmxnet3_device_t *vd, vmxnet3_rx_comp *rx_comp)
Definition: input.c:48