FD.io VPP  v17.04-9-g99c0734
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2016 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #define _GNU_SOURCE
19 #include <stdint.h>
20 #include <net/if.h>
21 #include <sys/ioctl.h>
22 #include <sys/uio.h>
23 
24 #include <vlib/vlib.h>
25 #include <vlib/unix/unix.h>
26 #include <vnet/ethernet/ethernet.h>
27 
28 #include <memif/memif.h>
29 
30 #define foreach_memif_tx_func_error \
31 _(NO_FREE_SLOTS, "no free tx slots") \
32 _(PENDING_MSGS, "pending msgs in tx ring")
33 
34 typedef enum
35 {
36 #define _(f,s) MEMIF_TX_ERROR_##f,
38 #undef _
41 
42 static char *memif_tx_func_error_strings[] = {
43 #define _(n,s) s,
45 #undef _
46 };
47 
48 
49 static u8 *
50 format_memif_device_name (u8 * s, va_list * args)
51 {
52  u32 i = va_arg (*args, u32);
53 
54  s = format (s, "memif%u", i);
55  return s;
56 }
57 
58 static u8 *
59 format_memif_device (u8 * s, va_list * args)
60 {
61  u32 dev_instance = va_arg (*args, u32);
62  int verbose = va_arg (*args, int);
63  uword indent = format_get_indent (s);
64 
65  s = format (s, "MEMIF interface");
66  if (verbose)
67  {
68  s = format (s, "\n%U instance %u", format_white_space, indent + 2,
69  dev_instance);
70  }
71  return s;
72 }
73 
74 static u8 *
75 format_memif_tx_trace (u8 * s, va_list * args)
76 {
77  s = format (s, "Unimplemented...");
78  return s;
79 }
80 
83 {
84  if (PREDICT_FALSE (mif->lockp != 0))
85  {
86  while (__sync_lock_test_and_set (mif->lockp, 1))
87  ;
88  }
89 }
90 
93 {
94  if (PREDICT_FALSE (mif->lockp != 0))
95  *mif->lockp = 0;
96 }
97 
100 {
101  vlib_buffer_t *b = vlib_get_buffer (vm, bi);
102  vlib_prefetch_buffer_header (b, LOAD);
104 }
105 
108  vlib_frame_t * frame, memif_if_t * mif,
109  memif_ring_type_t type)
110 {
111  u8 rid = 0;
112  memif_ring_t *ring = memif_get_ring (mif, type, rid);
113  u32 *buffers = vlib_frame_args (frame);
114  u32 n_left = frame->n_vectors;
115  u16 ring_size = 1 << mif->log2_ring_size;
116  u16 mask = ring_size - 1;
117  u16 head, tail;
118  u16 free_slots;
119 
120  memif_interface_lock (mif);
121 
122  /* free consumed buffers */
123 
124  head = ring->head;
125  tail = ring->tail;
126 
127  if (tail > head)
128  free_slots = tail - head;
129  else
130  free_slots = ring_size - head + tail;
131 
132  while (n_left > 5 && free_slots > 1)
133  {
134  if (PREDICT_TRUE (head + 5 < ring_size))
135  {
136  CLIB_PREFETCH (memif_get_buffer (mif, ring, head + 2),
137  CLIB_CACHE_LINE_BYTES, STORE);
138  CLIB_PREFETCH (memif_get_buffer (mif, ring, head + 3),
139  CLIB_CACHE_LINE_BYTES, STORE);
140  CLIB_PREFETCH (&ring->desc[head + 4], CLIB_CACHE_LINE_BYTES, STORE);
141  CLIB_PREFETCH (&ring->desc[head + 5], CLIB_CACHE_LINE_BYTES, STORE);
142  }
143  else
144  {
145  CLIB_PREFETCH (memif_get_buffer (mif, ring, (head + 2) % mask),
146  CLIB_CACHE_LINE_BYTES, STORE);
147  CLIB_PREFETCH (memif_get_buffer (mif, ring, (head + 3) % mask),
148  CLIB_CACHE_LINE_BYTES, STORE);
149  CLIB_PREFETCH (&ring->desc[(head + 4) % mask],
150  CLIB_CACHE_LINE_BYTES, STORE);
151  CLIB_PREFETCH (&ring->desc[(head + 5) % mask],
152  CLIB_CACHE_LINE_BYTES, STORE);
153  }
154 
155  memif_prefetch_buffer_and_data (vm, buffers[2]);
156  memif_prefetch_buffer_and_data (vm, buffers[3]);
157 
158  vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[0]);
159  vlib_buffer_t *b1 = vlib_get_buffer (vm, buffers[1]);
160 
161  void *mb0 = memif_get_buffer (mif, ring, head);
163  ring->desc[head].length = b0->current_length;
164  head = (head + 1) & mask;
165 
166  void *mb1 = memif_get_buffer (mif, ring, head);
168  ring->desc[head].length = b1->current_length;
169  head = (head + 1) & mask;
170 
172  {
176  }
178  {
182  }
183 
184 
185  buffers += 2;
186  n_left -= 2;
187  free_slots -= 2;
188  }
189 
190  while (n_left && free_slots)
191  {
192  vlib_buffer_t *b0 = vlib_get_buffer (vm, buffers[0]);
193  void *mb0 = memif_get_buffer (mif, ring, head);
195 
197  {
201  }
202  ring->desc[head].length = b0->current_length;
203  head = (head + 1) & mask;
204 
205  buffers++;
206  n_left--;
207  free_slots--;
208  }
209 
211  ring->head = head;
212 
214 
215  if (n_left)
216  {
217  vlib_error_count (vm, node->node_index, MEMIF_TX_ERROR_NO_FREE_SLOTS,
218  n_left);
219  vlib_buffer_free (vm, buffers, n_left);
220  }
221 
222  vlib_buffer_free (vm, vlib_frame_args (frame), frame->n_vectors);
223  if (mif->interrupt_line.fd > 0)
224  {
225  u8 b = rid;
226  CLIB_UNUSED (int r) = write (mif->interrupt_line.fd, &b, sizeof (b));
227  }
228 
229  return frame->n_vectors;
230 }
231 
232 static uword
234  vlib_node_runtime_t * node, vlib_frame_t * frame)
235 {
236  memif_main_t *nm = &memif_main;
237  vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
239 
240  if (mif->flags & MEMIF_IF_FLAG_IS_SLAVE)
241  return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_S2M);
242  else
243  return memif_interface_tx_inline (vm, node, frame, mif, MEMIF_RING_M2S);
244 }
245 
246 static void
248  u32 node_index)
249 {
250  memif_main_t *apm = &memif_main;
251  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
253 
254  /* Shut off redirection */
255  if (node_index == ~0)
256  {
257  mif->per_interface_next_index = node_index;
258  return;
259  }
260 
262  vlib_node_add_next (vlib_get_main (), memif_input_node.index, node_index);
263 }
264 
265 static void
267 {
268  /* Nothing for now */
269 }
270 
271 static clib_error_t *
273 {
274  memif_main_t *apm = &memif_main;
276  memif_msg_t msg = { 0 };
277  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
279  static clib_error_t *error = 0;
280 
283  else
284  {
285  mif->flags &= ~MEMIF_IF_FLAG_ADMIN_UP;
286  if (!(mif->flags & MEMIF_IF_FLAG_DELETING)
287  && mif->connection.index != ~0)
288  {
289  msg.version = MEMIF_VERSION;
291  if (send (mif->connection.fd, &msg, sizeof (msg), 0) < 0)
292  {
293  clib_unix_warning ("Failed to send disconnect request");
294  error = clib_error_return_unix (0, "send fd %d",
295  mif->connection.fd);
296  memif_disconnect (vm, mif);
297  }
298  }
299  }
300 
301  return error;
302 }
303 
304 static clib_error_t *
306  u32 hw_if_index,
307  struct vnet_sw_interface_t *st, int is_add)
308 {
309  /* Nothing for now */
310  return 0;
311 }
312 
313 /* *INDENT-OFF* */
315  .name = "memif",
316  .tx_function = memif_interface_tx,
317  .format_device_name = format_memif_device_name,
318  .format_device = format_memif_device,
319  .format_tx_trace = format_memif_tx_trace,
320  .tx_function_n_errors = MEMIF_TX_N_ERROR,
321  .tx_function_error_strings = memif_tx_func_error_strings,
322  .rx_redirect_to_node = memif_set_interface_next_node,
323  .clear_counters = memif_clear_hw_interface_counters,
324  .admin_up_down_function = memif_interface_admin_up_down,
325  .subif_add_del_function = memif_subif_add_del_function,
326 };
327 
330 /* *INDENT-ON* */
331 
332 /*
333  * fd.io coding-style-patch-verification: ON
334  *
335  * Local Variables:
336  * eval: (c-set-style "gnu")
337  * End:
338  */
memif_if_t * interfaces
Definition: memif.h:143
void memif_disconnect(vlib_main_t *vm, memif_if_t *mif)
Definition: memif.c:89
static_always_inline void memif_interface_unlock(memif_if_t *mif)
Definition: device.c:92
memif_ring_type_t
Definition: memif.h:218
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
#define CLIB_UNUSED(x)
Definition: clib.h:79
memif_desc_t desc[0]
Definition: memif.h:61
memif_tx_func_error_t
Definition: device.c:34
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:290
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:463
volatile u32 * lockp
Definition: memif.h:101
static uword memif_interface_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: device.c:233
#define PREDICT_TRUE(x)
Definition: clib.h:98
static_always_inline void * memif_get_buffer(memif_if_t *mif, memif_ring_t *ring, u16 slot)
Definition: memif.h:240
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:104
static void vlib_error_count(vlib_main_t *vm, uword node_index, uword counter, uword increment)
Definition: error_funcs.h:57
vlib_node_registration_t memif_input_node
(constructor) VLIB_REGISTER_NODE (memif_input_node)
Definition: node.c:358
static void memif_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:247
#define MEMIF_MSG_TYPE_DISCONNECT
Definition: memif.h:27
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u16 head
Definition: memif.h:59
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:418
memif_file_t connection
Definition: memif.h:117
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1062
u8 type
Definition: memif.h:24
#define MEMIF_IF_FLAG_IS_SLAVE
Definition: memif.h:104
VNET_DEVICE_CLASS(af_packet_device_class)
#define static_always_inline
Definition: clib.h:85
static_always_inline void memif_interface_lock(memif_if_t *mif)
Definition: device.c:82
static uword format_get_indent(u8 *s)
Definition: format.h:72
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
u8 log2_ring_size
Definition: memif.h:123
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:164
memif_file_t interrupt_line
Definition: memif.h:118
u32 per_interface_next_index
Definition: memif.h:114
static clib_error_t * memif_subif_add_del_function(vnet_main_t *vnm, u32 hw_if_index, struct vnet_sw_interface_t *st, int is_add)
Definition: device.c:305
static void memif_clear_hw_interface_counters(u32 instance)
Definition: device.c:266
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:397
static char * memif_tx_func_error_strings[]
Definition: device.c:42
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:71
#define clib_error_return_unix(e, args...)
Definition: error.h:114
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:188
#define PREDICT_FALSE(x)
Definition: clib.h:97
u32 node_index
Node index.
Definition: node.h:436
static u8 * format_memif_tx_trace(u8 *s, va_list *args)
Definition: device.c:75
#define foreach_memif_tx_func_error
Definition: device.c:30
static_always_inline uword memif_interface_tx_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, memif_if_t *mif, memif_ring_type_t type)
Definition: device.c:107
u16 n_vectors
Definition: node.h:344
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
vlib_main_t * vm
Definition: buffer.c:276
#define MEMIF_IF_FLAG_ADMIN_UP
Definition: memif.h:103
static_always_inline void memif_prefetch_buffer_and_data(vlib_main_t *vm, u32 bi)
Definition: device.c:99
#define clib_memcpy(a, b, c)
Definition: string.h:69
static clib_error_t * memif_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:272
#define VNET_SW_INTERFACE_FLAG_ADMIN_UP
Definition: interface.h:536
static u8 * format_memif_device_name(u8 *s, va_list *args)
Definition: device.c:50
unsigned int u32
Definition: types.h:88
u32 flags
Definition: memif.h:102
u32 length
Definition: memif.h:49
static void * vlib_frame_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:270
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
u64 uword
Definition: types.h:112
vnet_device_class_t memif_device_class
#define MEMIF_IF_FLAG_DELETING
Definition: memif.h:107
unsigned short u16
Definition: types.h:57
u32 index
Definition: memif.h:79
unsigned char u8
Definition: types.h:56
#define clib_unix_warning(format, args...)
Definition: error.h:68
#define MEMIF_VERSION
Definition: memif.h:23
static_always_inline memif_ring_t * memif_get_ring(memif_if_t *mif, memif_ring_type_t type, u16 ring_num)
Definition: memif.h:225
u16 version
Definition: memif.h:20
static u8 * format_memif_device(u8 *s, va_list *args)
Definition: device.c:59
u8 data[0]
Packet data.
Definition: buffer.h:152
u16 tail
Definition: memif.h:60
int fd
Definition: memif.h:78
u32 flags
Definition: vhost-user.h:78
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
memif_main_t memif_main
Definition: memif.c:47
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
#define VLIB_DEVICE_TX_FUNCTION_MULTIARCH(dev, fn)
Definition: interface.h:217