FD.io VPP  v16.12-rc0-308-g931be3a
Vector Packet Processing
node.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vlib/vlib.h>
16 #include <vnet/vnet.h>
17 #include <vnet/pg/pg.h>
18 #include <vppinfra/error.h>
19 #include <vnet/ip/ip.h>
21 
22 typedef struct
23 {
27 
28 /* packet trace format function */
29 static u8 *
30 format_export_trace (u8 * s, va_list * args)
31 {
32  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
33  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
34  export_trace_t *t = va_arg (*args, export_trace_t *);
35 
36  s = format (s, "EXPORT: flow_label %d, next index %d",
37  t->flow_label, t->next_index);
38  return s;
39 }
40 
42 
43 #define foreach_export_error \
44 _(RECORDED, "Packets recorded for export")
45 
46 typedef enum
47 {
48 #define _(sym,str) EXPORT_ERROR_##sym,
50 #undef _
53 
54 static char *export_error_strings[] = {
55 #define _(sym,string) string,
57 #undef _
58 };
59 
60 typedef enum
61 {
65 
66 always_inline void
67 copy3cachelines (void *dst, const void *src, size_t n)
68 {
69 #if 0
71  {
72  /* Copy only the first 1/2 cache lines whatever is available */
73  if (n >= 64)
74  clib_mov64 ((u8 *) dst, (const u8 *) src);
75  if (n >= 128)
76  clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64);
77  return;
78  }
79  clib_mov64 ((u8 *) dst, (const u8 *) src);
80  clib_mov64 ((u8 *) dst + 64, (const u8 *) src + 64);
81  clib_mov64 ((u8 *) dst + 128, (const u8 *) src + 128);
82 #endif
83 #if 1
84 
85  u64 *copy_dst, *copy_src;
86  int i;
87  copy_dst = (u64 *) dst;
88  copy_src = (u64 *) src;
90  {
91  for (i = 0; i < n / 64; i++)
92  {
93  copy_dst[0] = copy_src[0];
94  copy_dst[1] = copy_src[1];
95  copy_dst[2] = copy_src[2];
96  copy_dst[3] = copy_src[3];
97  copy_dst[4] = copy_src[4];
98  copy_dst[5] = copy_src[5];
99  copy_dst[6] = copy_src[6];
100  copy_dst[7] = copy_src[7];
101  copy_dst += 8;
102  copy_src += 8;
103  }
104  return;
105  }
106  for (i = 0; i < 3; i++)
107  {
108  copy_dst[0] = copy_src[0];
109  copy_dst[1] = copy_src[1];
110  copy_dst[2] = copy_src[2];
111  copy_dst[3] = copy_src[3];
112  copy_dst[4] = copy_src[4];
113  copy_dst[5] = copy_src[5];
114  copy_dst[6] = copy_src[6];
115  copy_dst[7] = copy_src[7];
116  copy_dst += 8;
117  copy_src += 8;
118  }
119 #endif
120 }
121 
122 static uword
124  vlib_node_runtime_t * node, vlib_frame_t * frame)
125 {
127  u32 n_left_from, *from, *to_next;
128  export_next_t next_index;
129  u32 pkts_recorded = 0;
130  ioam_export_buffer_t *my_buf = 0;
131  vlib_buffer_t *eb0 = 0;
132  u32 ebi0 = 0;
133  from = vlib_frame_vector_args (frame);
134  n_left_from = frame->n_vectors;
135  next_index = node->cached_next_index;
136 
137  while (__sync_lock_test_and_set (em->lockp[vm->cpu_index], 1))
138  ;
139  my_buf = ioam_export_get_my_buffer (vm->cpu_index);
140  my_buf->touched_at = vlib_time_now (vm);
141  while (n_left_from > 0)
142  {
143  u32 n_left_to_next;
144 
145  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
146  while (n_left_from >= 4 && n_left_to_next >= 2)
147  {
148  u32 next0 = EXPORT_NEXT_POP_HBYH;
149  u32 next1 = EXPORT_NEXT_POP_HBYH;
150  u32 bi0, bi1;
151  ip6_header_t *ip60, *ip61;
152  vlib_buffer_t *p0, *p1;
153  u32 ip_len0, ip_len1;
154 
155  /* Prefetch next iteration. */
156  {
157  vlib_buffer_t *p2, *p3;
158 
159  p2 = vlib_get_buffer (vm, from[2]);
160  p3 = vlib_get_buffer (vm, from[3]);
161 
162  vlib_prefetch_buffer_header (p2, LOAD);
163  vlib_prefetch_buffer_header (p3, LOAD);
164 
165  /* IPv6 + HbyH header + Trace option */
166  /* 40 + 2 + [4 hdr] + [16]* no_of_nodes */
167  /* 3 cache lines can get v6 hdr + trace option with upto 9 node trace */
168  CLIB_PREFETCH (p2->data, 3 * CLIB_CACHE_LINE_BYTES, LOAD);
169  CLIB_PREFETCH (p3->data, 3 * CLIB_CACHE_LINE_BYTES, LOAD);
170  }
171 
172  /* speculatively enqueue p0 and p1 to the current next frame */
173  to_next[0] = bi0 = from[0];
174  to_next[1] = bi1 = from[1];
175  from += 2;
176  to_next += 2;
177  n_left_from -= 2;
178  n_left_to_next -= 2;
179 
180  p0 = vlib_get_buffer (vm, bi0);
181  p1 = vlib_get_buffer (vm, bi1);
182 
183  ip60 = vlib_buffer_get_current (p0);
184  ip61 = vlib_buffer_get_current (p1);
185 
186  ip_len0 =
187  clib_net_to_host_u16 (ip60->payload_length) +
188  sizeof (ip6_header_t);
189  ip_len1 =
190  clib_net_to_host_u16 (ip61->payload_length) +
191  sizeof (ip6_header_t);
192 
193  ebi0 = my_buf->buffer_index;
194  eb0 = vlib_get_buffer (vm, ebi0);
195  if (PREDICT_FALSE (eb0 == 0))
196  goto NO_BUFFER1;
197 
198  ip_len0 =
199  ip_len0 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len0;
200  ip_len1 =
201  ip_len1 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len1;
202 
203  copy3cachelines (eb0->data + eb0->current_length, ip60, ip_len0);
205  /* To maintain uniform size per export, each
206  * record is default size, ip6 hdr can be
207  * used to parse the record correctly
208  */
209  my_buf->records_in_this_buffer++;
210  /* if number of buf exceeds max that fits in a MTU sized buffer
211  * ship it to the queue and pick new one
212  */
214  {
215  ioam_export_send_buffer (vm, my_buf);
216  ioam_export_init_buffer (vm, my_buf);
217  }
218 
219  ebi0 = my_buf->buffer_index;
220  eb0 = vlib_get_buffer (vm, ebi0);
221  if (PREDICT_FALSE (eb0 == 0))
222  goto NO_BUFFER1;
223 
224  copy3cachelines (eb0->data + eb0->current_length, ip61, ip_len1);
226  my_buf->records_in_this_buffer++;
228  {
229  ioam_export_send_buffer (vm, my_buf);
230  ioam_export_init_buffer (vm, my_buf);
231  }
232 
233  pkts_recorded += 2;
234 
235  if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
236  {
237  if (p0->flags & VLIB_BUFFER_IS_TRACED)
238  {
239  export_trace_t *t =
240  vlib_add_trace (vm, node, p0, sizeof (*t));
241  t->flow_label =
242  clib_net_to_host_u32 (ip60->
243  ip_version_traffic_class_and_flow_label);
244  t->next_index = next0;
245  }
246  if (p1->flags & VLIB_BUFFER_IS_TRACED)
247  {
248  export_trace_t *t =
249  vlib_add_trace (vm, node, p1, sizeof (*t));
250  t->flow_label =
251  clib_net_to_host_u32 (ip61->
252  ip_version_traffic_class_and_flow_label);
253  t->next_index = next1;
254  }
255  }
256  NO_BUFFER1:
257  /* verify speculative enqueues, maybe switch current next frame */
258  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
259  to_next, n_left_to_next,
260  bi0, bi1, next0, next1);
261  }
262 
263  while (n_left_from > 0 && n_left_to_next > 0)
264  {
265  u32 bi0;
266  vlib_buffer_t *p0;
267  u32 next0 = EXPORT_NEXT_POP_HBYH;
268  ip6_header_t *ip60;
269  u32 ip_len0;
270 
271  /* speculatively enqueue p0 to the current next frame */
272  bi0 = from[0];
273  to_next[0] = bi0;
274  from += 1;
275  to_next += 1;
276  n_left_from -= 1;
277  n_left_to_next -= 1;
278 
279  p0 = vlib_get_buffer (vm, bi0);
280  ip60 = vlib_buffer_get_current (p0);
281  ip_len0 =
282  clib_net_to_host_u16 (ip60->payload_length) +
283  sizeof (ip6_header_t);
284 
285  ebi0 = my_buf->buffer_index;
286  eb0 = vlib_get_buffer (vm, ebi0);
287  if (PREDICT_FALSE (eb0 == 0))
288  goto NO_BUFFER;
289 
290  ip_len0 =
291  ip_len0 > DEFAULT_EXPORT_SIZE ? DEFAULT_EXPORT_SIZE : ip_len0;
292  copy3cachelines (eb0->data + eb0->current_length, ip60, ip_len0);
294  /* To maintain uniform size per export, each
295  * record is default size, ip6 hdr can be
296  * used to parse the record correctly
297  */
298  my_buf->records_in_this_buffer++;
299  /* if number of buf exceeds max that fits in a MTU sized buffer
300  * ship it to the queue and pick new one
301  */
303  {
304  ioam_export_send_buffer (vm, my_buf);
305  ioam_export_init_buffer (vm, my_buf);
306  }
308  && (p0->flags & VLIB_BUFFER_IS_TRACED)))
309  {
310  export_trace_t *t = vlib_add_trace (vm, node, p0, sizeof (*t));
311  t->flow_label =
312  clib_net_to_host_u32 (ip60->
313  ip_version_traffic_class_and_flow_label);
314  t->next_index = next0;
315  }
316 
317  pkts_recorded += 1;
318  NO_BUFFER:
319  /* verify speculative enqueue, maybe switch current next frame */
320  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
321  to_next, n_left_to_next,
322  bi0, next0);
323  }
324 
325  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
326  }
327 
329  EXPORT_ERROR_RECORDED, pkts_recorded);
330  *em->lockp[vm->cpu_index] = 0;
331  return frame->n_vectors;
332 }
333 
334 /*
335  * Node for IP6 export
336  */
338 {
339  .function = ip6_export_node_fn,
340  .name = "ip6-export",
341  .vector_size = sizeof (u32),
342  .format_trace = format_export_trace,
344  .n_errors = ARRAY_LEN (export_error_strings),
345  .error_strings = export_error_strings,
346  .n_next_nodes = EXPORT_N_NEXT,
347  /* edit / add dispositions here */
348  .next_nodes =
349  {
350  [EXPORT_NEXT_POP_HBYH] = "ip6-pop-hop-by-hop"
351  },
352 };
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:457
#define foreach_export_error
Definition: node.c:43
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
#define DEFAULT_EXPORT_RECORDS
Definition: ioam_export.h:84
#define CLIB_UNUSED(x)
Definition: clib.h:79
static void clib_mov64(u8 *dst, const u8 *src)
Definition: memcpy_avx.h:70
bad routing header type(not 4)") sr_error (NO_MORE_SEGMENTS
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:182
static u8 * format_export_trace(u8 *s, va_list *args)
Definition: node.c:30
u32 next_index
Definition: node.c:24
struct _vlib_node_registration vlib_node_registration_t
static int ioam_export_send_buffer(vlib_main_t *vm, ioam_export_buffer_t *eb)
Definition: ioam_export.h:266
static void copy3cachelines(void *dst, const void *src, size_t n)
Definition: node.c:67
#define always_inline
Definition: clib.h:84
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:190
u32 cpu_index
Definition: main.h:159
unsigned long u64
Definition: types.h:89
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:82
vlib_node_registration_t export_node
(constructor) VLIB_REGISTER_NODE (export_node)
Definition: node.c:41
volatile u32 ** lockp
Definition: ioam_export.h:59
export_next_t
Definition: node.c:60
#define PREDICT_FALSE(x)
Definition: clib.h:97
u32 flow_label
Definition: node.c:25
static ioam_export_buffer_t * ioam_export_get_my_buffer(u32 thread_id)
Definition: ioam_export.h:86
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:216
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:350
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1113
u16 n_vectors
Definition: node.h:344
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
#define ARRAY_LEN(x)
Definition: clib.h:59
static uword ip6_export_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: node.c:123
static int ioam_export_init_buffer(vlib_main_t *vm, ioam_export_buffer_t *eb)
Definition: ioam_export.h:105
u16 cached_next_index
Definition: node.h:463
unsigned int u32
Definition: types.h:88
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:259
#define VLIB_BUFFER_IS_TRACED
Definition: buffer.h:95
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
static char * export_error_strings[]
Definition: node.c:54
u16 payload_length
Definition: ip6_packet.h:291
unsigned char u8
Definition: types.h:56
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:253
#define DEFAULT_EXPORT_SIZE
Definition: ioam_export.h:79
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:166
export_error_t
Definition: node.c:46
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:418
u8 data[0]
Packet data.
Definition: buffer.h:154
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:85
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:69
ioam_export_main_t ioam_export_main
Definition: ioam_export.h:75