FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
decap.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 SUSE LLC.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vlib/vlib.h>
17 #include <vnet/pg/pg.h>
18 #include <vnet/geneve/geneve.h>
19 
22 
23 typedef struct
24 {
30 
31 static u8 *
32 format_geneve_rx_trace (u8 * s, va_list * args)
33 {
34  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
35  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
36  geneve_rx_trace_t *t = va_arg (*args, geneve_rx_trace_t *);
37 
38  if (t->tunnel_index != ~0)
39  {
40  s =
41  format (s,
42  "GENEVE decap from geneve_tunnel%d vni %d next %d error %d",
43  t->tunnel_index, t->vni_rsvd, t->next_index, t->error);
44  }
45  else
46  {
47  s = format (s, "GENEVE decap error - tunnel for vni %d does not exist",
48  t->vni_rsvd);
49  }
50  return s;
51 }
52 
55 {
56  u32 fib_index, sw_if_index;
57 
58  sw_if_index = vnet_buffer (b)->sw_if_index[VLIB_RX];
59 
60  if (is_ip4)
61  fib_index = (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
63  vnet_buffer (b)->sw_if_index[VLIB_TX];
64  else
65  fib_index = (vnet_buffer (b)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
67  vnet_buffer (b)->sw_if_index[VLIB_TX];
68 
69  return (fib_index == t->encap_fib_index);
70 }
71 
74  vlib_node_runtime_t * node,
75  vlib_frame_t * from_frame, u32 is_ip4)
76 {
77  u32 n_left_from, next_index, *from, *to_next;
78  geneve_main_t *vxm = &geneve_main;
79  vnet_main_t *vnm = vxm->vnet_main;
81  u32 last_tunnel_index = ~0;
82  geneve4_tunnel_key_t last_key4;
83  geneve6_tunnel_key_t last_key6;
84  u32 pkts_decapsulated = 0;
85  u32 thread_index = vm->thread_index;
86  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
87 
88  if (is_ip4)
89  last_key4.as_u64 = ~0;
90  else
91  clib_memset (&last_key6, 0xff, sizeof (last_key6));
92 
93  from = vlib_frame_vector_args (from_frame);
94  n_left_from = from_frame->n_vectors;
95 
96  next_index = node->cached_next_index;
97  stats_sw_if_index = node->runtime_data[0];
98  stats_n_packets = stats_n_bytes = 0;
99 
100  while (n_left_from > 0)
101  {
102  u32 n_left_to_next;
103 
104  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
105  while (n_left_from >= 4 && n_left_to_next >= 2)
106  {
107  u32 bi0, bi1;
108  vlib_buffer_t *b0, *b1;
109  u32 next0, next1;
110  ip4_header_t *ip4_0, *ip4_1;
111  ip6_header_t *ip6_0, *ip6_1;
112  geneve_header_t *geneve0, *geneve1;
113  uword *p0, *p1;
114  u32 tunnel_index0, tunnel_index1;
115  geneve_tunnel_t *t0, *t1, *mt0 = NULL, *mt1 = NULL;
116  geneve4_tunnel_key_t key4_0, key4_1;
117  geneve6_tunnel_key_t key6_0, key6_1;
118  u32 error0, error1;
119  u32 sw_if_index0, sw_if_index1, len0, len1;
120 
121  /* Prefetch next iteration. */
122  {
123  vlib_buffer_t *p2, *p3;
124 
125  p2 = vlib_get_buffer (vm, from[2]);
126  p3 = vlib_get_buffer (vm, from[3]);
127 
128  vlib_prefetch_buffer_header (p2, LOAD);
129  vlib_prefetch_buffer_header (p3, LOAD);
130 
131  CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
132  CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
133  }
134 
135  bi0 = from[0];
136  bi1 = from[1];
137  to_next[0] = bi0;
138  to_next[1] = bi1;
139  from += 2;
140  to_next += 2;
141  n_left_to_next -= 2;
142  n_left_from -= 2;
143 
144  b0 = vlib_get_buffer (vm, bi0);
145  b1 = vlib_get_buffer (vm, bi1);
146 
147  /* udp leaves current_data pointing at the geneve header */
148  geneve0 = vlib_buffer_get_current (b0);
149  geneve1 = vlib_buffer_get_current (b1);
150 
151  vnet_geneve_hdr_1word_ntoh (geneve0);
152  vnet_geneve_hdr_1word_ntoh (geneve1);
153 
154  if (is_ip4)
155  {
157  (b0, -(word) (sizeof (udp_header_t) + sizeof (ip4_header_t)));
159  (b1, -(word) (sizeof (udp_header_t) + sizeof (ip4_header_t)));
160  ip4_0 = vlib_buffer_get_current (b0);
161  ip4_1 = vlib_buffer_get_current (b1);
162  }
163  else
164  {
166  (b0, -(word) (sizeof (udp_header_t) + sizeof (ip6_header_t)));
168  (b1, -(word) (sizeof (udp_header_t) + sizeof (ip6_header_t)));
169  ip6_0 = vlib_buffer_get_current (b0);
170  ip6_1 = vlib_buffer_get_current (b1);
171  }
172 
173  /* pop (ip, udp, geneve) */
174  if (is_ip4)
175  {
177  (b0,
178  sizeof (*ip4_0) + sizeof (udp_header_t) +
180  vnet_get_geneve_options_len (geneve0));
182  sizeof (*ip4_1) + sizeof (udp_header_t) +
184  vnet_get_geneve_options_len (geneve1));
185  }
186  else
187  {
189  (b0,
190  sizeof (*ip6_0) + sizeof (udp_header_t) +
192  vnet_get_geneve_options_len (geneve0));
194  sizeof (*ip6_0) + sizeof (udp_header_t) +
196  vnet_get_geneve_options_len (geneve1));
197  }
198 
199  tunnel_index0 = ~0;
200  error0 = 0;
201 
202  tunnel_index1 = ~0;
203  error1 = 0;
204 
205  if (PREDICT_FALSE
207  {
208  error0 = GENEVE_ERROR_BAD_FLAGS;
209  next0 = GENEVE_INPUT_NEXT_DROP;
210  goto trace0;
211  }
212 #if SUPPORT_OPTIONS_HEADER==1
213  if (PREDICT_FALSE (vnet_get_geneve_critical_bit (geneve0) == 1))
214  {
215  error0 = GENEVE_ERROR_BAD_FLAGS;
216  next0 = GENEVE_INPUT_NEXT_DROP;
217  goto trace0;
218  }
219 #endif
220  if (is_ip4)
221  {
222  key4_0.remote = ip4_0->src_address.as_u32;
223  key4_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
224 
225  /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
226  if (PREDICT_FALSE (key4_0.as_u64 != last_key4.as_u64))
227  {
228  p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
229  if (PREDICT_FALSE (p0 == NULL))
230  {
231  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
232  next0 = GENEVE_INPUT_NEXT_DROP;
233  goto trace0;
234  }
235  last_key4.as_u64 = key4_0.as_u64;
236  tunnel_index0 = last_tunnel_index = p0[0];
237  }
238  else
239  tunnel_index0 = last_tunnel_index;
240  t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
241 
242  /* Validate GENEVE tunnel encap-fib index agaist packet */
243  if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
244  {
245  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
246  next0 = GENEVE_INPUT_NEXT_DROP;
247  goto trace0;
248  }
249 
250  /* Validate GENEVE tunnel SIP against packet DIP */
251  if (PREDICT_TRUE
252  (ip4_0->dst_address.as_u32 == t0->local.ip4.as_u32))
253  goto next0; /* valid packet */
254  if (PREDICT_FALSE
256  {
257  key4_0.remote = ip4_0->dst_address.as_u32;
258  key4_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
259  /* Make sure mcast GENEVE tunnel exist by packet DIP and VNI */
260  p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
261  if (PREDICT_TRUE (p0 != NULL))
262  {
263  mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
264  goto next0; /* valid packet */
265  }
266  }
267  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
268  next0 = GENEVE_INPUT_NEXT_DROP;
269  goto trace0;
270 
271  }
272  else /* !is_ip4 */
273  {
274  key6_0.remote.as_u64[0] = ip6_0->src_address.as_u64[0];
275  key6_0.remote.as_u64[1] = ip6_0->src_address.as_u64[1];
276  key6_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
277 
278  /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
279  if (PREDICT_FALSE
280  (memcmp (&key6_0, &last_key6, sizeof (last_key6)) != 0))
281  {
282  p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
283  if (PREDICT_FALSE (p0 == NULL))
284  {
285  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
286  next0 = GENEVE_INPUT_NEXT_DROP;
287  goto trace0;
288  }
289  clib_memcpy_fast (&last_key6, &key6_0, sizeof (key6_0));
290  tunnel_index0 = last_tunnel_index = p0[0];
291  }
292  else
293  tunnel_index0 = last_tunnel_index;
294  t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
295 
296  /* Validate GENEVE tunnel encap-fib index agaist packet */
297  if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
298  {
299  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
300  next0 = GENEVE_INPUT_NEXT_DROP;
301  goto trace0;
302  }
303 
304  /* Validate GENEVE tunnel SIP against packet DIP */
306  &t0->local.ip6)))
307  goto next0; /* valid packet */
308  if (PREDICT_FALSE
310  {
311  key6_0.remote.as_u64[0] = ip6_0->dst_address.as_u64[0];
312  key6_0.remote.as_u64[1] = ip6_0->dst_address.as_u64[1];
313  key6_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
314  p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
315  if (PREDICT_TRUE (p0 != NULL))
316  {
317  mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
318  goto next0; /* valid packet */
319  }
320  }
321  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
322  next0 = GENEVE_INPUT_NEXT_DROP;
323  goto trace0;
324  }
325 
326  next0:
327  next0 = t0->decap_next_index;
328  sw_if_index0 = t0->sw_if_index;
329  len0 = vlib_buffer_length_in_chain (vm, b0);
330 
331  /* Required to make the l2 tag push / pop code work on l2 subifs */
332  if (PREDICT_TRUE (next0 == GENEVE_INPUT_NEXT_L2_INPUT))
333  vnet_update_l2_len (b0);
334 
335  /* Set packet input sw_if_index to unicast GENEVE tunnel for learning */
336  vnet_buffer (b0)->sw_if_index[VLIB_RX] = sw_if_index0;
337  sw_if_index0 = (mt0) ? mt0->sw_if_index : sw_if_index0;
338 
339  pkts_decapsulated++;
340  stats_n_packets += 1;
341  stats_n_bytes += len0;
342 
343  /* Batch stats increment on the same geneve tunnel so counter
344  is not incremented per packet */
345  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
346  {
347  stats_n_packets -= 1;
348  stats_n_bytes -= len0;
349  if (stats_n_packets)
352  thread_index, stats_sw_if_index,
353  stats_n_packets, stats_n_bytes);
354  stats_n_packets = 1;
355  stats_n_bytes = len0;
356  stats_sw_if_index = sw_if_index0;
357  }
358 
359  trace0:
360  b0->error = error0 ? node->errors[error0] : 0;
361 
362  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
363  {
365  = vlib_add_trace (vm, node, b0, sizeof (*tr));
366  tr->next_index = next0;
367  tr->error = error0;
368  tr->tunnel_index = tunnel_index0;
369  tr->vni_rsvd = vnet_get_geneve_vni (geneve0);
370  }
371 
372  if (PREDICT_FALSE
374  {
375  error1 = GENEVE_ERROR_BAD_FLAGS;
376  next1 = GENEVE_INPUT_NEXT_DROP;
377  goto trace1;
378  }
379 #if SUPPORT_OPTIONS_HEADER==1
380  if (PREDICT_FALSE (vnet_get_geneve_critical_bit (geneve1) == 1))
381  {
382  error1 = GENEVE_ERROR_BAD_FLAGS;
383  next1 = GENEVE_INPUT_NEXT_DROP;
384  goto trace1;
385  }
386 #endif
387  if (is_ip4)
388  {
389  key4_1.remote = ip4_1->src_address.as_u32;
390  key4_1.vni = vnet_get_geneve_vni_bigendian (geneve1);
391 
392  /* Make sure unicast GENEVE tunnel exist by packet SIP and VNI */
393  if (PREDICT_FALSE (key4_1.as_u64 != last_key4.as_u64))
394  {
395  p1 = hash_get (vxm->geneve4_tunnel_by_key, key4_1.as_u64);
396  if (PREDICT_FALSE (p1 == NULL))
397  {
398  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
399  next1 = GENEVE_INPUT_NEXT_DROP;
400  goto trace1;
401  }
402  last_key4.as_u64 = key4_1.as_u64;
403  tunnel_index1 = last_tunnel_index = p1[0];
404  }
405  else
406  tunnel_index1 = last_tunnel_index;
407  t1 = pool_elt_at_index (vxm->tunnels, tunnel_index1);
408 
409  /* Validate GENEVE tunnel encap-fib index agaist packet */
410  if (PREDICT_FALSE (validate_geneve_fib (b1, t1, is_ip4) == 0))
411  {
412  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
413  next1 = GENEVE_INPUT_NEXT_DROP;
414  goto trace1;
415  }
416 
417  /* Validate GENEVE tunnel SIP against packet DIP */
418  if (PREDICT_TRUE
419  (ip4_1->dst_address.as_u32 == t1->local.ip4.as_u32))
420  goto next1; /* valid packet */
421  if (PREDICT_FALSE
423  {
424  key4_1.remote = ip4_1->dst_address.as_u32;
425  key4_1.vni = vnet_get_geneve_vni_bigendian (geneve1);
426  /* Make sure mcast GENEVE tunnel exist by packet DIP and VNI */
427  p1 = hash_get (vxm->geneve4_tunnel_by_key, key4_1.as_u64);
428  if (PREDICT_TRUE (p1 != NULL))
429  {
430  mt1 = pool_elt_at_index (vxm->tunnels, p1[0]);
431  goto next1; /* valid packet */
432  }
433  }
434  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
435  next1 = GENEVE_INPUT_NEXT_DROP;
436  goto trace1;
437 
438  }
439  else /* !is_ip4 */
440  {
441  key6_1.remote.as_u64[0] = ip6_1->src_address.as_u64[0];
442  key6_1.remote.as_u64[1] = ip6_1->src_address.as_u64[1];
443  key6_1.vni = vnet_get_geneve_vni_bigendian (geneve1);
444 
445  /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
446  if (PREDICT_FALSE
447  (memcmp (&key6_1, &last_key6, sizeof (last_key6)) != 0))
448  {
449  p1 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_1);
450 
451  if (PREDICT_FALSE (p1 == NULL))
452  {
453  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
454  next1 = GENEVE_INPUT_NEXT_DROP;
455  goto trace1;
456  }
457 
458  clib_memcpy_fast (&last_key6, &key6_1, sizeof (key6_1));
459  tunnel_index1 = last_tunnel_index = p1[0];
460  }
461  else
462  tunnel_index1 = last_tunnel_index;
463  t1 = pool_elt_at_index (vxm->tunnels, tunnel_index1);
464 
465  /* Validate GENEVE tunnel encap-fib index agaist packet */
466  if (PREDICT_FALSE (validate_geneve_fib (b1, t1, is_ip4) == 0))
467  {
468  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
469  next1 = GENEVE_INPUT_NEXT_DROP;
470  goto trace1;
471  }
472 
473  /* Validate GENEVE tunnel SIP against packet DIP */
475  &t1->local.ip6)))
476  goto next1; /* valid packet */
477  if (PREDICT_FALSE
479  {
480  key6_1.remote.as_u64[0] = ip6_1->dst_address.as_u64[0];
481  key6_1.remote.as_u64[1] = ip6_1->dst_address.as_u64[1];
482  key6_1.vni = vnet_get_geneve_vni_bigendian (geneve1);
483  p1 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_1);
484  if (PREDICT_TRUE (p1 != NULL))
485  {
486  mt1 = pool_elt_at_index (vxm->tunnels, p1[0]);
487  goto next1; /* valid packet */
488  }
489  }
490  error1 = GENEVE_ERROR_NO_SUCH_TUNNEL;
491  next1 = GENEVE_INPUT_NEXT_DROP;
492  goto trace1;
493  }
494 
495  next1:
496  next1 = t1->decap_next_index;
497  sw_if_index1 = t1->sw_if_index;
498  len1 = vlib_buffer_length_in_chain (vm, b1);
499 
500  /* Required to make the l2 tag push / pop code work on l2 subifs */
501  if (PREDICT_TRUE (next1 == GENEVE_INPUT_NEXT_L2_INPUT))
502  vnet_update_l2_len (b1);
503 
504  /* Set packet input sw_if_index to unicast GENEVE tunnel for learning */
505  vnet_buffer (b1)->sw_if_index[VLIB_RX] = sw_if_index1;
506  sw_if_index1 = (mt1) ? mt1->sw_if_index : sw_if_index1;
507 
508  pkts_decapsulated++;
509  stats_n_packets += 1;
510  stats_n_bytes += len1;
511 
512  /* Batch stats increment on the same geneve tunnel so counter
513  is not incremented per packet */
514  if (PREDICT_FALSE (sw_if_index1 != stats_sw_if_index))
515  {
516  stats_n_packets -= 1;
517  stats_n_bytes -= len1;
518  if (stats_n_packets)
521  thread_index, stats_sw_if_index,
522  stats_n_packets, stats_n_bytes);
523  stats_n_packets = 1;
524  stats_n_bytes = len1;
525  stats_sw_if_index = sw_if_index1;
526  }
527 
528  trace1:
529  b1->error = error1 ? node->errors[error1] : 0;
530 
531  if (PREDICT_FALSE (b1->flags & VLIB_BUFFER_IS_TRACED))
532  {
534  = vlib_add_trace (vm, node, b1, sizeof (*tr));
535  tr->next_index = next1;
536  tr->error = error1;
537  tr->tunnel_index = tunnel_index1;
538  tr->vni_rsvd = vnet_get_geneve_vni (geneve1);
539  }
540 
541  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
542  to_next, n_left_to_next,
543  bi0, bi1, next0, next1);
544  }
545 
546  while (n_left_from > 0 && n_left_to_next > 0)
547  {
548  u32 bi0;
549  vlib_buffer_t *b0;
550  u32 next0;
551  ip4_header_t *ip4_0;
552  ip6_header_t *ip6_0;
553  geneve_header_t *geneve0;
554  uword *p0;
555  u32 tunnel_index0;
556  geneve_tunnel_t *t0, *mt0 = NULL;
557  geneve4_tunnel_key_t key4_0;
558  geneve6_tunnel_key_t key6_0;
559  u32 error0;
560  u32 sw_if_index0, len0;
561 
562  bi0 = from[0];
563  to_next[0] = bi0;
564  from += 1;
565  to_next += 1;
566  n_left_from -= 1;
567  n_left_to_next -= 1;
568 
569  b0 = vlib_get_buffer (vm, bi0);
570 
571  /* udp leaves current_data pointing at the geneve header */
572  geneve0 = vlib_buffer_get_current (b0);
573  vnet_geneve_hdr_1word_ntoh (geneve0);
574 
575  if (is_ip4)
576  {
578  (b0, -(word) (sizeof (udp_header_t) + sizeof (ip4_header_t)));
579  ip4_0 = vlib_buffer_get_current (b0);
580  }
581  else
582  {
584  (b0, -(word) (sizeof (udp_header_t) + sizeof (ip6_header_t)));
585  ip6_0 = vlib_buffer_get_current (b0);
586  }
587 
588  /* pop (ip, udp, geneve) */
589  if (is_ip4)
590  {
592  (b0,
593  sizeof (*ip4_0) + sizeof (udp_header_t) +
595  vnet_get_geneve_options_len (geneve0));
596  }
597  else
598  {
600  (b0,
601  sizeof (*ip6_0) + sizeof (udp_header_t) +
603  vnet_get_geneve_options_len (geneve0));
604  }
605 
606  tunnel_index0 = ~0;
607  error0 = 0;
608 
609  if (PREDICT_FALSE
611  {
612  error0 = GENEVE_ERROR_BAD_FLAGS;
613  next0 = GENEVE_INPUT_NEXT_DROP;
614  goto trace00;
615  }
616 #if SUPPORT_OPTIONS_HEADER==1
617  if (PREDICT_FALSE (vnet_get_geneve_critical_bit (geneve0) == 1))
618  {
619  error0 = GENEVE_ERROR_BAD_FLAGS;
620  next0 = GENEVE_INPUT_NEXT_DROP;
621  goto trace00;
622  }
623 #endif
624  if (is_ip4)
625  {
626  key4_0.remote = ip4_0->src_address.as_u32;
627  key4_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
628 
629  /* Make sure unicast GENEVE tunnel exist by packet SIP and VNI */
630  if (PREDICT_FALSE (key4_0.as_u64 != last_key4.as_u64))
631  {
632  p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
633  if (PREDICT_FALSE (p0 == NULL))
634  {
635  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
636  next0 = GENEVE_INPUT_NEXT_DROP;
637  goto trace00;
638  }
639  last_key4.as_u64 = key4_0.as_u64;
640  tunnel_index0 = last_tunnel_index = p0[0];
641  }
642  else
643  tunnel_index0 = last_tunnel_index;
644  t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
645 
646  /* Validate GENEVE tunnel encap-fib index agaist packet */
647  if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
648  {
649  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
650  next0 = GENEVE_INPUT_NEXT_DROP;
651  goto trace00;
652  }
653 
654  /* Validate GENEVE tunnel SIP against packet DIP */
655  if (PREDICT_TRUE
656  (ip4_0->dst_address.as_u32 == t0->local.ip4.as_u32))
657  goto next00; /* valid packet */
658  if (PREDICT_FALSE
660  {
661  key4_0.remote = ip4_0->dst_address.as_u32;
662  key4_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
663  /* Make sure mcast GENEVE tunnel exist by packet DIP and VNI */
664  p0 = hash_get (vxm->geneve4_tunnel_by_key, key4_0.as_u64);
665  if (PREDICT_TRUE (p0 != NULL))
666  {
667  mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
668  goto next00; /* valid packet */
669  }
670  }
671  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
672  next0 = GENEVE_INPUT_NEXT_DROP;
673  goto trace00;
674 
675  }
676  else /* !is_ip4 */
677  {
678  key6_0.remote.as_u64[0] = ip6_0->src_address.as_u64[0];
679  key6_0.remote.as_u64[1] = ip6_0->src_address.as_u64[1];
680  key6_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
681 
682  /* Make sure GENEVE tunnel exist according to packet SIP and VNI */
683  if (PREDICT_FALSE
684  (memcmp (&key6_0, &last_key6, sizeof (last_key6)) != 0))
685  {
686  p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
687  if (PREDICT_FALSE (p0 == NULL))
688  {
689  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
690  next0 = GENEVE_INPUT_NEXT_DROP;
691  goto trace00;
692  }
693  clib_memcpy_fast (&last_key6, &key6_0, sizeof (key6_0));
694  tunnel_index0 = last_tunnel_index = p0[0];
695  }
696  else
697  tunnel_index0 = last_tunnel_index;
698  t0 = pool_elt_at_index (vxm->tunnels, tunnel_index0);
699 
700  /* Validate GENEVE tunnel encap-fib index agaist packet */
701  if (PREDICT_FALSE (validate_geneve_fib (b0, t0, is_ip4) == 0))
702  {
703  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
704  next0 = GENEVE_INPUT_NEXT_DROP;
705  goto trace00;
706  }
707 
708  /* Validate GENEVE tunnel SIP against packet DIP */
710  &t0->local.ip6)))
711  goto next00; /* valid packet */
712  if (PREDICT_FALSE
714  {
715  key6_0.remote.as_u64[0] = ip6_0->dst_address.as_u64[0];
716  key6_0.remote.as_u64[1] = ip6_0->dst_address.as_u64[1];
717  key6_0.vni = vnet_get_geneve_vni_bigendian (geneve0);
718  p0 = hash_get_mem (vxm->geneve6_tunnel_by_key, &key6_0);
719  if (PREDICT_TRUE (p0 != NULL))
720  {
721  mt0 = pool_elt_at_index (vxm->tunnels, p0[0]);
722  goto next00; /* valid packet */
723  }
724  }
725  error0 = GENEVE_ERROR_NO_SUCH_TUNNEL;
726  next0 = GENEVE_INPUT_NEXT_DROP;
727  goto trace00;
728  }
729 
730  next00:
731  next0 = t0->decap_next_index;
732  sw_if_index0 = t0->sw_if_index;
733  len0 = vlib_buffer_length_in_chain (vm, b0);
734 
735  /* Required to make the l2 tag push / pop code work on l2 subifs */
736  if (PREDICT_TRUE (next0 == GENEVE_INPUT_NEXT_L2_INPUT))
737  vnet_update_l2_len (b0);
738 
739  /* Set packet input sw_if_index to unicast GENEVE tunnel for learning */
740  vnet_buffer (b0)->sw_if_index[VLIB_RX] = sw_if_index0;
741  sw_if_index0 = (mt0) ? mt0->sw_if_index : sw_if_index0;
742 
743  pkts_decapsulated++;
744  stats_n_packets += 1;
745  stats_n_bytes += len0;
746 
747  /* Batch stats increment on the same geneve tunnel so counter
748  is not incremented per packet */
749  if (PREDICT_FALSE (sw_if_index0 != stats_sw_if_index))
750  {
751  stats_n_packets -= 1;
752  stats_n_bytes -= len0;
753  if (stats_n_packets)
756  thread_index, stats_sw_if_index,
757  stats_n_packets, stats_n_bytes);
758  stats_n_packets = 1;
759  stats_n_bytes = len0;
760  stats_sw_if_index = sw_if_index0;
761  }
762 
763  trace00:
764  b0->error = error0 ? node->errors[error0] : 0;
765 
766  if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
767  {
769  = vlib_add_trace (vm, node, b0, sizeof (*tr));
770  tr->next_index = next0;
771  tr->error = error0;
772  tr->tunnel_index = tunnel_index0;
773  tr->vni_rsvd = vnet_get_geneve_vni (geneve0);
774  }
775  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
776  to_next, n_left_to_next,
777  bi0, next0);
778  }
779 
780  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
781  }
782  /* Do we still need this now that tunnel tx stats is kept? */
783  vlib_node_increment_counter (vm, is_ip4 ?
785  index : geneve6_input_node.index,
786  GENEVE_ERROR_DECAPSULATED, pkts_decapsulated);
787 
788  /* Increment any remaining batch stats */
789  if (stats_n_packets)
790  {
793  thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
794  node->runtime_data[0] = stats_sw_if_index;
795  }
796 
797  return from_frame->n_vectors;
798 }
799 
800 static uword
802  vlib_node_runtime_t * node, vlib_frame_t * from_frame)
803 {
804  return geneve_input (vm, node, from_frame, /* is_ip4 */ 1);
805 }
806 
807 static uword
809  vlib_node_runtime_t * node, vlib_frame_t * from_frame)
810 {
811  return geneve_input (vm, node, from_frame, /* is_ip4 */ 0);
812 }
813 
814 static char *geneve_error_strings[] = {
815 #define geneve_error(n,s) s,
817 #undef geneve_error
818 #undef _
819 };
820 
821 /* *INDENT-OFF* */
823  .function = geneve4_input,
824  .name = "geneve4-input",
825  /* Takes a vector of packets. */
826  .vector_size = sizeof (u32),
827  .n_errors = GENEVE_N_ERROR,
828  .error_strings = geneve_error_strings,
829  .n_next_nodes = GENEVE_INPUT_N_NEXT,
830  .next_nodes = {
831 #define _(s,n) [GENEVE_INPUT_NEXT_##s] = n,
833 #undef _
834  },
835 
836 //temp .format_buffer = format_geneve_header,
837  .format_trace = format_geneve_rx_trace,
838  // $$$$ .unformat_buffer = unformat_geneve_header,
839 };
840 
842 
844  .function = geneve6_input,
845  .name = "geneve6-input",
846  /* Takes a vector of packets. */
847  .vector_size = sizeof (u32),
848  .n_errors = GENEVE_N_ERROR,
849  .error_strings = geneve_error_strings,
850  .n_next_nodes = GENEVE_INPUT_N_NEXT,
851  .next_nodes = {
852 #define _(s,n) [GENEVE_INPUT_NEXT_##s] = n,
854 #undef _
855  },
856 //temp .format_buffer = format_geneve_header,
857  .format_trace = format_geneve_rx_trace,
858  // $$$$ .unformat_buffer = unformat_geneve_header,
859 };
860 
862 /* *INDENT-ON* */
863 
864 typedef enum
865 {
870 
873  vlib_node_runtime_t * node,
874  vlib_frame_t * frame, u32 is_ip4)
875 {
876  geneve_main_t *vxm = &geneve_main;
877  u32 *from, *to_next, n_left_from, n_left_to_next, next_index;
878  vlib_node_runtime_t *error_node =
880  ip4_address_t addr4; /* last IPv4 address matching a local VTEP address */
881  ip6_address_t addr6; /* last IPv6 address matching a local VTEP address */
882 
883  from = vlib_frame_vector_args (frame);
884  n_left_from = frame->n_vectors;
885  next_index = node->cached_next_index;
886 
887  if (node->flags & VLIB_NODE_FLAG_TRACE)
888  ip4_forward_next_trace (vm, node, frame, VLIB_TX);
889 
890  if (is_ip4)
891  addr4.data_u32 = ~0;
892  else
893  ip6_address_set_zero (&addr6);
894 
895  while (n_left_from > 0)
896  {
897  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
898 
899  while (n_left_from >= 4 && n_left_to_next >= 2)
900  {
901  vlib_buffer_t *b0, *b1;
902  ip4_header_t *ip40, *ip41;
903  ip6_header_t *ip60, *ip61;
904  udp_header_t *udp0, *udp1;
905  u32 bi0, ip_len0, udp_len0, flags0, next0;
906  u32 bi1, ip_len1, udp_len1, flags1, next1;
907  i32 len_diff0, len_diff1;
908  u8 error0, good_udp0, proto0;
909  u8 error1, good_udp1, proto1;
910 
911  /* Prefetch next iteration. */
912  {
913  vlib_buffer_t *p2, *p3;
914 
915  p2 = vlib_get_buffer (vm, from[2]);
916  p3 = vlib_get_buffer (vm, from[3]);
917 
918  vlib_prefetch_buffer_header (p2, LOAD);
919  vlib_prefetch_buffer_header (p3, LOAD);
920 
921  CLIB_PREFETCH (p2->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
922  CLIB_PREFETCH (p3->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
923  }
924 
925  bi0 = to_next[0] = from[0];
926  bi1 = to_next[1] = from[1];
927  from += 2;
928  n_left_from -= 2;
929  to_next += 2;
930  n_left_to_next -= 2;
931 
932  b0 = vlib_get_buffer (vm, bi0);
933  b1 = vlib_get_buffer (vm, bi1);
934  if (is_ip4)
935  {
936  ip40 = vlib_buffer_get_current (b0);
937  ip41 = vlib_buffer_get_current (b1);
938  }
939  else
940  {
941  ip60 = vlib_buffer_get_current (b0);
942  ip61 = vlib_buffer_get_current (b1);
943  }
944 
945  /* Setup packet for next IP feature */
946  vnet_feature_next (&next0, b0);
947  vnet_feature_next (&next1, b1);
948 
949  if (is_ip4)
950  {
951  /* Treat IP frag packets as "experimental" protocol for now
952  until support of IP frag reassembly is implemented */
953  proto0 = ip4_is_fragment (ip40) ? 0xfe : ip40->protocol;
954  proto1 = ip4_is_fragment (ip41) ? 0xfe : ip41->protocol;
955  }
956  else
957  {
958  proto0 = ip60->protocol;
959  proto1 = ip61->protocol;
960  }
961 
962  /* Process packet 0 */
963  if (proto0 != IP_PROTOCOL_UDP)
964  goto exit0; /* not UDP packet */
965 
966  if (is_ip4)
967  udp0 = ip4_next_header (ip40);
968  else
969  udp0 = ip6_next_header (ip60);
970 
971  if (udp0->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_geneve))
972  goto exit0; /* not GENEVE packet */
973 
974  /* Validate DIP against VTEPs */
975  if (is_ip4)
976  {
977  if (addr4.as_u32 != ip40->dst_address.as_u32)
978  {
979  if (!hash_get (vxm->vtep4, ip40->dst_address.as_u32))
980  goto exit0; /* no local VTEP for GENEVE packet */
981  addr4 = ip40->dst_address;
982  }
983  }
984  else
985  {
986  if (!ip6_address_is_equal (&addr6, &ip60->dst_address))
987  {
988  if (!hash_get_mem (vxm->vtep6, &ip60->dst_address))
989  goto exit0; /* no local VTEP for GENEVE packet */
990  addr6 = ip60->dst_address;
991  }
992  }
993 
994  flags0 = b0->flags;
995  good_udp0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
996 
997  /* Don't verify UDP checksum for packets with explicit zero checksum. */
998  good_udp0 |= udp0->checksum == 0;
999 
1000  /* Verify UDP length */
1001  if (is_ip4)
1002  ip_len0 = clib_net_to_host_u16 (ip40->length);
1003  else
1004  ip_len0 = clib_net_to_host_u16 (ip60->payload_length);
1005  udp_len0 = clib_net_to_host_u16 (udp0->length);
1006  len_diff0 = ip_len0 - udp_len0;
1007 
1008  /* Verify UDP checksum */
1009  if (PREDICT_FALSE (!good_udp0))
1010  {
1011  if ((flags0 & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED) == 0)
1012  {
1013  if (is_ip4)
1014  flags0 = ip4_tcp_udp_validate_checksum (vm, b0);
1015  else
1016  flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
1017  good_udp0 =
1018  (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1019  }
1020  }
1021 
1022  if (is_ip4)
1023  {
1024  error0 = good_udp0 ? 0 : IP4_ERROR_UDP_CHECKSUM;
1025  error0 = (len_diff0 >= 0) ? error0 : IP4_ERROR_UDP_LENGTH;
1026  }
1027  else
1028  {
1029  error0 = good_udp0 ? 0 : IP6_ERROR_UDP_CHECKSUM;
1030  error0 = (len_diff0 >= 0) ? error0 : IP6_ERROR_UDP_LENGTH;
1031  }
1032 
1033  next0 = error0 ?
1035  b0->error = error0 ? error_node->errors[error0] : 0;
1036 
1037  /* geneve-input node expect current at GENEVE header */
1038  if (is_ip4)
1039  vlib_buffer_advance (b0,
1040  sizeof (ip4_header_t) +
1041  sizeof (udp_header_t));
1042  else
1043  vlib_buffer_advance (b0,
1044  sizeof (ip6_header_t) +
1045  sizeof (udp_header_t));
1046 
1047  exit0:
1048  /* Process packet 1 */
1049  if (proto1 != IP_PROTOCOL_UDP)
1050  goto exit1; /* not UDP packet */
1051 
1052  if (is_ip4)
1053  udp1 = ip4_next_header (ip41);
1054  else
1055  udp1 = ip6_next_header (ip61);
1056 
1057  if (udp1->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_geneve))
1058  goto exit1; /* not GENEVE packet */
1059 
1060  /* Validate DIP against VTEPs */
1061  if (is_ip4)
1062  {
1063  if (addr4.as_u32 != ip41->dst_address.as_u32)
1064  {
1065  if (!hash_get (vxm->vtep4, ip41->dst_address.as_u32))
1066  goto exit1; /* no local VTEP for GENEVE packet */
1067  addr4 = ip41->dst_address;
1068  }
1069  }
1070  else
1071  {
1072  if (!ip6_address_is_equal (&addr6, &ip61->dst_address))
1073  {
1074  if (!hash_get_mem (vxm->vtep6, &ip61->dst_address))
1075  goto exit1; /* no local VTEP for GENEVE packet */
1076  addr6 = ip61->dst_address;
1077  }
1078  }
1079 
1080  flags1 = b1->flags;
1081  good_udp1 = (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1082 
1083  /* Don't verify UDP checksum for packets with explicit zero checksum. */
1084  good_udp1 |= udp1->checksum == 0;
1085 
1086  /* Verify UDP length */
1087  if (is_ip4)
1088  ip_len1 = clib_net_to_host_u16 (ip41->length);
1089  else
1090  ip_len1 = clib_net_to_host_u16 (ip61->payload_length);
1091  udp_len1 = clib_net_to_host_u16 (udp1->length);
1092  len_diff1 = ip_len1 - udp_len1;
1093 
1094  /* Verify UDP checksum */
1095  if (PREDICT_FALSE (!good_udp1))
1096  {
1097  if ((flags1 & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED) == 0)
1098  {
1099  if (is_ip4)
1100  flags1 = ip4_tcp_udp_validate_checksum (vm, b1);
1101  else
1102  flags1 = ip6_tcp_udp_icmp_validate_checksum (vm, b1);
1103  good_udp1 =
1104  (flags1 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1105  }
1106  }
1107 
1108  if (is_ip4)
1109  {
1110  error1 = good_udp1 ? 0 : IP4_ERROR_UDP_CHECKSUM;
1111  error1 = (len_diff1 >= 0) ? error1 : IP4_ERROR_UDP_LENGTH;
1112  }
1113  else
1114  {
1115  error1 = good_udp1 ? 0 : IP6_ERROR_UDP_CHECKSUM;
1116  error1 = (len_diff1 >= 0) ? error1 : IP6_ERROR_UDP_LENGTH;
1117  }
1118 
1119  next1 = error1 ?
1121  b1->error = error1 ? error_node->errors[error1] : 0;
1122 
1123  /* geneve-input node expect current at GENEVE header */
1124  if (is_ip4)
1125  vlib_buffer_advance (b1,
1126  sizeof (ip4_header_t) +
1127  sizeof (udp_header_t));
1128  else
1129  vlib_buffer_advance (b1,
1130  sizeof (ip6_header_t) +
1131  sizeof (udp_header_t));
1132 
1133  exit1:
1134  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
1135  to_next, n_left_to_next,
1136  bi0, bi1, next0, next1);
1137  }
1138 
1139  while (n_left_from > 0 && n_left_to_next > 0)
1140  {
1141  vlib_buffer_t *b0;
1142  ip4_header_t *ip40;
1143  ip6_header_t *ip60;
1144  udp_header_t *udp0;
1145  u32 bi0, ip_len0, udp_len0, flags0, next0;
1146  i32 len_diff0;
1147  u8 error0, good_udp0, proto0;
1148 
1149  bi0 = to_next[0] = from[0];
1150  from += 1;
1151  n_left_from -= 1;
1152  to_next += 1;
1153  n_left_to_next -= 1;
1154 
1155  b0 = vlib_get_buffer (vm, bi0);
1156  if (is_ip4)
1157  ip40 = vlib_buffer_get_current (b0);
1158  else
1159  ip60 = vlib_buffer_get_current (b0);
1160 
1161  /* Setup packet for next IP feature */
1162  vnet_feature_next (&next0, b0);
1163 
1164  if (is_ip4)
1165  /* Treat IP4 frag packets as "experimental" protocol for now
1166  until support of IP frag reassembly is implemented */
1167  proto0 = ip4_is_fragment (ip40) ? 0xfe : ip40->protocol;
1168  else
1169  proto0 = ip60->protocol;
1170 
1171  if (proto0 != IP_PROTOCOL_UDP)
1172  goto exit; /* not UDP packet */
1173 
1174  if (is_ip4)
1175  udp0 = ip4_next_header (ip40);
1176  else
1177  udp0 = ip6_next_header (ip60);
1178 
1179  if (udp0->dst_port != clib_host_to_net_u16 (UDP_DST_PORT_geneve))
1180  goto exit; /* not GENEVE packet */
1181 
1182  /* Validate DIP against VTEPs */
1183  if (is_ip4)
1184  {
1185  if (addr4.as_u32 != ip40->dst_address.as_u32)
1186  {
1187  if (!hash_get (vxm->vtep4, ip40->dst_address.as_u32))
1188  goto exit; /* no local VTEP for GENEVE packet */
1189  addr4 = ip40->dst_address;
1190  }
1191  }
1192  else
1193  {
1194  if (!ip6_address_is_equal (&addr6, &ip60->dst_address))
1195  {
1196  if (!hash_get_mem (vxm->vtep6, &ip60->dst_address))
1197  goto exit; /* no local VTEP for GENEVE packet */
1198  addr6 = ip60->dst_address;
1199  }
1200  }
1201 
1202  flags0 = b0->flags;
1203  good_udp0 = (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1204 
1205  /* Don't verify UDP checksum for packets with explicit zero checksum. */
1206  good_udp0 |= udp0->checksum == 0;
1207 
1208  /* Verify UDP length */
1209  if (is_ip4)
1210  ip_len0 = clib_net_to_host_u16 (ip40->length);
1211  else
1212  ip_len0 = clib_net_to_host_u16 (ip60->payload_length);
1213  udp_len0 = clib_net_to_host_u16 (udp0->length);
1214  len_diff0 = ip_len0 - udp_len0;
1215 
1216  /* Verify UDP checksum */
1217  if (PREDICT_FALSE (!good_udp0))
1218  {
1219  if ((flags0 & VNET_BUFFER_F_L4_CHECKSUM_COMPUTED) == 0)
1220  {
1221  if (is_ip4)
1222  flags0 = ip4_tcp_udp_validate_checksum (vm, b0);
1223  else
1224  flags0 = ip6_tcp_udp_icmp_validate_checksum (vm, b0);
1225  good_udp0 =
1226  (flags0 & VNET_BUFFER_F_L4_CHECKSUM_CORRECT) != 0;
1227  }
1228  }
1229 
1230  if (is_ip4)
1231  {
1232  error0 = good_udp0 ? 0 : IP4_ERROR_UDP_CHECKSUM;
1233  error0 = (len_diff0 >= 0) ? error0 : IP4_ERROR_UDP_LENGTH;
1234  }
1235  else
1236  {
1237  error0 = good_udp0 ? 0 : IP6_ERROR_UDP_CHECKSUM;
1238  error0 = (len_diff0 >= 0) ? error0 : IP6_ERROR_UDP_LENGTH;
1239  }
1240 
1241  next0 = error0 ?
1243  b0->error = error0 ? error_node->errors[error0] : 0;
1244 
1245  /* geneve-input node expect current at GENEVE header */
1246  if (is_ip4)
1247  vlib_buffer_advance (b0,
1248  sizeof (ip4_header_t) +
1249  sizeof (udp_header_t));
1250  else
1251  vlib_buffer_advance (b0,
1252  sizeof (ip6_header_t) +
1253  sizeof (udp_header_t));
1254 
1255  exit:
1256  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1257  to_next, n_left_to_next,
1258  bi0, next0);
1259  }
1260 
1261  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1262  }
1263 
1264  return frame->n_vectors;
1265 }
1266 
1267 static uword
1269  vlib_node_runtime_t * node, vlib_frame_t * frame)
1270 {
1271  return ip_geneve_bypass_inline (vm, node, frame, /* is_ip4 */ 1);
1272 }
1273 
1274 /* *INDENT-OFF* */
1276 {
1277  .function = ip4_geneve_bypass,.name = "ip4-geneve-bypass",.vector_size =
1278  sizeof (u32),.n_next_nodes = IP_GENEVE_BYPASS_N_NEXT,.next_nodes =
1279  {
1280  [IP_GENEVE_BYPASS_NEXT_DROP] = "error-drop",
1281  [IP_GENEVE_BYPASS_NEXT_GENEVE] = "geneve4-input",}
1282 ,.format_buffer = format_ip4_header,.format_trace =
1284 
1286 /* Dummy init function to get us linked in. */
1288 {
1289  return 0;
1290 }
1291 
1293 /* *INDENT-ON* */
1294 
1295 static uword
1297  vlib_node_runtime_t * node, vlib_frame_t * frame)
1298 {
1299  return ip_geneve_bypass_inline (vm, node, frame, /* is_ip4 */ 0);
1300 }
1301 
1302 /* *INDENT-OFF* */
1304 {
1305  .function = ip6_geneve_bypass,.name = "ip6-geneve-bypass",.vector_size =
1306  sizeof (u32),.n_next_nodes = IP_GENEVE_BYPASS_N_NEXT,.next_nodes =
1307  {
1308  [IP_GENEVE_BYPASS_NEXT_DROP] = "error-drop",
1309  [IP_GENEVE_BYPASS_NEXT_GENEVE] = "geneve6-input",}
1310 ,.format_buffer = format_ip6_header,.format_trace =
1312 /* *INDENT-ON* */
1313 
1315 /* Dummy init function to get us linked in. */
1317 {
1318  return 0;
1319 }
1320 
1322 
1323 /*
1324  * fd.io coding-style-patch-verification: ON
1325  *
1326  * Local Variables:
1327  * eval: (c-set-style "gnu")
1328  * End:
1329  */
static u8 vnet_get_geneve_critical_bit(geneve_header_t *h)
static u32 vnet_get_geneve_vni(geneve_header_t *h)
#define CLIB_UNUSED(x)
Definition: clib.h:82
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:545
ip4_address_t src_address
Definition: ip4_packet.h:170
vnet_interface_main_t interface_main
Definition: vnet.h:56
format_function_t format_ip4_header
Definition: format.h:83
u32 tunnel_index
Definition: decap.c:26
#define PREDICT_TRUE(x)
Definition: clib.h:112
u64 as_u64[2]
Definition: ip6_packet.h:51
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
#define NULL
Definition: clib.h:58
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 decap_next_index
Definition: geneve.h:103
u32 thread_index
Definition: main.h:179
uword * vtep6
Definition: geneve.h:165
u32 * fib_index_by_sw_if_index
Table index indexed by software interface.
Definition: ip4.h:112
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
static uword ip4_address_is_multicast(const ip4_address_t *a)
Definition: ip4_packet.h:318
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:493
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:267
ip6_address_t src_address
Definition: ip6_packet.h:378
unsigned char u8
Definition: types.h:56
static char * geneve_error_strings[]
Definition: decap.c:814
#define GENEVE_BASE_HEADER_LENGTH
Definition: geneve_packet.h:95
vlib_node_registration_t ip4_geneve_bypass_node
(constructor) VLIB_REGISTER_NODE (ip4_geneve_bypass_node)
Definition: decap.c:1275
geneve_tunnel_t * tunnels
Definition: geneve.h:156
static int ip4_is_fragment(const ip4_header_t *i)
Definition: ip4_packet.h:213
static u8 vnet_get_geneve_options_len(geneve_header_t *h)
i64 word
Definition: types.h:111
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
u32 sw_if_index
Definition: vxlan_gbp.api:37
#define always_inline
Definition: clib.h:98
ip4_address_t dst_address
Definition: ip4_packet.h:170
#define foreach_geneve_input_next
Definition: geneve.h:133
static u32 vnet_get_geneve_vni_bigendian(geneve_header_t *h)
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:839
static uword ip_geneve_bypass_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, u32 is_ip4)
Definition: decap.c:872
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:188
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:241
unsigned int u32
Definition: types.h:88
clib_error_t * ip6_geneve_bypass_init(vlib_main_t *vm)
Definition: decap.c:1316
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
static void vnet_geneve_hdr_1word_ntoh(geneve_header_t *h)
vlib_node_registration_t ip4_input_node
Global ip4 input node.
Definition: ip4_input.c:317
u32 encap_fib_index
Definition: geneve.h:106
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:214
#define PREDICT_FALSE(x)
Definition: clib.h:111
static u32 validate_geneve_fib(vlib_buffer_t *b, geneve_tunnel_t *t, u32 is_ip4)
Definition: decap.c:54
u32 ip4_tcp_udp_validate_checksum(vlib_main_t *vm, vlib_buffer_t *p0)
Definition: ip4_forward.c:1187
vlib_node_registration_t ip6_geneve_bypass_node
(constructor) VLIB_REGISTER_NODE (ip6_geneve_bypass_node)
Definition: decap.c:1303
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:338
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:139
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1150
vlib_node_registration_t geneve4_input_node
(constructor) VLIB_REGISTER_NODE (geneve4_input_node)
Definition: decap.c:20
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
uword * vtep4
Definition: geneve.h:164
u16 n_vectors
Definition: node.h:420
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:79
vlib_main_t * vm
Definition: buffer.c:301
static void ip6_address_set_zero(ip6_address_t *a)
Definition: ip6_packet.h:271
static_always_inline void vnet_feature_next(u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:295
ip46_address_t local
Definition: geneve.h:96
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:452
uword * geneve4_tunnel_by_key
Definition: geneve.h:159
static void * ip6_next_header(ip6_header_t *i)
Definition: ip6_packet.h:405
ip_vxan_bypass_next_t
Definition: decap.c:864
signed int i32
Definition: types.h:77
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:537
u32 sw_if_index
Definition: geneve.h:109
static uword geneve6_input(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: decap.c:808
static u8 * format_geneve_rx_trace(u8 *s, va_list *args)
Definition: decap.c:32
ip6_main_t ip6_main
Definition: ip6_forward.c:2624
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:233
static u8 vnet_get_geneve_version(geneve_header_t *h)
format_function_t format_ip6_header
Definition: format.h:97
u32 next_index
Definition: decap.c:25
static uword ip6_address_is_equal(const ip6_address_t *a, const ip6_address_t *b)
Definition: ip6_packet.h:235
static uword ip6_address_is_multicast(const ip6_address_t *a)
Definition: ip6_packet.h:172
vnet_main_t * vnet_main
Definition: geneve.h:178
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:57
#define vec_elt(v, i)
Get vector value at index i.
struct _vlib_node_registration vlib_node_registration_t
Definition: defs.h:47
u16 payload_length
Definition: ip6_packet.h:369
static uword ip4_geneve_bypass(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: decap.c:1268
u32 ip6_tcp_udp_icmp_validate_checksum(vlib_main_t *vm, vlib_buffer_t *p0)
Definition: ip6_forward.c:1020
static void vnet_update_l2_len(vlib_buffer_t *b)
Definition: l2_input.h:226
void ip4_forward_next_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vlib_rx_or_tx_t which_adj_index)
Definition: ip4_forward.c:1041
u64 uword
Definition: types.h:112
uword * geneve6_tunnel_by_key
Definition: geneve.h:160
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
u8 * format_ip4_forward_next_trace(u8 *s, va_list *args)
Definition: ip4_forward.c:991
#define GENEVE_VERSION
Definition: geneve_packet.h:98
#define hash_get_mem(h, key)
Definition: hash.h:269
static uword ip6_geneve_bypass(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: decap.c:1296
vlib_node_registration_t geneve6_input_node
(constructor) VLIB_REGISTER_NODE (geneve6_input_node)
Definition: decap.c:21
#define vnet_buffer(b)
Definition: buffer.h:368
VLIB_NODE_FUNCTION_MULTIARCH(l2t_decap_node, l2t_decap_node_fn)
ip4_main_t ip4_main
Global ip4 main structure.
Definition: ip4_forward.c:903
geneve_main_t geneve_main
Definition: geneve.c:38
u8 data[0]
Packet data.
Definition: buffer.h:176
u16 flags
Copy of main node flags.
Definition: node.h:531
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:326
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
clib_error_t * ip4_geneve_bypass_init(vlib_main_t *vm)
Definition: decap.c:1287
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:117
u32 * fib_index_by_sw_if_index
Definition: ip6.h:193
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:62
static uword geneve4_input(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: decap.c:801
Definition: defs.h:46
ip6_address_t dst_address
Definition: ip6_packet.h:378
static uword geneve_input(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame, u32 is_ip4)
Definition: decap.c:73
u8 * format_ip6_forward_next_trace(u8 *s, va_list *args)
Definition: ip6_forward.c:802