FD.io VPP  v17.01-9-ge7dcee4
Vector Packet Processing
ip4_forward.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * ip/ip4_forward.c: IP v4 forwarding
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <vnet/vnet.h>
41 #include <vnet/ip/ip.h>
42 #include <vnet/ethernet/ethernet.h> /* for ethernet_header_t */
43 #include <vnet/ethernet/arp_packet.h> /* for ethernet_arp_header_t */
44 #include <vnet/ppp/ppp.h>
45 #include <vnet/srp/srp.h> /* for srp_hw_interface_class */
46 #include <vnet/api_errno.h> /* for API error numbers */
47 #include <vnet/fib/fib_table.h> /* for FIB table and entry creation */
48 #include <vnet/fib/fib_entry.h> /* for FIB table and entry creation */
49 #include <vnet/fib/fib_urpf_list.h> /* for FIB uRPF check */
50 #include <vnet/fib/ip4_fib.h>
51 #include <vnet/dpo/load_balance.h>
52 #include <vnet/dpo/classify_dpo.h>
53 
54 /**
55  * @file
56  * @brief IPv4 Forwarding.
57  *
58  * This file contains the source code for IPv4 forwarding.
59  */
60 
61 void
63  vlib_node_runtime_t * node,
64  vlib_frame_t * frame,
65  vlib_rx_or_tx_t which_adj_index);
66 
69  vlib_node_runtime_t * node,
70  vlib_frame_t * frame,
71  int lookup_for_responses_to_locally_received_packets)
72 {
73  ip4_main_t *im = &ip4_main;
75  u32 n_left_from, n_left_to_next, *from, *to_next;
76  ip_lookup_next_t next;
77  u32 cpu_index = os_get_cpu_number ();
78 
79  from = vlib_frame_vector_args (frame);
80  n_left_from = frame->n_vectors;
81  next = node->cached_next_index;
82 
83  while (n_left_from > 0)
84  {
85  vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
86 
87  while (n_left_from >= 8 && n_left_to_next >= 4)
88  {
89  vlib_buffer_t *p0, *p1, *p2, *p3;
90  ip4_header_t *ip0, *ip1, *ip2, *ip3;
91  __attribute__ ((unused)) tcp_header_t *tcp0, *tcp1, *tcp2, *tcp3;
92  ip_lookup_next_t next0, next1, next2, next3;
93  const load_balance_t *lb0, *lb1, *lb2, *lb3;
94  ip4_fib_mtrie_t *mtrie0, *mtrie1, *mtrie2, *mtrie3;
95  ip4_fib_mtrie_leaf_t leaf0, leaf1, leaf2, leaf3;
96  ip4_address_t *dst_addr0, *dst_addr1, *dst_addr2, *dst_addr3;
97  __attribute__ ((unused)) u32 pi0, fib_index0, lb_index0,
98  is_tcp_udp0;
99  __attribute__ ((unused)) u32 pi1, fib_index1, lb_index1,
100  is_tcp_udp1;
101  __attribute__ ((unused)) u32 pi2, fib_index2, lb_index2,
102  is_tcp_udp2;
103  __attribute__ ((unused)) u32 pi3, fib_index3, lb_index3,
104  is_tcp_udp3;
105  flow_hash_config_t flow_hash_config0, flow_hash_config1;
106  flow_hash_config_t flow_hash_config2, flow_hash_config3;
107  u32 hash_c0, hash_c1, hash_c2, hash_c3;
108  const dpo_id_t *dpo0, *dpo1, *dpo2, *dpo3;
109 
110  /* Prefetch next iteration. */
111  {
112  vlib_buffer_t *p4, *p5, *p6, *p7;
113 
114  p4 = vlib_get_buffer (vm, from[4]);
115  p5 = vlib_get_buffer (vm, from[5]);
116  p6 = vlib_get_buffer (vm, from[6]);
117  p7 = vlib_get_buffer (vm, from[7]);
118 
119  vlib_prefetch_buffer_header (p4, LOAD);
120  vlib_prefetch_buffer_header (p5, LOAD);
121  vlib_prefetch_buffer_header (p6, LOAD);
122  vlib_prefetch_buffer_header (p7, LOAD);
123 
124  CLIB_PREFETCH (p4->data, sizeof (ip0[0]), LOAD);
125  CLIB_PREFETCH (p5->data, sizeof (ip0[0]), LOAD);
126  CLIB_PREFETCH (p6->data, sizeof (ip0[0]), LOAD);
127  CLIB_PREFETCH (p7->data, sizeof (ip0[0]), LOAD);
128  }
129 
130  pi0 = to_next[0] = from[0];
131  pi1 = to_next[1] = from[1];
132  pi2 = to_next[2] = from[2];
133  pi3 = to_next[3] = from[3];
134 
135  from += 4;
136  to_next += 4;
137  n_left_to_next -= 4;
138  n_left_from -= 4;
139 
140  p0 = vlib_get_buffer (vm, pi0);
141  p1 = vlib_get_buffer (vm, pi1);
142  p2 = vlib_get_buffer (vm, pi2);
143  p3 = vlib_get_buffer (vm, pi3);
144 
145  ip0 = vlib_buffer_get_current (p0);
146  ip1 = vlib_buffer_get_current (p1);
147  ip2 = vlib_buffer_get_current (p2);
148  ip3 = vlib_buffer_get_current (p3);
149 
150  dst_addr0 = &ip0->dst_address;
151  dst_addr1 = &ip1->dst_address;
152  dst_addr2 = &ip2->dst_address;
153  dst_addr3 = &ip3->dst_address;
154 
155  fib_index0 =
157  vnet_buffer (p0)->sw_if_index[VLIB_RX]);
158  fib_index1 =
160  vnet_buffer (p1)->sw_if_index[VLIB_RX]);
161  fib_index2 =
163  vnet_buffer (p2)->sw_if_index[VLIB_RX]);
164  fib_index3 =
166  vnet_buffer (p3)->sw_if_index[VLIB_RX]);
167  fib_index0 =
168  (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
169  (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
170  fib_index1 =
171  (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
172  (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
173  fib_index2 =
174  (vnet_buffer (p2)->sw_if_index[VLIB_TX] ==
175  (u32) ~ 0) ? fib_index2 : vnet_buffer (p2)->sw_if_index[VLIB_TX];
176  fib_index3 =
177  (vnet_buffer (p3)->sw_if_index[VLIB_TX] ==
178  (u32) ~ 0) ? fib_index3 : vnet_buffer (p3)->sw_if_index[VLIB_TX];
179 
180 
181  if (!lookup_for_responses_to_locally_received_packets)
182  {
183  mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
184  mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
185  mtrie2 = &ip4_fib_get (fib_index2)->mtrie;
186  mtrie3 = &ip4_fib_get (fib_index3)->mtrie;
187 
188  leaf0 = leaf1 = leaf2 = leaf3 = IP4_FIB_MTRIE_LEAF_ROOT;
189 
190  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
191  leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 0);
192  leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 0);
193  leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 0);
194  }
195 
196  tcp0 = (void *) (ip0 + 1);
197  tcp1 = (void *) (ip1 + 1);
198  tcp2 = (void *) (ip2 + 1);
199  tcp3 = (void *) (ip3 + 1);
200 
201  is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
202  || ip0->protocol == IP_PROTOCOL_UDP);
203  is_tcp_udp1 = (ip1->protocol == IP_PROTOCOL_TCP
204  || ip1->protocol == IP_PROTOCOL_UDP);
205  is_tcp_udp2 = (ip2->protocol == IP_PROTOCOL_TCP
206  || ip2->protocol == IP_PROTOCOL_UDP);
207  is_tcp_udp3 = (ip1->protocol == IP_PROTOCOL_TCP
208  || ip1->protocol == IP_PROTOCOL_UDP);
209 
210  if (!lookup_for_responses_to_locally_received_packets)
211  {
212  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
213  leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 1);
214  leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 1);
215  leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 1);
216  }
217 
218  if (!lookup_for_responses_to_locally_received_packets)
219  {
220  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
221  leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 2);
222  leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 2);
223  leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 2);
224  }
225 
226  if (!lookup_for_responses_to_locally_received_packets)
227  {
228  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
229  leaf1 = ip4_fib_mtrie_lookup_step (mtrie1, leaf1, dst_addr1, 3);
230  leaf2 = ip4_fib_mtrie_lookup_step (mtrie2, leaf2, dst_addr2, 3);
231  leaf3 = ip4_fib_mtrie_lookup_step (mtrie3, leaf3, dst_addr3, 3);
232  }
233 
234  if (lookup_for_responses_to_locally_received_packets)
235  {
236  lb_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
237  lb_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_RX];
238  lb_index2 = vnet_buffer (p2)->ip.adj_index[VLIB_RX];
239  lb_index3 = vnet_buffer (p3)->ip.adj_index[VLIB_RX];
240  }
241  else
242  {
243  /* Handle default route. */
244  leaf0 =
245  (leaf0 ==
246  IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
247  leaf1 =
248  (leaf1 ==
249  IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
250  leaf2 =
251  (leaf2 ==
252  IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie2->default_leaf : leaf2);
253  leaf3 =
254  (leaf3 ==
255  IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie3->default_leaf : leaf3);
256  lb_index0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
257  lb_index1 = ip4_fib_mtrie_leaf_get_adj_index (leaf1);
258  lb_index2 = ip4_fib_mtrie_leaf_get_adj_index (leaf2);
259  lb_index3 = ip4_fib_mtrie_leaf_get_adj_index (leaf3);
260  }
261 
262  lb0 = load_balance_get (lb_index0);
263  lb1 = load_balance_get (lb_index1);
264  lb2 = load_balance_get (lb_index2);
265  lb3 = load_balance_get (lb_index3);
266 
267  /* Use flow hash to compute multipath adjacency. */
268  hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
269  hash_c1 = vnet_buffer (p1)->ip.flow_hash = 0;
270  hash_c2 = vnet_buffer (p2)->ip.flow_hash = 0;
271  hash_c3 = vnet_buffer (p3)->ip.flow_hash = 0;
272  if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
273  {
274  flow_hash_config0 = lb0->lb_hash_config;
275  hash_c0 = vnet_buffer (p0)->ip.flow_hash =
276  ip4_compute_flow_hash (ip0, flow_hash_config0);
277  }
278  if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
279  {
280  flow_hash_config1 = lb1->lb_hash_config;
281  hash_c1 = vnet_buffer (p1)->ip.flow_hash =
282  ip4_compute_flow_hash (ip1, flow_hash_config1);
283  }
284  if (PREDICT_FALSE (lb2->lb_n_buckets > 1))
285  {
286  flow_hash_config2 = lb2->lb_hash_config;
287  hash_c2 = vnet_buffer (p2)->ip.flow_hash =
288  ip4_compute_flow_hash (ip2, flow_hash_config2);
289  }
290  if (PREDICT_FALSE (lb3->lb_n_buckets > 1))
291  {
292  flow_hash_config3 = lb3->lb_hash_config;
293  hash_c3 = vnet_buffer (p3)->ip.flow_hash =
294  ip4_compute_flow_hash (ip3, flow_hash_config3);
295  }
296 
297  ASSERT (lb0->lb_n_buckets > 0);
298  ASSERT (is_pow2 (lb0->lb_n_buckets));
299  ASSERT (lb1->lb_n_buckets > 0);
300  ASSERT (is_pow2 (lb1->lb_n_buckets));
301  ASSERT (lb2->lb_n_buckets > 0);
302  ASSERT (is_pow2 (lb2->lb_n_buckets));
303  ASSERT (lb3->lb_n_buckets > 0);
304  ASSERT (is_pow2 (lb3->lb_n_buckets));
305 
306  dpo0 = load_balance_get_bucket_i (lb0,
307  (hash_c0 &
308  (lb0->lb_n_buckets_minus_1)));
309  dpo1 = load_balance_get_bucket_i (lb1,
310  (hash_c1 &
311  (lb1->lb_n_buckets_minus_1)));
312  dpo2 = load_balance_get_bucket_i (lb2,
313  (hash_c2 &
314  (lb2->lb_n_buckets_minus_1)));
315  dpo3 = load_balance_get_bucket_i (lb3,
316  (hash_c3 &
317  (lb3->lb_n_buckets_minus_1)));
318 
319  next0 = dpo0->dpoi_next_node;
320  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
321  next1 = dpo1->dpoi_next_node;
322  vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
323  next2 = dpo2->dpoi_next_node;
324  vnet_buffer (p2)->ip.adj_index[VLIB_TX] = dpo2->dpoi_index;
325  next3 = dpo3->dpoi_next_node;
326  vnet_buffer (p3)->ip.adj_index[VLIB_TX] = dpo3->dpoi_index;
327 
329  (cm, cpu_index, lb_index0, 1,
331  + sizeof (ethernet_header_t));
333  (cm, cpu_index, lb_index1, 1,
335  + sizeof (ethernet_header_t));
337  (cm, cpu_index, lb_index2, 1,
339  + sizeof (ethernet_header_t));
341  (cm, cpu_index, lb_index3, 1,
343  + sizeof (ethernet_header_t));
344 
345  vlib_validate_buffer_enqueue_x4 (vm, node, next,
346  to_next, n_left_to_next,
347  pi0, pi1, pi2, pi3,
348  next0, next1, next2, next3);
349  }
350 
351  while (n_left_from > 0 && n_left_to_next > 0)
352  {
353  vlib_buffer_t *p0;
354  ip4_header_t *ip0;
355  __attribute__ ((unused)) tcp_header_t *tcp0;
356  ip_lookup_next_t next0;
357  const load_balance_t *lb0;
358  ip4_fib_mtrie_t *mtrie0;
359  ip4_fib_mtrie_leaf_t leaf0;
360  ip4_address_t *dst_addr0;
361  __attribute__ ((unused)) u32 pi0, fib_index0, is_tcp_udp0, lbi0;
362  flow_hash_config_t flow_hash_config0;
363  const dpo_id_t *dpo0;
364  u32 hash_c0;
365 
366  pi0 = from[0];
367  to_next[0] = pi0;
368 
369  p0 = vlib_get_buffer (vm, pi0);
370 
371  ip0 = vlib_buffer_get_current (p0);
372 
373  dst_addr0 = &ip0->dst_address;
374 
375  fib_index0 =
377  vnet_buffer (p0)->sw_if_index[VLIB_RX]);
378  fib_index0 =
379  (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
380  (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
381 
382  if (!lookup_for_responses_to_locally_received_packets)
383  {
384  mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
385 
386  leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
387 
388  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 0);
389  }
390 
391  tcp0 = (void *) (ip0 + 1);
392 
393  is_tcp_udp0 = (ip0->protocol == IP_PROTOCOL_TCP
394  || ip0->protocol == IP_PROTOCOL_UDP);
395 
396  if (!lookup_for_responses_to_locally_received_packets)
397  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 1);
398 
399  if (!lookup_for_responses_to_locally_received_packets)
400  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 2);
401 
402  if (!lookup_for_responses_to_locally_received_packets)
403  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, dst_addr0, 3);
404 
405  if (lookup_for_responses_to_locally_received_packets)
406  lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_RX];
407  else
408  {
409  /* Handle default route. */
410  leaf0 =
411  (leaf0 ==
412  IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
413  lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
414  }
415 
416  lb0 = load_balance_get (lbi0);
417 
418  /* Use flow hash to compute multipath adjacency. */
419  hash_c0 = vnet_buffer (p0)->ip.flow_hash = 0;
420  if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
421  {
422  flow_hash_config0 = lb0->lb_hash_config;
423 
424  hash_c0 = vnet_buffer (p0)->ip.flow_hash =
425  ip4_compute_flow_hash (ip0, flow_hash_config0);
426  }
427 
428  ASSERT (lb0->lb_n_buckets > 0);
429  ASSERT (is_pow2 (lb0->lb_n_buckets));
430 
431  dpo0 = load_balance_get_bucket_i (lb0,
432  (hash_c0 &
433  (lb0->lb_n_buckets_minus_1)));
434 
435  next0 = dpo0->dpoi_next_node;
436  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
437 
439  (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
440 
441  from += 1;
442  to_next += 1;
443  n_left_to_next -= 1;
444  n_left_from -= 1;
445 
446  if (PREDICT_FALSE (next0 != next))
447  {
448  n_left_to_next += 1;
449  vlib_put_next_frame (vm, node, next, n_left_to_next);
450  next = next0;
451  vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
452  to_next[0] = pi0;
453  to_next += 1;
454  n_left_to_next -= 1;
455  }
456  }
457 
458  vlib_put_next_frame (vm, node, next, n_left_to_next);
459  }
460 
461  if (node->flags & VLIB_NODE_FLAG_TRACE)
462  ip4_forward_next_trace (vm, node, frame, VLIB_TX);
463 
464  return frame->n_vectors;
465 }
466 
467 /** @brief IPv4 lookup node.
468  @node ip4-lookup
469 
470  This is the main IPv4 lookup dispatch node.
471 
472  @param vm vlib_main_t corresponding to the current thread
473  @param node vlib_node_runtime_t
474  @param frame vlib_frame_t whose contents should be dispatched
475 
476  @par Graph mechanics: buffer metadata, next index usage
477 
478  @em Uses:
479  - <code>vnet_buffer(b)->sw_if_index[VLIB_RX]</code>
480  - Indicates the @c sw_if_index value of the interface that the
481  packet was received on.
482  - <code>vnet_buffer(b)->sw_if_index[VLIB_TX]</code>
483  - When the value is @c ~0 then the node performs a longest prefix
484  match (LPM) for the packet destination address in the FIB attached
485  to the receive interface.
486  - Otherwise perform LPM for the packet destination address in the
487  indicated FIB. In this case <code>[VLIB_TX]</code> is a FIB index
488  value (0, 1, ...) and not a VRF id.
489 
490  @em Sets:
491  - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
492  - The lookup result adjacency index.
493 
494  <em>Next Index:</em>
495  - Dispatches the packet to the node index found in
496  ip_adjacency_t @c adj->lookup_next_index
497  (where @c adj is the lookup result adjacency).
498 */
499 static uword
501  vlib_node_runtime_t * node, vlib_frame_t * frame)
502 {
503  return ip4_lookup_inline (vm, node, frame,
504  /* lookup_for_responses_to_locally_received_packets */
505  0);
506 
507 }
508 
509 static u8 *format_ip4_lookup_trace (u8 * s, va_list * args);
510 
512 {
513 .function = ip4_lookup,.name = "ip4-lookup",.vector_size =
514  sizeof (u32),.format_trace = format_ip4_lookup_trace,.n_next_nodes =
515  IP_LOOKUP_N_NEXT,.next_nodes = IP4_LOOKUP_NEXT_NODES,};
516 
518 
521  vlib_node_runtime_t * node, vlib_frame_t * frame)
522 {
524  u32 n_left_from, n_left_to_next, *from, *to_next;
525  ip_lookup_next_t next;
526  u32 cpu_index = os_get_cpu_number ();
527 
528  from = vlib_frame_vector_args (frame);
529  n_left_from = frame->n_vectors;
530  next = node->cached_next_index;
531 
532  if (node->flags & VLIB_NODE_FLAG_TRACE)
533  ip4_forward_next_trace (vm, node, frame, VLIB_TX);
534 
535  while (n_left_from > 0)
536  {
537  vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
538 
539 
540  while (n_left_from >= 4 && n_left_to_next >= 2)
541  {
542  ip_lookup_next_t next0, next1;
543  const load_balance_t *lb0, *lb1;
544  vlib_buffer_t *p0, *p1;
545  u32 pi0, lbi0, hc0, pi1, lbi1, hc1;
546  const ip4_header_t *ip0, *ip1;
547  const dpo_id_t *dpo0, *dpo1;
548 
549  /* Prefetch next iteration. */
550  {
551  vlib_buffer_t *p2, *p3;
552 
553  p2 = vlib_get_buffer (vm, from[2]);
554  p3 = vlib_get_buffer (vm, from[3]);
555 
556  vlib_prefetch_buffer_header (p2, STORE);
557  vlib_prefetch_buffer_header (p3, STORE);
558 
559  CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
560  CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
561  }
562 
563  pi0 = to_next[0] = from[0];
564  pi1 = to_next[1] = from[1];
565 
566  from += 2;
567  n_left_from -= 2;
568  to_next += 2;
569  n_left_to_next -= 2;
570 
571  p0 = vlib_get_buffer (vm, pi0);
572  p1 = vlib_get_buffer (vm, pi1);
573 
574  ip0 = vlib_buffer_get_current (p0);
575  ip1 = vlib_buffer_get_current (p1);
576  lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
577  lbi1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
578 
579  lb0 = load_balance_get (lbi0);
580  lb1 = load_balance_get (lbi1);
581 
582  /*
583  * this node is for via FIBs we can re-use the hash value from the
584  * to node if present.
585  * We don't want to use the same hash value at each level in the recursion
586  * graph as that would lead to polarisation
587  */
588  hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
589  hc1 = vnet_buffer (p1)->ip.flow_hash = 0;
590 
591  if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
592  {
593  if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
594  {
595  hc0 = vnet_buffer (p0)->ip.flow_hash =
596  vnet_buffer (p0)->ip.flow_hash >> 1;
597  }
598  else
599  {
600  hc0 = vnet_buffer (p0)->ip.flow_hash =
601  ip4_compute_flow_hash (ip0, hc0);
602  }
603  }
604  if (PREDICT_FALSE (lb1->lb_n_buckets > 1))
605  {
606  if (PREDICT_TRUE (vnet_buffer (p1)->ip.flow_hash))
607  {
608  hc1 = vnet_buffer (p1)->ip.flow_hash =
609  vnet_buffer (p1)->ip.flow_hash >> 1;
610  }
611  else
612  {
613  hc1 = vnet_buffer (p1)->ip.flow_hash =
614  ip4_compute_flow_hash (ip1, hc1);
615  }
616  }
617 
618  dpo0 =
620  hc0 & (lb0->lb_n_buckets_minus_1));
621  dpo1 =
623  hc1 & (lb1->lb_n_buckets_minus_1));
624 
625  next0 = dpo0->dpoi_next_node;
626  next1 = dpo1->dpoi_next_node;
627 
628  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
629  vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
630 
632  (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
634  (cm, cpu_index, lbi1, 1, vlib_buffer_length_in_chain (vm, p1));
635 
636  vlib_validate_buffer_enqueue_x2 (vm, node, next,
637  to_next, n_left_to_next,
638  pi0, pi1, next0, next1);
639  }
640 
641  while (n_left_from > 0 && n_left_to_next > 0)
642  {
643  ip_lookup_next_t next0;
644  const load_balance_t *lb0;
645  vlib_buffer_t *p0;
646  u32 pi0, lbi0, hc0;
647  const ip4_header_t *ip0;
648  const dpo_id_t *dpo0;
649 
650  pi0 = from[0];
651  to_next[0] = pi0;
652  from += 1;
653  to_next += 1;
654  n_left_to_next -= 1;
655  n_left_from -= 1;
656 
657  p0 = vlib_get_buffer (vm, pi0);
658 
659  ip0 = vlib_buffer_get_current (p0);
660  lbi0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
661 
662  lb0 = load_balance_get (lbi0);
663 
664  hc0 = vnet_buffer (p0)->ip.flow_hash = 0;
665  if (PREDICT_FALSE (lb0->lb_n_buckets > 1))
666  {
667  if (PREDICT_TRUE (vnet_buffer (p0)->ip.flow_hash))
668  {
669  hc0 = vnet_buffer (p0)->ip.flow_hash =
670  vnet_buffer (p0)->ip.flow_hash >> 1;
671  }
672  else
673  {
674  hc0 = vnet_buffer (p0)->ip.flow_hash =
675  ip4_compute_flow_hash (ip0, hc0);
676  }
677  }
678 
679  dpo0 =
681  hc0 & (lb0->lb_n_buckets_minus_1));
682 
683  next0 = dpo0->dpoi_next_node;
684  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
685 
687  (cm, cpu_index, lbi0, 1, vlib_buffer_length_in_chain (vm, p0));
688 
689  vlib_validate_buffer_enqueue_x1 (vm, node, next,
690  to_next, n_left_to_next,
691  pi0, next0);
692  }
693 
694  vlib_put_next_frame (vm, node, next, n_left_to_next);
695  }
696 
697  return frame->n_vectors;
698 }
699 
701 {
702 .function = ip4_load_balance,.name = "ip4-load-balance",.vector_size =
703  sizeof (u32),.sibling_of = "ip4-lookup",.format_trace =
705 
707 
708 /* get first interface address */
711  ip_interface_address_t ** result_ia)
712 {
713  ip_lookup_main_t *lm = &im->lookup_main;
714  ip_interface_address_t *ia = 0;
715  ip4_address_t *result = 0;
716 
717  foreach_ip_interface_address (lm, ia, sw_if_index,
718  1 /* honor unnumbered */ ,
719  (
720  {
721  ip4_address_t * a =
723  result = a;
724  break;
725  }
726  ));
727  if (result_ia)
728  *result_ia = result ? ia : 0;
729  return result;
730 }
731 
732 static void
734  ip4_main_t * im, u32 fib_index,
736 {
737  ip_lookup_main_t *lm = &im->lookup_main;
739  fib_prefix_t pfx = {
740  .fp_len = a->address_length,
741  .fp_proto = FIB_PROTOCOL_IP4,
742  .fp_addr.ip4 = *address,
743  };
744 
745  a->neighbor_probe_adj_index = ~0;
746 
747  if (pfx.fp_len < 32)
748  {
749  fib_node_index_t fei;
750 
752  sw_if_index, ~0, // invalid FIB index
753  1, NULL, // no out-label stack
756  }
757 
758  pfx.fp_len = 32;
759 
760  if (sw_if_index < vec_len (lm->classify_table_index_by_sw_if_index))
761  {
762  u32 classify_table_index =
763  lm->classify_table_index_by_sw_if_index[sw_if_index];
764  if (classify_table_index != (u32) ~ 0)
765  {
766  dpo_id_t dpo = DPO_INVALID;
767 
768  dpo_set (&dpo,
769  DPO_CLASSIFY,
771  classify_dpo_create (DPO_PROTO_IP4, classify_table_index));
772 
774  &pfx,
776  FIB_ENTRY_FLAG_NONE, &dpo);
777  dpo_reset (&dpo);
778  }
779  }
780 
781  fib_table_entry_update_one_path (fib_index, &pfx, FIB_SOURCE_INTERFACE, (FIB_ENTRY_FLAG_CONNECTED | FIB_ENTRY_FLAG_LOCAL), FIB_PROTOCOL_IP4, &pfx.fp_addr, sw_if_index, ~0, // invalid FIB index
782  1, NULL, // no out-label stack
784 }
785 
786 static void
788  u32 fib_index,
789  ip4_address_t * address, u32 address_length)
790 {
791  fib_prefix_t pfx = {
792  .fp_len = address_length,
793  .fp_proto = FIB_PROTOCOL_IP4,
794  .fp_addr.ip4 = *address,
795  };
796 
797  if (pfx.fp_len < 32)
798  {
799  fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
800  }
801 
802  pfx.fp_len = 32;
803  fib_table_entry_delete (fib_index, &pfx, FIB_SOURCE_INTERFACE);
804 }
805 
806 void
807 ip4_sw_interface_enable_disable (u32 sw_if_index, u32 is_enable)
808 {
809  ip4_main_t *im = &ip4_main;
810 
812 
813  /*
814  * enable/disable only on the 1<->0 transition
815  */
816  if (is_enable)
817  {
818  if (1 != ++im->ip_enabled_by_sw_if_index[sw_if_index])
819  return;
820  }
821  else
822  {
823  ASSERT (im->ip_enabled_by_sw_if_index[sw_if_index] > 0);
824  if (0 != --im->ip_enabled_by_sw_if_index[sw_if_index])
825  return;
826  }
827  vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
828  !is_enable, 0, 0);
829 
830  vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
831  !is_enable, 0, 0);
832 
833 }
834 
835 static clib_error_t *
837  u32 sw_if_index,
838  ip4_address_t * address,
839  u32 address_length, u32 is_del)
840 {
841  vnet_main_t *vnm = vnet_get_main ();
842  ip4_main_t *im = &ip4_main;
843  ip_lookup_main_t *lm = &im->lookup_main;
844  clib_error_t *error = 0;
845  u32 if_address_index, elts_before;
846  ip4_address_fib_t ip4_af, *addr_fib = 0;
847 
848  vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
849  ip4_addr_fib_init (&ip4_af, address,
850  vec_elt (im->fib_index_by_sw_if_index, sw_if_index));
851  vec_add1 (addr_fib, ip4_af);
852 
853  /* FIXME-LATER
854  * there is no support for adj-fib handling in the presence of overlapping
855  * subnets on interfaces. Easy fix - disallow overlapping subnets, like
856  * most routers do.
857  */
858  if (!is_del)
859  {
860  /* When adding an address check that it does not conflict
861  with an existing address. */
863  foreach_ip_interface_address (&im->lookup_main, ia, sw_if_index,
864  0 /* honor unnumbered */ ,
865  (
866  {
867  ip4_address_t * x =
868  ip_interface_address_get_address
869  (&im->lookup_main, ia);
870  if (ip4_destination_matches_route
871  (im, address, x, ia->address_length)
872  ||
873  ip4_destination_matches_route (im,
874  x,
875  address,
876  address_length))
877  return
878  clib_error_create
879  ("failed to add %U which conflicts with %U for interface %U",
880  format_ip4_address_and_length, address,
881  address_length,
882  format_ip4_address_and_length, x,
883  ia->address_length,
884  format_vnet_sw_if_index_name, vnm,
885  sw_if_index);}
886  ));
887  }
888 
889  elts_before = pool_elts (lm->if_address_pool);
890 
892  (lm, sw_if_index, addr_fib, address_length, is_del, &if_address_index);
893  if (error)
894  goto done;
895 
896  ip4_sw_interface_enable_disable (sw_if_index, !is_del);
897 
898  if (is_del)
899  ip4_del_interface_routes (im, ip4_af.fib_index, address, address_length);
900  else
901  ip4_add_interface_routes (sw_if_index,
902  im, ip4_af.fib_index,
904  (lm->if_address_pool, if_address_index));
905 
906  /* If pool did not grow/shrink: add duplicate address. */
907  if (elts_before != pool_elts (lm->if_address_pool))
908  {
911  cb->function (im, cb->function_opaque, sw_if_index,
912  address, address_length, if_address_index, is_del);
913  }
914 
915 done:
916  vec_free (addr_fib);
917  return error;
918 }
919 
920 clib_error_t *
922  ip4_address_t * address, u32 address_length,
923  u32 is_del)
924 {
926  (vm, sw_if_index, address, address_length, is_del);
927 }
928 
929 /* Built-in ip4 unicast rx feature path definition */
930 /* *INDENT-OFF* */
931 VNET_FEATURE_ARC_INIT (ip4_unicast, static) =
932 {
933  .arc_name = "ip4-unicast",
934  .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
935  .end_node = "ip4-lookup",
936  .arc_index_ptr = &ip4_main.lookup_main.ucast_feature_arc_index,
937 };
938 
940 {
941  .arc_name = "ip4-unicast",
942  .node_name = "ip4-flow-classify",
943  .runs_before = VNET_FEATURES ("ip4-inacl"),
944 };
945 
946 VNET_FEATURE_INIT (ip4_inacl, static) =
947 {
948  .arc_name = "ip4-unicast",
949  .node_name = "ip4-inacl",
950  .runs_before = VNET_FEATURES ("ip4-source-check-via-rx"),
951 };
952 
953 VNET_FEATURE_INIT (ip4_source_check_1, static) =
954 {
955  .arc_name = "ip4-unicast",
956  .node_name = "ip4-source-check-via-rx",
957  .runs_before = VNET_FEATURES ("ip4-source-check-via-any"),
958 };
959 
960 VNET_FEATURE_INIT (ip4_source_check_2, static) =
961 {
962  .arc_name = "ip4-unicast",
963  .node_name = "ip4-source-check-via-any",
964  .runs_before = VNET_FEATURES ("ip4-policer-classify"),
965 };
966 
968 {
969  .arc_name = "ip4-unicast",
970  .node_name = "ip4-source-and-port-range-check-rx",
971  .runs_before = VNET_FEATURES ("ip4-policer-classify"),
972 };
973 
975 {
976  .arc_name = "ip4-unicast",
977  .node_name = "ip4-policer-classify",
978  .runs_before = VNET_FEATURES ("ipsec-input-ip4"),
979 };
980 
981 VNET_FEATURE_INIT (ip4_ipsec, static) =
982 {
983  .arc_name = "ip4-unicast",
984  .node_name = "ipsec-input-ip4",
985  .runs_before = VNET_FEATURES ("vpath-input-ip4"),
986 };
987 
988 VNET_FEATURE_INIT (ip4_vpath, static) =
989 {
990  .arc_name = "ip4-unicast",
991  .node_name = "vpath-input-ip4",
992  .runs_before = VNET_FEATURES ("ip4-vxlan-bypass"),
993 };
994 
996 {
997  .arc_name = "ip4-unicast",
998  .node_name = "ip4-vxlan-bypass",
999  .runs_before = VNET_FEATURES ("ip4-lookup"),
1000 };
1001 
1002 VNET_FEATURE_INIT (ip4_lookup, static) =
1003 {
1004  .arc_name = "ip4-unicast",
1005  .node_name = "ip4-lookup",
1006  .runs_before = VNET_FEATURES ("ip4-drop"),
1007 };
1008 
1009 VNET_FEATURE_INIT (ip4_drop, static) =
1010 {
1011  .arc_name = "ip4-unicast",
1012  .node_name = "ip4-drop",
1013  .runs_before = 0, /* not before any other features */
1014 };
1015 
1016 
1017 /* Built-in ip4 multicast rx feature path definition */
1018 VNET_FEATURE_ARC_INIT (ip4_multicast, static) =
1019 {
1020  .arc_name = "ip4-multicast",
1021  .start_nodes = VNET_FEATURES ("ip4-input", "ip4-input-no-checksum"),
1022  .end_node = "ip4-lookup-multicast",
1023  .arc_index_ptr = &ip4_main.lookup_main.mcast_feature_arc_index,
1024 };
1025 
1026 VNET_FEATURE_INIT (ip4_vpath_mc, static) =
1027 {
1028  .arc_name = "ip4-multicast",
1029  .node_name = "vpath-input-ip4",
1030  .runs_before = VNET_FEATURES ("ip4-lookup-multicast"),
1031 };
1032 
1033 VNET_FEATURE_INIT (ip4_lookup_mc, static) =
1034 {
1035  .arc_name = "ip4-multicast",
1036  .node_name = "ip4-lookup-multicast",
1037  .runs_before = VNET_FEATURES ("ip4-drop"),
1038 };
1039 
1040 VNET_FEATURE_INIT (ip4_mc_drop, static) =
1041 {
1042  .arc_name = "ip4-multicast",
1043  .node_name = "ip4-drop",
1044  .runs_before = 0, /* last feature */
1045 };
1046 
1047 /* Source and port-range check ip4 tx feature path definition */
1048 VNET_FEATURE_ARC_INIT (ip4_output, static) =
1049 {
1050  .arc_name = "ip4-output",
1051  .start_nodes = VNET_FEATURES ("ip4-rewrite", "ip4-midchain"),
1052  .end_node = "interface-output",
1053  .arc_index_ptr = &ip4_main.lookup_main.output_feature_arc_index,
1054 };
1055 
1057 {
1058  .arc_name = "ip4-output",
1059  .node_name = "ip4-source-and-port-range-check-tx",
1060  .runs_before = VNET_FEATURES ("ipsec-output-ip4"),
1061 };
1062 
1063 VNET_FEATURE_INIT (ip4_ipsec_output, static) =
1064 {
1065  .arc_name = "ip4-output",
1066  .node_name = "ipsec-output-ip4",
1067  .runs_before = VNET_FEATURES ("interface-output"),
1068 };
1069 
1070 /* Built-in ip4 tx feature path definition */
1071 VNET_FEATURE_INIT (ip4_interface_output, static) =
1072 {
1073  .arc_name = "ip4-output",
1074  .node_name = "interface-output",
1075  .runs_before = 0, /* not before any other features */
1076 };
1077 /* *INDENT-ON* */
1078 
1079 static clib_error_t *
1080 ip4_sw_interface_add_del (vnet_main_t * vnm, u32 sw_if_index, u32 is_add)
1081 {
1082  ip4_main_t *im = &ip4_main;
1083 
1084  /* Fill in lookup tables with default table (0). */
1085  vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
1086 
1087  vnet_feature_enable_disable ("ip4-unicast", "ip4-drop", sw_if_index,
1088  is_add, 0, 0);
1089 
1090  vnet_feature_enable_disable ("ip4-multicast", "ip4-drop", sw_if_index,
1091  is_add, 0, 0);
1092 
1093  return /* no error */ 0;
1094 }
1095 
1097 
1098 /* Global IP4 main. */
1100 
1101 clib_error_t *
1103 {
1104  ip4_main_t *im = &ip4_main;
1105  clib_error_t *error;
1106  uword i;
1107 
1108  if ((error = vlib_call_init_function (vm, vnet_feature_init)))
1109  return error;
1110 
1111  for (i = 0; i < ARRAY_LEN (im->fib_masks); i++)
1112  {
1113  u32 m;
1114 
1115  if (i < 32)
1116  m = pow2_mask (i) << (32 - i);
1117  else
1118  m = ~0;
1119  im->fib_masks[i] = clib_host_to_net_u32 (m);
1120  }
1121 
1122  ip_lookup_init (&im->lookup_main, /* is_ip6 */ 0);
1123 
1124  /* Create FIB with index 0 and table id of 0. */
1126 
1127  {
1128  pg_node_t *pn;
1129  pn = pg_get_node (ip4_lookup_node.index);
1131  }
1132 
1133  {
1135 
1136  memset (&h, 0, sizeof (h));
1137 
1138  /* Set target ethernet address to all zeros. */
1139  memset (h.ip4_over_ethernet[1].ethernet, 0,
1140  sizeof (h.ip4_over_ethernet[1].ethernet));
1141 
1142 #define _16(f,v) h.f = clib_host_to_net_u16 (v);
1143 #define _8(f,v) h.f = v;
1144  _16 (l2_type, ETHERNET_ARP_HARDWARE_TYPE_ethernet);
1145  _16 (l3_type, ETHERNET_TYPE_IP4);
1146  _8 (n_l2_address_bytes, 6);
1147  _8 (n_l3_address_bytes, 4);
1148  _16 (opcode, ETHERNET_ARP_OPCODE_request);
1149 #undef _16
1150 #undef _8
1151 
1153  /* data */ &h,
1154  sizeof (h),
1155  /* alloc chunk size */ 8,
1156  "ip4 arp");
1157  }
1158 
1159  return error;
1160 }
1161 
1163 
1164 typedef struct
1165 {
1166  /* Adjacency taken. */
1170 
1171  /* Packet data, possibly *after* rewrite. */
1172  u8 packet_data[64 - 1 * sizeof (u32)];
1173 }
1175 
1176 u8 *
1177 format_ip4_forward_next_trace (u8 * s, va_list * args)
1178 {
1179  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1180  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1181  ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1182  uword indent = format_get_indent (s);
1183  s = format (s, "%U%U",
1184  format_white_space, indent,
1185  format_ip4_header, t->packet_data, sizeof (t->packet_data));
1186  return s;
1187 }
1188 
1189 static u8 *
1190 format_ip4_lookup_trace (u8 * s, va_list * args)
1191 {
1192  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1193  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1194  ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1195  uword indent = format_get_indent (s);
1196 
1197  s = format (s, "fib %d dpo-idx %d flow hash: 0x%08x",
1198  t->fib_index, t->dpo_index, t->flow_hash);
1199  s = format (s, "\n%U%U",
1200  format_white_space, indent,
1201  format_ip4_header, t->packet_data, sizeof (t->packet_data));
1202  return s;
1203 }
1204 
1205 static u8 *
1206 format_ip4_rewrite_trace (u8 * s, va_list * args)
1207 {
1208  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
1209  CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
1210  ip4_forward_next_trace_t *t = va_arg (*args, ip4_forward_next_trace_t *);
1211  vnet_main_t *vnm = vnet_get_main ();
1212  uword indent = format_get_indent (s);
1213 
1214  s = format (s, "tx_sw_if_index %d dpo-idx %d : %U flow hash: 0x%08x",
1217  s = format (s, "\n%U%U",
1218  format_white_space, indent,
1220  vnm, t->dpo_index, t->packet_data, sizeof (t->packet_data));
1221  return s;
1222 }
1223 
1224 /* Common trace function for all ip4-forward next nodes. */
1225 void
1227  vlib_node_runtime_t * node,
1228  vlib_frame_t * frame, vlib_rx_or_tx_t which_adj_index)
1229 {
1230  u32 *from, n_left;
1231  ip4_main_t *im = &ip4_main;
1232 
1233  n_left = frame->n_vectors;
1234  from = vlib_frame_vector_args (frame);
1235 
1236  while (n_left >= 4)
1237  {
1238  u32 bi0, bi1;
1239  vlib_buffer_t *b0, *b1;
1240  ip4_forward_next_trace_t *t0, *t1;
1241 
1242  /* Prefetch next iteration. */
1243  vlib_prefetch_buffer_with_index (vm, from[2], LOAD);
1244  vlib_prefetch_buffer_with_index (vm, from[3], LOAD);
1245 
1246  bi0 = from[0];
1247  bi1 = from[1];
1248 
1249  b0 = vlib_get_buffer (vm, bi0);
1250  b1 = vlib_get_buffer (vm, bi1);
1251 
1252  if (b0->flags & VLIB_BUFFER_IS_TRACED)
1253  {
1254  t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1255  t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1256  t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1257  t0->fib_index =
1258  (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1259  (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1261  vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1262 
1263  clib_memcpy (t0->packet_data,
1265  sizeof (t0->packet_data));
1266  }
1267  if (b1->flags & VLIB_BUFFER_IS_TRACED)
1268  {
1269  t1 = vlib_add_trace (vm, node, b1, sizeof (t1[0]));
1270  t1->dpo_index = vnet_buffer (b1)->ip.adj_index[which_adj_index];
1271  t1->flow_hash = vnet_buffer (b1)->ip.flow_hash;
1272  t1->fib_index =
1273  (vnet_buffer (b1)->sw_if_index[VLIB_TX] !=
1274  (u32) ~ 0) ? vnet_buffer (b1)->sw_if_index[VLIB_TX] :
1276  vnet_buffer (b1)->sw_if_index[VLIB_RX]);
1278  sizeof (t1->packet_data));
1279  }
1280  from += 2;
1281  n_left -= 2;
1282  }
1283 
1284  while (n_left >= 1)
1285  {
1286  u32 bi0;
1287  vlib_buffer_t *b0;
1289 
1290  bi0 = from[0];
1291 
1292  b0 = vlib_get_buffer (vm, bi0);
1293 
1294  if (b0->flags & VLIB_BUFFER_IS_TRACED)
1295  {
1296  t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
1297  t0->dpo_index = vnet_buffer (b0)->ip.adj_index[which_adj_index];
1298  t0->flow_hash = vnet_buffer (b0)->ip.flow_hash;
1299  t0->fib_index =
1300  (vnet_buffer (b0)->sw_if_index[VLIB_TX] !=
1301  (u32) ~ 0) ? vnet_buffer (b0)->sw_if_index[VLIB_TX] :
1303  vnet_buffer (b0)->sw_if_index[VLIB_RX]);
1305  sizeof (t0->packet_data));
1306  }
1307  from += 1;
1308  n_left -= 1;
1309  }
1310 }
1311 
1312 static uword
1314  vlib_node_runtime_t * node,
1315  vlib_frame_t * frame, ip4_error_t error_code)
1316 {
1317  u32 *buffers = vlib_frame_vector_args (frame);
1318  uword n_packets = frame->n_vectors;
1319 
1320  vlib_error_drop_buffers (vm, node, buffers,
1321  /* stride */ 1,
1322  n_packets,
1323  /* next */ 0,
1324  ip4_input_node.index, error_code);
1325 
1326  if (node->flags & VLIB_NODE_FLAG_TRACE)
1327  ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1328 
1329  return n_packets;
1330 }
1331 
1332 static uword
1334 {
1335  return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_DROP);
1336 }
1337 
1338 static uword
1340 {
1341  return ip4_drop_or_punt (vm, node, frame, IP4_ERROR_ADJACENCY_PUNT);
1342 }
1343 
1345 {
1346  .function = ip4_drop,.name = "ip4-drop",.vector_size =
1347  sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_next_nodes =
1348  1,.next_nodes =
1349  {
1350  [0] = "error-drop",}
1351 ,};
1352 
1354 
1356 {
1357  .function = ip4_punt,.name = "ip4-punt",.vector_size =
1358  sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_next_nodes =
1359  1,.next_nodes =
1360  {
1361  [0] = "error-punt",}
1362 ,};
1363 
1365 
1366 /* Compute TCP/UDP/ICMP4 checksum in software. */
1367 u16
1369  ip4_header_t * ip0)
1370 {
1371  ip_csum_t sum0;
1372  u32 ip_header_length, payload_length_host_byte_order;
1373  u32 n_this_buffer, n_bytes_left;
1374  u16 sum16;
1375  void *data_this_buffer;
1376 
1377  /* Initialize checksum with ip header. */
1378  ip_header_length = ip4_header_bytes (ip0);
1379  payload_length_host_byte_order =
1380  clib_net_to_host_u16 (ip0->length) - ip_header_length;
1381  sum0 =
1382  clib_host_to_net_u32 (payload_length_host_byte_order +
1383  (ip0->protocol << 16));
1384 
1385  if (BITS (uword) == 32)
1386  {
1387  sum0 =
1388  ip_csum_with_carry (sum0,
1390  sum0 =
1391  ip_csum_with_carry (sum0,
1393  }
1394  else
1395  sum0 =
1397 
1398  n_bytes_left = n_this_buffer = payload_length_host_byte_order;
1399  data_this_buffer = (void *) ip0 + ip_header_length;
1400  if (n_this_buffer + ip_header_length > p0->current_length)
1401  n_this_buffer =
1402  p0->current_length >
1403  ip_header_length ? p0->current_length - ip_header_length : 0;
1404  while (1)
1405  {
1406  sum0 = ip_incremental_checksum (sum0, data_this_buffer, n_this_buffer);
1407  n_bytes_left -= n_this_buffer;
1408  if (n_bytes_left == 0)
1409  break;
1410 
1412  p0 = vlib_get_buffer (vm, p0->next_buffer);
1413  data_this_buffer = vlib_buffer_get_current (p0);
1414  n_this_buffer = p0->current_length;
1415  }
1416 
1417  sum16 = ~ip_csum_fold (sum0);
1418 
1419  return sum16;
1420 }
1421 
1422 u32
1424 {
1426  udp_header_t *udp0;
1427  u16 sum16;
1428 
1429  ASSERT (ip0->protocol == IP_PROTOCOL_TCP
1430  || ip0->protocol == IP_PROTOCOL_UDP);
1431 
1432  udp0 = (void *) (ip0 + 1);
1433  if (ip0->protocol == IP_PROTOCOL_UDP && udp0->checksum == 0)
1434  {
1437  return p0->flags;
1438  }
1439 
1440  sum16 = ip4_tcp_udp_compute_checksum (vm, p0, ip0);
1441 
1443  | ((sum16 == 0) << LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT));
1444 
1445  return p0->flags;
1446 }
1447 
1448 static uword
1450 {
1451  ip4_main_t *im = &ip4_main;
1452  ip_lookup_main_t *lm = &im->lookup_main;
1453  ip_local_next_t next_index;
1454  u32 *from, *to_next, n_left_from, n_left_to_next;
1455  vlib_node_runtime_t *error_node =
1457 
1458  from = vlib_frame_vector_args (frame);
1459  n_left_from = frame->n_vectors;
1460  next_index = node->cached_next_index;
1461 
1462  if (node->flags & VLIB_NODE_FLAG_TRACE)
1463  ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1464 
1465  while (n_left_from > 0)
1466  {
1467  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1468 
1469  while (n_left_from >= 4 && n_left_to_next >= 2)
1470  {
1471  vlib_buffer_t *p0, *p1;
1472  ip4_header_t *ip0, *ip1;
1473  udp_header_t *udp0, *udp1;
1474  ip4_fib_mtrie_t *mtrie0, *mtrie1;
1475  ip4_fib_mtrie_leaf_t leaf0, leaf1;
1476  const dpo_id_t *dpo0, *dpo1;
1477  const load_balance_t *lb0, *lb1;
1478  u32 pi0, ip_len0, udp_len0, flags0, next0, fib_index0, lbi0;
1479  u32 pi1, ip_len1, udp_len1, flags1, next1, fib_index1, lbi1;
1480  i32 len_diff0, len_diff1;
1481  u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1482  u8 error1, is_udp1, is_tcp_udp1, good_tcp_udp1, proto1;
1483  u8 enqueue_code;
1484 
1485  pi0 = to_next[0] = from[0];
1486  pi1 = to_next[1] = from[1];
1487  from += 2;
1488  n_left_from -= 2;
1489  to_next += 2;
1490  n_left_to_next -= 2;
1491 
1492  p0 = vlib_get_buffer (vm, pi0);
1493  p1 = vlib_get_buffer (vm, pi1);
1494 
1495  ip0 = vlib_buffer_get_current (p0);
1496  ip1 = vlib_buffer_get_current (p1);
1497 
1498  vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data;
1499  vnet_buffer (p1)->ip.start_of_ip_header = p1->current_data;
1500 
1501  fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
1502  vnet_buffer (p0)->sw_if_index[VLIB_RX]);
1503  fib_index0 = (vnet_buffer (p0)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
1504  fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1505 
1506  fib_index1 = vec_elt (im->fib_index_by_sw_if_index,
1507  vnet_buffer (p1)->sw_if_index[VLIB_RX]);
1508  fib_index1 = (vnet_buffer (p1)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
1509  fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
1510 
1511  mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1512  mtrie1 = &ip4_fib_get (fib_index1)->mtrie;
1513 
1514  leaf0 = leaf1 = IP4_FIB_MTRIE_LEAF_ROOT;
1515 
1516  leaf0 =
1517  ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
1518  leaf1 =
1519  ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 0);
1520 
1521  /* Treat IP frag packets as "experimental" protocol for now
1522  until support of IP frag reassembly is implemented */
1523  proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1524  proto1 = ip4_is_fragment (ip1) ? 0xfe : ip1->protocol;
1525  is_udp0 = proto0 == IP_PROTOCOL_UDP;
1526  is_udp1 = proto1 == IP_PROTOCOL_UDP;
1527  is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1528  is_tcp_udp1 = is_udp1 || proto1 == IP_PROTOCOL_TCP;
1529 
1530  flags0 = p0->flags;
1531  flags1 = p1->flags;
1532 
1533  good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1534  good_tcp_udp1 = (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1535 
1536  udp0 = ip4_next_header (ip0);
1537  udp1 = ip4_next_header (ip1);
1538 
1539  /* Don't verify UDP checksum for packets with explicit zero checksum. */
1540  good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1541  good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1542 
1543  leaf0 =
1544  ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1545  leaf1 =
1546  ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 1);
1547 
1548  /* Verify UDP length. */
1549  ip_len0 = clib_net_to_host_u16 (ip0->length);
1550  ip_len1 = clib_net_to_host_u16 (ip1->length);
1551  udp_len0 = clib_net_to_host_u16 (udp0->length);
1552  udp_len1 = clib_net_to_host_u16 (udp1->length);
1553 
1554  len_diff0 = ip_len0 - udp_len0;
1555  len_diff1 = ip_len1 - udp_len1;
1556 
1557  len_diff0 = is_udp0 ? len_diff0 : 0;
1558  len_diff1 = is_udp1 ? len_diff1 : 0;
1559 
1560  if (PREDICT_FALSE (!(is_tcp_udp0 & is_tcp_udp1
1561  & good_tcp_udp0 & good_tcp_udp1)))
1562  {
1563  if (is_tcp_udp0)
1564  {
1565  if (is_tcp_udp0
1566  && !(flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1567  flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1568  good_tcp_udp0 =
1569  (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1570  good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1571  }
1572  if (is_tcp_udp1)
1573  {
1574  if (is_tcp_udp1
1575  && !(flags1 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1576  flags1 = ip4_tcp_udp_validate_checksum (vm, p1);
1577  good_tcp_udp1 =
1578  (flags1 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1579  good_tcp_udp1 |= is_udp1 && udp1->checksum == 0;
1580  }
1581  }
1582 
1583  good_tcp_udp0 &= len_diff0 >= 0;
1584  good_tcp_udp1 &= len_diff1 >= 0;
1585 
1586  leaf0 =
1587  ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1588  leaf1 =
1589  ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 2);
1590 
1591  error0 = error1 = IP4_ERROR_UNKNOWN_PROTOCOL;
1592 
1593  error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1594  error1 = len_diff1 < 0 ? IP4_ERROR_UDP_LENGTH : error1;
1595 
1596  ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1597  error0 = (is_tcp_udp0 && !good_tcp_udp0
1598  ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1599  error1 = (is_tcp_udp1 && !good_tcp_udp1
1600  ? IP4_ERROR_TCP_CHECKSUM + is_udp1 : error1);
1601 
1602  leaf0 =
1603  ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1604  leaf1 =
1605  ip4_fib_mtrie_lookup_step (mtrie1, leaf1, &ip1->src_address, 3);
1606  leaf0 =
1607  (leaf0 ==
1608  IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
1609  leaf1 =
1610  (leaf1 ==
1611  IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie1->default_leaf : leaf1);
1612 
1613  vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0 =
1615  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1616 
1617  vnet_buffer (p1)->ip.adj_index[VLIB_RX] = lbi1 =
1619  vnet_buffer (p1)->ip.adj_index[VLIB_TX] = lbi1;
1620 
1621  lb0 = load_balance_get (lbi0);
1622  lb1 = load_balance_get (lbi1);
1623  dpo0 = load_balance_get_bucket_i (lb0, 0);
1624  dpo1 = load_balance_get_bucket_i (lb1, 0);
1625 
1626  /*
1627  * Must have a route to source otherwise we drop the packet.
1628  * ip4 broadcasts are accepted, e.g. to make dhcp client work
1629  *
1630  * The checks are:
1631  * - the source is a recieve => it's from us => bogus, do this
1632  * first since it sets a different error code.
1633  * - uRPF check for any route to source - accept if passes.
1634  * - allow packets destined to the broadcast address from unknown sources
1635  */
1636  error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1637  dpo0->dpoi_type == DPO_RECEIVE) ?
1638  IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1639  error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1640  !fib_urpf_check_size (lb0->lb_urpf) &&
1641  ip0->dst_address.as_u32 != 0xFFFFFFFF)
1642  ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1643  error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1644  dpo1->dpoi_type == DPO_RECEIVE) ?
1645  IP4_ERROR_SPOOFED_LOCAL_PACKETS : error1);
1646  error1 = ((error1 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1647  !fib_urpf_check_size (lb1->lb_urpf) &&
1648  ip1->dst_address.as_u32 != 0xFFFFFFFF)
1649  ? IP4_ERROR_SRC_LOOKUP_MISS : error1);
1650 
1651  next0 = lm->local_next_by_ip_protocol[proto0];
1652  next1 = lm->local_next_by_ip_protocol[proto1];
1653 
1654  next0 =
1655  error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1656  next1 =
1657  error1 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next1;
1658 
1659  p0->error = error0 ? error_node->errors[error0] : 0;
1660  p1->error = error1 ? error_node->errors[error1] : 0;
1661 
1662  enqueue_code = (next0 != next_index) + 2 * (next1 != next_index);
1663 
1664  if (PREDICT_FALSE (enqueue_code != 0))
1665  {
1666  switch (enqueue_code)
1667  {
1668  case 1:
1669  /* A B A */
1670  to_next[-2] = pi1;
1671  to_next -= 1;
1672  n_left_to_next += 1;
1673  vlib_set_next_frame_buffer (vm, node, next0, pi0);
1674  break;
1675 
1676  case 2:
1677  /* A A B */
1678  to_next -= 1;
1679  n_left_to_next += 1;
1680  vlib_set_next_frame_buffer (vm, node, next1, pi1);
1681  break;
1682 
1683  case 3:
1684  /* A B B or A B C */
1685  to_next -= 2;
1686  n_left_to_next += 2;
1687  vlib_set_next_frame_buffer (vm, node, next0, pi0);
1688  vlib_set_next_frame_buffer (vm, node, next1, pi1);
1689  if (next0 == next1)
1690  {
1691  vlib_put_next_frame (vm, node, next_index,
1692  n_left_to_next);
1693  next_index = next1;
1694  vlib_get_next_frame (vm, node, next_index, to_next,
1695  n_left_to_next);
1696  }
1697  break;
1698  }
1699  }
1700  }
1701 
1702  while (n_left_from > 0 && n_left_to_next > 0)
1703  {
1704  vlib_buffer_t *p0;
1705  ip4_header_t *ip0;
1706  udp_header_t *udp0;
1707  ip4_fib_mtrie_t *mtrie0;
1708  ip4_fib_mtrie_leaf_t leaf0;
1709  u32 pi0, next0, ip_len0, udp_len0, flags0, fib_index0, lbi0;
1710  i32 len_diff0;
1711  u8 error0, is_udp0, is_tcp_udp0, good_tcp_udp0, proto0;
1712  load_balance_t *lb0;
1713  const dpo_id_t *dpo0;
1714 
1715  pi0 = to_next[0] = from[0];
1716  from += 1;
1717  n_left_from -= 1;
1718  to_next += 1;
1719  n_left_to_next -= 1;
1720 
1721  p0 = vlib_get_buffer (vm, pi0);
1722 
1723  ip0 = vlib_buffer_get_current (p0);
1724 
1725  vnet_buffer (p0)->ip.start_of_ip_header = p0->current_data;
1726 
1727  fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
1728  vnet_buffer (p0)->sw_if_index[VLIB_RX]);
1729  fib_index0 = (vnet_buffer (p0)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
1730  fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
1731 
1732  mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
1733 
1734  leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
1735 
1736  leaf0 =
1737  ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 0);
1738 
1739  /* Treat IP frag packets as "experimental" protocol for now
1740  until support of IP frag reassembly is implemented */
1741  proto0 = ip4_is_fragment (ip0) ? 0xfe : ip0->protocol;
1742  is_udp0 = proto0 == IP_PROTOCOL_UDP;
1743  is_tcp_udp0 = is_udp0 || proto0 == IP_PROTOCOL_TCP;
1744 
1745  flags0 = p0->flags;
1746 
1747  good_tcp_udp0 = (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1748 
1749  udp0 = ip4_next_header (ip0);
1750 
1751  /* Don't verify UDP checksum for packets with explicit zero checksum. */
1752  good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1753 
1754  leaf0 =
1755  ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 1);
1756 
1757  /* Verify UDP length. */
1758  ip_len0 = clib_net_to_host_u16 (ip0->length);
1759  udp_len0 = clib_net_to_host_u16 (udp0->length);
1760 
1761  len_diff0 = ip_len0 - udp_len0;
1762 
1763  len_diff0 = is_udp0 ? len_diff0 : 0;
1764 
1765  if (PREDICT_FALSE (!(is_tcp_udp0 & good_tcp_udp0)))
1766  {
1767  if (is_tcp_udp0)
1768  {
1769  if (is_tcp_udp0
1770  && !(flags0 & IP_BUFFER_L4_CHECKSUM_COMPUTED))
1771  flags0 = ip4_tcp_udp_validate_checksum (vm, p0);
1772  good_tcp_udp0 =
1773  (flags0 & IP_BUFFER_L4_CHECKSUM_CORRECT) != 0;
1774  good_tcp_udp0 |= is_udp0 && udp0->checksum == 0;
1775  }
1776  }
1777 
1778  good_tcp_udp0 &= len_diff0 >= 0;
1779 
1780  leaf0 =
1781  ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 2);
1782 
1783  error0 = IP4_ERROR_UNKNOWN_PROTOCOL;
1784 
1785  error0 = len_diff0 < 0 ? IP4_ERROR_UDP_LENGTH : error0;
1786 
1787  ASSERT (IP4_ERROR_TCP_CHECKSUM + 1 == IP4_ERROR_UDP_CHECKSUM);
1788  error0 = (is_tcp_udp0 && !good_tcp_udp0
1789  ? IP4_ERROR_TCP_CHECKSUM + is_udp0 : error0);
1790 
1791  leaf0 =
1792  ip4_fib_mtrie_lookup_step (mtrie0, leaf0, &ip0->src_address, 3);
1793  leaf0 =
1794  (leaf0 ==
1795  IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
1796 
1797  lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
1798  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = lbi0;
1799 
1800  lb0 = load_balance_get (lbi0);
1801  dpo0 = load_balance_get_bucket_i (lb0, 0);
1802 
1803  vnet_buffer (p0)->ip.adj_index[VLIB_TX] =
1804  vnet_buffer (p0)->ip.adj_index[VLIB_RX] = lbi0;
1805 
1806  error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1807  dpo0->dpoi_type == DPO_RECEIVE) ?
1808  IP4_ERROR_SPOOFED_LOCAL_PACKETS : error0);
1809  error0 = ((error0 == IP4_ERROR_UNKNOWN_PROTOCOL &&
1810  !fib_urpf_check_size (lb0->lb_urpf) &&
1811  ip0->dst_address.as_u32 != 0xFFFFFFFF)
1812  ? IP4_ERROR_SRC_LOOKUP_MISS : error0);
1813 
1814  next0 = lm->local_next_by_ip_protocol[proto0];
1815 
1816  next0 =
1817  error0 != IP4_ERROR_UNKNOWN_PROTOCOL ? IP_LOCAL_NEXT_DROP : next0;
1818 
1819  p0->error = error0 ? error_node->errors[error0] : 0;
1820 
1821  if (PREDICT_FALSE (next0 != next_index))
1822  {
1823  n_left_to_next += 1;
1824  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1825 
1826  next_index = next0;
1827  vlib_get_next_frame (vm, node, next_index, to_next,
1828  n_left_to_next);
1829  to_next[0] = pi0;
1830  to_next += 1;
1831  n_left_to_next -= 1;
1832  }
1833  }
1834 
1835  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1836  }
1837 
1838  return frame->n_vectors;
1839 }
1840 
1842 {
1843  .function = ip4_local,.name = "ip4-local",.vector_size =
1844  sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_next_nodes =
1845  IP_LOCAL_N_NEXT,.next_nodes =
1846  {
1847  [IP_LOCAL_NEXT_DROP] = "error-drop",
1848  [IP_LOCAL_NEXT_PUNT] = "error-punt",
1849  [IP_LOCAL_NEXT_UDP_LOOKUP] = "ip4-udp-lookup",
1850  [IP_LOCAL_NEXT_ICMP] = "ip4-icmp-input",}
1851 ,};
1852 
1854 
1855 void
1856 ip4_register_protocol (u32 protocol, u32 node_index)
1857 {
1858  vlib_main_t *vm = vlib_get_main ();
1859  ip4_main_t *im = &ip4_main;
1860  ip_lookup_main_t *lm = &im->lookup_main;
1861 
1862  ASSERT (protocol < ARRAY_LEN (lm->local_next_by_ip_protocol));
1863  lm->local_next_by_ip_protocol[protocol] =
1864  vlib_node_add_next (vm, ip4_local_node.index, node_index);
1865 }
1866 
1867 static clib_error_t *
1869  unformat_input_t * input, vlib_cli_command_t * cmd)
1870 {
1871  ip4_main_t *im = &ip4_main;
1872  ip_lookup_main_t *lm = &im->lookup_main;
1873  int i;
1874 
1875  vlib_cli_output (vm, "Protocols handled by ip4_local");
1876  for (i = 0; i < ARRAY_LEN (lm->local_next_by_ip_protocol); i++)
1877  {
1879  vlib_cli_output (vm, "%d", i);
1880  }
1881  return 0;
1882 }
1883 
1884 
1885 
1886 /*?
1887  * Display the set of protocols handled by the local IPv4 stack.
1888  *
1889  * @cliexpar
1890  * Example of how to display local protocol table:
1891  * @cliexstart{show ip local}
1892  * Protocols handled by ip4_local
1893  * 1
1894  * 17
1895  * 47
1896  * @cliexend
1897 ?*/
1898 /* *INDENT-OFF* */
1899 VLIB_CLI_COMMAND (show_ip_local, static) =
1900 {
1901  .path = "show ip local",
1902  .function = show_ip_local_command_fn,
1903  .short_help = "show ip local",
1904 };
1905 /* *INDENT-ON* */
1906 
1909  vlib_node_runtime_t * node,
1910  vlib_frame_t * frame, int is_glean)
1911 {
1912  vnet_main_t *vnm = vnet_get_main ();
1913  ip4_main_t *im = &ip4_main;
1914  ip_lookup_main_t *lm = &im->lookup_main;
1915  u32 *from, *to_next_drop;
1916  uword n_left_from, n_left_to_next_drop, next_index;
1917  static f64 time_last_seed_change = -1e100;
1918  static u32 hash_seeds[3];
1919  static uword hash_bitmap[256 / BITS (uword)];
1920  f64 time_now;
1921 
1922  if (node->flags & VLIB_NODE_FLAG_TRACE)
1923  ip4_forward_next_trace (vm, node, frame, VLIB_TX);
1924 
1925  time_now = vlib_time_now (vm);
1926  if (time_now - time_last_seed_change > 1e-3)
1927  {
1928  uword i;
1930  sizeof (hash_seeds));
1931  for (i = 0; i < ARRAY_LEN (hash_seeds); i++)
1932  hash_seeds[i] = r[i];
1933 
1934  /* Mark all hash keys as been no-seen before. */
1935  for (i = 0; i < ARRAY_LEN (hash_bitmap); i++)
1936  hash_bitmap[i] = 0;
1937 
1938  time_last_seed_change = time_now;
1939  }
1940 
1941  from = vlib_frame_vector_args (frame);
1942  n_left_from = frame->n_vectors;
1943  next_index = node->cached_next_index;
1944  if (next_index == IP4_ARP_NEXT_DROP)
1945  next_index = IP4_ARP_N_NEXT; /* point to first interface */
1946 
1947  while (n_left_from > 0)
1948  {
1950  to_next_drop, n_left_to_next_drop);
1951 
1952  while (n_left_from > 0 && n_left_to_next_drop > 0)
1953  {
1954  u32 pi0, adj_index0, a0, b0, c0, m0, sw_if_index0, drop0;
1955  ip_adjacency_t *adj0;
1956  vlib_buffer_t *p0;
1957  ip4_header_t *ip0;
1958  uword bm0;
1959 
1960  pi0 = from[0];
1961 
1962  p0 = vlib_get_buffer (vm, pi0);
1963 
1964  adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
1965  adj0 = ip_get_adjacency (lm, adj_index0);
1966  ip0 = vlib_buffer_get_current (p0);
1967 
1968  a0 = hash_seeds[0];
1969  b0 = hash_seeds[1];
1970  c0 = hash_seeds[2];
1971 
1972  sw_if_index0 = adj0->rewrite_header.sw_if_index;
1973  vnet_buffer (p0)->sw_if_index[VLIB_TX] = sw_if_index0;
1974 
1975  if (is_glean)
1976  {
1977  /*
1978  * this is the Glean case, so we are ARPing for the
1979  * packet's destination
1980  */
1981  a0 ^= ip0->dst_address.data_u32;
1982  }
1983  else
1984  {
1985  a0 ^= adj0->sub_type.nbr.next_hop.ip4.data_u32;
1986  }
1987  b0 ^= sw_if_index0;
1988 
1989  hash_v3_finalize32 (a0, b0, c0);
1990 
1991  c0 &= BITS (hash_bitmap) - 1;
1992  c0 = c0 / BITS (uword);
1993  m0 = (uword) 1 << (c0 % BITS (uword));
1994 
1995  bm0 = hash_bitmap[c0];
1996  drop0 = (bm0 & m0) != 0;
1997 
1998  /* Mark it as seen. */
1999  hash_bitmap[c0] = bm0 | m0;
2000 
2001  from += 1;
2002  n_left_from -= 1;
2003  to_next_drop[0] = pi0;
2004  to_next_drop += 1;
2005  n_left_to_next_drop -= 1;
2006 
2007  p0->error =
2008  node->errors[drop0 ? IP4_ARP_ERROR_DROP :
2010 
2011  /*
2012  * the adj has been updated to a rewrite but the node the DPO that got
2013  * us here hasn't - yet. no big deal. we'll drop while we wait.
2014  */
2016  continue;
2017 
2018  if (drop0)
2019  continue;
2020 
2021  /*
2022  * Can happen if the control-plane is programming tables
2023  * with traffic flowing; at least that's today's lame excuse.
2024  */
2025  if ((is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_GLEAN) ||
2026  (!is_glean && adj0->lookup_next_index != IP_LOOKUP_NEXT_ARP))
2027  {
2028  p0->error = node->errors[IP4_ARP_ERROR_NON_ARP_ADJ];
2029  }
2030  else
2031  /* Send ARP request. */
2032  {
2033  u32 bi0 = 0;
2034  vlib_buffer_t *b0;
2036  vnet_hw_interface_t *hw_if0;
2037 
2038  h0 =
2041  &bi0);
2042 
2043  /* Add rewrite/encap string for ARP packet. */
2044  vnet_rewrite_one_header (adj0[0], h0,
2045  sizeof (ethernet_header_t));
2046 
2047  hw_if0 = vnet_get_sup_hw_interface (vnm, sw_if_index0);
2048 
2049  /* Src ethernet address in ARP header. */
2050  clib_memcpy (h0->ip4_over_ethernet[0].ethernet,
2051  hw_if0->hw_address,
2052  sizeof (h0->ip4_over_ethernet[0].ethernet));
2053 
2054  if (is_glean)
2055  {
2056  /* The interface's source address is stashed in the Glean Adj */
2057  h0->ip4_over_ethernet[0].ip4 =
2058  adj0->sub_type.glean.receive_addr.ip4;
2059 
2060  /* Copy in destination address we are requesting. This is the
2061  * glean case, so it's the packet's destination.*/
2062  h0->ip4_over_ethernet[1].ip4.data_u32 =
2063  ip0->dst_address.data_u32;
2064  }
2065  else
2066  {
2067  /* Src IP address in ARP header. */
2068  if (ip4_src_address_for_packet (lm, sw_if_index0,
2069  &h0->
2070  ip4_over_ethernet[0].ip4))
2071  {
2072  /* No source address available */
2073  p0->error =
2075  vlib_buffer_free (vm, &bi0, 1);
2076  continue;
2077  }
2078 
2079  /* Copy in destination address we are requesting from the
2080  incomplete adj */
2081  h0->ip4_over_ethernet[1].ip4.data_u32 =
2082  adj0->sub_type.nbr.next_hop.ip4.as_u32;
2083  }
2084 
2085  vlib_buffer_copy_trace_flag (vm, p0, bi0);
2086  b0 = vlib_get_buffer (vm, bi0);
2087  vnet_buffer (b0)->sw_if_index[VLIB_TX] = sw_if_index0;
2088 
2089  vlib_buffer_advance (b0, -adj0->rewrite_header.data_bytes);
2090 
2091  vlib_set_next_frame_buffer (vm, node,
2092  adj0->rewrite_header.next_index,
2093  bi0);
2094  }
2095  }
2096 
2097  vlib_put_next_frame (vm, node, IP4_ARP_NEXT_DROP, n_left_to_next_drop);
2098  }
2099 
2100  return frame->n_vectors;
2101 }
2102 
2103 static uword
2105 {
2106  return (ip4_arp_inline (vm, node, frame, 0));
2107 }
2108 
2109 static uword
2111 {
2112  return (ip4_arp_inline (vm, node, frame, 1));
2113 }
2114 
2115 static char *ip4_arp_error_strings[] = {
2116  [IP4_ARP_ERROR_DROP] = "address overflow drops",
2117  [IP4_ARP_ERROR_REQUEST_SENT] = "ARP requests sent",
2118  [IP4_ARP_ERROR_NON_ARP_ADJ] = "ARPs to non-ARP adjacencies",
2119  [IP4_ARP_ERROR_REPLICATE_DROP] = "ARP replication completed",
2120  [IP4_ARP_ERROR_REPLICATE_FAIL] = "ARP replication failed",
2121  [IP4_ARP_ERROR_NO_SOURCE_ADDRESS] = "no source address for ARP request",
2122 };
2123 
2125 {
2126  .function = ip4_arp,.name = "ip4-arp",.vector_size =
2127  sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2128  ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2129  ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2130  {
2131  [IP4_ARP_NEXT_DROP] = "error-drop",}
2132 ,};
2133 
2135 {
2136  .function = ip4_glean,.name = "ip4-glean",.vector_size =
2137  sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_errors =
2138  ARRAY_LEN (ip4_arp_error_strings),.error_strings =
2139  ip4_arp_error_strings,.n_next_nodes = IP4_ARP_N_NEXT,.next_nodes =
2140  {
2141  [IP4_ARP_NEXT_DROP] = "error-drop",}
2142 ,};
2143 
2144 #define foreach_notrace_ip4_arp_error \
2145 _(DROP) \
2146 _(REQUEST_SENT) \
2147 _(REPLICATE_DROP) \
2148 _(REPLICATE_FAIL)
2149 
2150 clib_error_t *
2152 {
2154 
2155  /* don't trace ARP request packets */
2156 #define _(a) \
2157  vnet_pcap_drop_trace_filter_add_del \
2158  (rt->errors[IP4_ARP_ERROR_##a], \
2159  1 /* is_add */);
2161 #undef _
2162  return 0;
2163 }
2164 
2166 
2167 
2168 /* Send an ARP request to see if given destination is reachable on given interface. */
2169 clib_error_t *
2171 {
2172  vnet_main_t *vnm = vnet_get_main ();
2173  ip4_main_t *im = &ip4_main;
2175  ip4_address_t *src;
2177  ip_adjacency_t *adj;
2179  vnet_sw_interface_t *si;
2180  vlib_buffer_t *b;
2181  u32 bi = 0;
2182 
2183  si = vnet_get_sw_interface (vnm, sw_if_index);
2184 
2186  {
2187  return clib_error_return (0, "%U: interface %U down",
2188  format_ip4_address, dst,
2190  sw_if_index);
2191  }
2192 
2193  src =
2194  ip4_interface_address_matching_destination (im, dst, sw_if_index, &ia);
2195  if (!src)
2196  {
2197  vnm->api_errno = VNET_API_ERROR_NO_MATCHING_INTERFACE;
2198  return clib_error_return
2199  (0, "no matching interface address for destination %U (interface %U)",
2200  format_ip4_address, dst,
2201  format_vnet_sw_if_index_name, vnm, sw_if_index);
2202  }
2203 
2205 
2206  h =
2208  &bi);
2209 
2210  hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
2211 
2212  clib_memcpy (h->ip4_over_ethernet[0].ethernet, hi->hw_address,
2213  sizeof (h->ip4_over_ethernet[0].ethernet));
2214 
2215  h->ip4_over_ethernet[0].ip4 = src[0];
2216  h->ip4_over_ethernet[1].ip4 = dst[0];
2217 
2218  b = vlib_get_buffer (vm, bi);
2219  vnet_buffer (b)->sw_if_index[VLIB_RX] =
2220  vnet_buffer (b)->sw_if_index[VLIB_TX] = sw_if_index;
2221 
2222  /* Add encapsulation string for software interface (e.g. ethernet header). */
2223  vnet_rewrite_one_header (adj[0], h, sizeof (ethernet_header_t));
2224  vlib_buffer_advance (b, -adj->rewrite_header.data_bytes);
2225 
2226  {
2228  u32 *to_next = vlib_frame_vector_args (f);
2229  to_next[0] = bi;
2230  f->n_vectors = 1;
2232  }
2233 
2234  return /* no error */ 0;
2235 }
2236 
2237 typedef enum
2238 {
2242 
2245  vlib_node_runtime_t * node,
2246  vlib_frame_t * frame, int is_midchain)
2247 {
2248  ip_lookup_main_t *lm = &ip4_main.lookup_main;
2249  u32 *from = vlib_frame_vector_args (frame);
2250  u32 n_left_from, n_left_to_next, *to_next, next_index;
2251  vlib_node_runtime_t *error_node =
2253 
2254  n_left_from = frame->n_vectors;
2255  next_index = node->cached_next_index;
2256  u32 cpu_index = os_get_cpu_number ();
2257 
2258  while (n_left_from > 0)
2259  {
2260  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
2261 
2262  while (n_left_from >= 4 && n_left_to_next >= 2)
2263  {
2264  ip_adjacency_t *adj0, *adj1;
2265  vlib_buffer_t *p0, *p1;
2266  ip4_header_t *ip0, *ip1;
2267  u32 pi0, rw_len0, next0, error0, checksum0, adj_index0;
2268  u32 pi1, rw_len1, next1, error1, checksum1, adj_index1;
2269  u32 tx_sw_if_index0, tx_sw_if_index1;
2270 
2271  /* Prefetch next iteration. */
2272  {
2273  vlib_buffer_t *p2, *p3;
2274 
2275  p2 = vlib_get_buffer (vm, from[2]);
2276  p3 = vlib_get_buffer (vm, from[3]);
2277 
2278  vlib_prefetch_buffer_header (p2, STORE);
2279  vlib_prefetch_buffer_header (p3, STORE);
2280 
2281  CLIB_PREFETCH (p2->data, sizeof (ip0[0]), STORE);
2282  CLIB_PREFETCH (p3->data, sizeof (ip0[0]), STORE);
2283  }
2284 
2285  pi0 = to_next[0] = from[0];
2286  pi1 = to_next[1] = from[1];
2287 
2288  from += 2;
2289  n_left_from -= 2;
2290  to_next += 2;
2291  n_left_to_next -= 2;
2292 
2293  p0 = vlib_get_buffer (vm, pi0);
2294  p1 = vlib_get_buffer (vm, pi1);
2295 
2296  adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2297  adj_index1 = vnet_buffer (p1)->ip.adj_index[VLIB_TX];
2298 
2299  /* We should never rewrite a pkt using the MISS adjacency */
2300  ASSERT (adj_index0 && adj_index1);
2301 
2302  ip0 = vlib_buffer_get_current (p0);
2303  ip1 = vlib_buffer_get_current (p1);
2304 
2305  error0 = error1 = IP4_ERROR_NONE;
2306  next0 = next1 = IP4_REWRITE_NEXT_DROP;
2307 
2308  /* Decrement TTL & update checksum.
2309  Works either endian, so no need for byte swap. */
2311  {
2312  i32 ttl0 = ip0->ttl;
2313 
2314  /* Input node should have reject packets with ttl 0. */
2315  ASSERT (ip0->ttl > 0);
2316 
2317  checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2318  checksum0 += checksum0 >= 0xffff;
2319 
2320  ip0->checksum = checksum0;
2321  ttl0 -= 1;
2322  ip0->ttl = ttl0;
2323 
2324  /*
2325  * If the ttl drops below 1 when forwarding, generate
2326  * an ICMP response.
2327  */
2328  if (PREDICT_FALSE (ttl0 <= 0))
2329  {
2330  error0 = IP4_ERROR_TIME_EXPIRED;
2331  vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2332  icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2333  ICMP4_time_exceeded_ttl_exceeded_in_transit,
2334  0);
2336  }
2337 
2338  /* Verify checksum. */
2339  ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2340  }
2341  else
2342  {
2344  }
2346  {
2347  i32 ttl1 = ip1->ttl;
2348 
2349  /* Input node should have reject packets with ttl 0. */
2350  ASSERT (ip1->ttl > 0);
2351 
2352  checksum1 = ip1->checksum + clib_host_to_net_u16 (0x0100);
2353  checksum1 += checksum1 >= 0xffff;
2354 
2355  ip1->checksum = checksum1;
2356  ttl1 -= 1;
2357  ip1->ttl = ttl1;
2358 
2359  /*
2360  * If the ttl drops below 1 when forwarding, generate
2361  * an ICMP response.
2362  */
2363  if (PREDICT_FALSE (ttl1 <= 0))
2364  {
2365  error1 = IP4_ERROR_TIME_EXPIRED;
2366  vnet_buffer (p1)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2367  icmp4_error_set_vnet_buffer (p1, ICMP4_time_exceeded,
2368  ICMP4_time_exceeded_ttl_exceeded_in_transit,
2369  0);
2371  }
2372 
2373  /* Verify checksum. */
2374  ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2375  ASSERT (ip1->checksum == ip4_header_checksum (ip1));
2376  }
2377  else
2378  {
2380  }
2381 
2382  /* Rewrite packet header and updates lengths. */
2383  adj0 = ip_get_adjacency (lm, adj_index0);
2384  adj1 = ip_get_adjacency (lm, adj_index1);
2385 
2386  /* Worth pipelining. No guarantee that adj0,1 are hot... */
2387  rw_len0 = adj0[0].rewrite_header.data_bytes;
2388  rw_len1 = adj1[0].rewrite_header.data_bytes;
2389  vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2390  vnet_buffer (p1)->ip.save_rewrite_length = rw_len1;
2391 
2392  /* Check MTU of outgoing interface. */
2393  error0 =
2394  (vlib_buffer_length_in_chain (vm, p0) >
2395  adj0[0].
2396  rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2397  error0);
2398  error1 =
2399  (vlib_buffer_length_in_chain (vm, p1) >
2400  adj1[0].
2401  rewrite_header.max_l3_packet_bytes ? IP4_ERROR_MTU_EXCEEDED :
2402  error1);
2403 
2404  /*
2405  * We've already accounted for an ethernet_header_t elsewhere
2406  */
2407  if (PREDICT_FALSE (rw_len0 > sizeof (ethernet_header_t)))
2409  (&adjacency_counters, cpu_index, adj_index0,
2410  /* packet increment */ 0,
2411  /* byte increment */ rw_len0 - sizeof (ethernet_header_t));
2412 
2413  if (PREDICT_FALSE (rw_len1 > sizeof (ethernet_header_t)))
2415  (&adjacency_counters, cpu_index, adj_index1,
2416  /* packet increment */ 0,
2417  /* byte increment */ rw_len1 - sizeof (ethernet_header_t));
2418 
2419  /* Don't adjust the buffer for ttl issue; icmp-error node wants
2420  * to see the IP headerr */
2421  if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2422  {
2423  next0 = adj0[0].rewrite_header.next_index;
2424  p0->current_data -= rw_len0;
2425  p0->current_length += rw_len0;
2426  tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2427  vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2428 
2430  tx_sw_if_index0, &next0, p0);
2431  }
2432  if (PREDICT_TRUE (error1 == IP4_ERROR_NONE))
2433  {
2434  next1 = adj1[0].rewrite_header.next_index;
2435  p1->current_data -= rw_len1;
2436  p1->current_length += rw_len1;
2437 
2438  tx_sw_if_index1 = adj1[0].rewrite_header.sw_if_index;
2439  vnet_buffer (p1)->sw_if_index[VLIB_TX] = tx_sw_if_index1;
2440 
2442  tx_sw_if_index1, &next1, p1);
2443  }
2444 
2445  /* Guess we are only writing on simple Ethernet header. */
2446  vnet_rewrite_two_headers (adj0[0], adj1[0],
2447  ip0, ip1, sizeof (ethernet_header_t));
2448 
2449  if (is_midchain)
2450  {
2451  adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2452  adj1->sub_type.midchain.fixup_func (vm, adj1, p1);
2453  }
2454 
2455  vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
2456  to_next, n_left_to_next,
2457  pi0, pi1, next0, next1);
2458  }
2459 
2460  while (n_left_from > 0 && n_left_to_next > 0)
2461  {
2462  ip_adjacency_t *adj0;
2463  vlib_buffer_t *p0;
2464  ip4_header_t *ip0;
2465  u32 pi0, rw_len0, adj_index0, next0, error0, checksum0;
2466  u32 tx_sw_if_index0;
2467 
2468  pi0 = to_next[0] = from[0];
2469 
2470  p0 = vlib_get_buffer (vm, pi0);
2471 
2472  adj_index0 = vnet_buffer (p0)->ip.adj_index[VLIB_TX];
2473 
2474  /* We should never rewrite a pkt using the MISS adjacency */
2475  ASSERT (adj_index0);
2476 
2477  adj0 = ip_get_adjacency (lm, adj_index0);
2478 
2479  ip0 = vlib_buffer_get_current (p0);
2480 
2481  error0 = IP4_ERROR_NONE;
2482  next0 = IP4_REWRITE_NEXT_DROP; /* drop on error */
2483 
2484  /* Decrement TTL & update checksum. */
2486  {
2487  i32 ttl0 = ip0->ttl;
2488 
2489  checksum0 = ip0->checksum + clib_host_to_net_u16 (0x0100);
2490 
2491  checksum0 += checksum0 >= 0xffff;
2492 
2493  ip0->checksum = checksum0;
2494 
2495  ASSERT (ip0->ttl > 0);
2496 
2497  ttl0 -= 1;
2498 
2499  ip0->ttl = ttl0;
2500 
2501  ASSERT (ip0->checksum == ip4_header_checksum (ip0));
2502 
2503  if (PREDICT_FALSE (ttl0 <= 0))
2504  {
2505  /*
2506  * If the ttl drops below 1 when forwarding, generate
2507  * an ICMP response.
2508  */
2509  error0 = IP4_ERROR_TIME_EXPIRED;
2511  vnet_buffer (p0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
2512  icmp4_error_set_vnet_buffer (p0, ICMP4_time_exceeded,
2513  ICMP4_time_exceeded_ttl_exceeded_in_transit,
2514  0);
2515  }
2516  }
2517  else
2518  {
2520  }
2521 
2522  /* Guess we are only writing on simple Ethernet header. */
2523  vnet_rewrite_one_header (adj0[0], ip0, sizeof (ethernet_header_t));
2524 
2525  /* Update packet buffer attributes/set output interface. */
2526  rw_len0 = adj0[0].rewrite_header.data_bytes;
2527  vnet_buffer (p0)->ip.save_rewrite_length = rw_len0;
2528 
2529  if (PREDICT_FALSE (rw_len0 > sizeof (ethernet_header_t)))
2531  (&adjacency_counters, cpu_index, adj_index0,
2532  /* packet increment */ 0,
2533  /* byte increment */ rw_len0 - sizeof (ethernet_header_t));
2534 
2535  /* Check MTU of outgoing interface. */
2536  error0 = (vlib_buffer_length_in_chain (vm, p0)
2537  > adj0[0].rewrite_header.max_l3_packet_bytes
2538  ? IP4_ERROR_MTU_EXCEEDED : error0);
2539 
2540  p0->error = error_node->errors[error0];
2541 
2542  /* Don't adjust the buffer for ttl issue; icmp-error node wants
2543  * to see the IP headerr */
2544  if (PREDICT_TRUE (error0 == IP4_ERROR_NONE))
2545  {
2546  p0->current_data -= rw_len0;
2547  p0->current_length += rw_len0;
2548  tx_sw_if_index0 = adj0[0].rewrite_header.sw_if_index;
2549 
2550  vnet_buffer (p0)->sw_if_index[VLIB_TX] = tx_sw_if_index0;
2551  next0 = adj0[0].rewrite_header.next_index;
2552 
2553  if (is_midchain)
2554  {
2555  adj0->sub_type.midchain.fixup_func (vm, adj0, p0);
2556  }
2557 
2559  tx_sw_if_index0, &next0, p0);
2560 
2561  }
2562 
2563  from += 1;
2564  n_left_from -= 1;
2565  to_next += 1;
2566  n_left_to_next -= 1;
2567 
2568  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
2569  to_next, n_left_to_next,
2570  pi0, next0);
2571  }
2572 
2573  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
2574  }
2575 
2576  /* Need to do trace after rewrites to pick up new packet data. */
2577  if (node->flags & VLIB_NODE_FLAG_TRACE)
2578  ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2579 
2580  return frame->n_vectors;
2581 }
2582 
2583 
2584 /** @brief IPv4 rewrite node.
2585  @node ip4-rewrite
2586 
2587  This is the IPv4 transit-rewrite node: decrement TTL, fix the ipv4
2588  header checksum, fetch the ip adjacency, check the outbound mtu,
2589  apply the adjacency rewrite, and send pkts to the adjacency
2590  rewrite header's rewrite_next_index.
2591 
2592  @param vm vlib_main_t corresponding to the current thread
2593  @param node vlib_node_runtime_t
2594  @param frame vlib_frame_t whose contents should be dispatched
2595 
2596  @par Graph mechanics: buffer metadata, next index usage
2597 
2598  @em Uses:
2599  - <code>vnet_buffer(b)->ip.adj_index[VLIB_TX]</code>
2600  - the rewrite adjacency index
2601  - <code>adj->lookup_next_index</code>
2602  - Must be IP_LOOKUP_NEXT_REWRITE or IP_LOOKUP_NEXT_ARP, otherwise
2603  the packet will be dropped.
2604  - <code>adj->rewrite_header</code>
2605  - Rewrite string length, rewrite string, next_index
2606 
2607  @em Sets:
2608  - <code>b->current_data, b->current_length</code>
2609  - Updated net of applying the rewrite string
2610 
2611  <em>Next Indices:</em>
2612  - <code> adj->rewrite_header.next_index </code>
2613  or @c error-drop
2614 */
2615 static uword
2617  vlib_node_runtime_t * node, vlib_frame_t * frame)
2618 {
2619  return ip4_rewrite_inline (vm, node, frame, 0);
2620 }
2621 
2622 static uword
2624  vlib_node_runtime_t * node, vlib_frame_t * frame)
2625 {
2626  return ip4_rewrite_inline (vm, node, frame, 1);
2627 }
2628 
2629 
2631 {
2632  .function = ip4_rewrite,.name = "ip4-rewrite",.vector_size =
2633  sizeof (u32),.format_trace = format_ip4_rewrite_trace,.n_next_nodes =
2634  2,.next_nodes =
2635  {
2636  [IP4_REWRITE_NEXT_DROP] = "error-drop",
2637  [IP4_REWRITE_NEXT_ICMP_ERROR] = "ip4-icmp-error",}
2638 ,};
2639 
2641 
2643 {
2644 .function = ip4_midchain,.name = "ip4-midchain",.vector_size =
2645  sizeof (u32),.format_trace = format_ip4_forward_next_trace,.sibling_of =
2646  "ip4-rewrite",};
2647 
2649 
2650 static clib_error_t *
2652  unformat_input_t * input, vlib_cli_command_t * cmd)
2653 {
2654  vnet_main_t *vnm = vnet_get_main ();
2655  clib_error_t *error = 0;
2656  u32 sw_if_index, table_id;
2657 
2658  sw_if_index = ~0;
2659 
2660  if (!unformat_user (input, unformat_vnet_sw_interface, vnm, &sw_if_index))
2661  {
2662  error = clib_error_return (0, "unknown interface `%U'",
2663  format_unformat_error, input);
2664  goto done;
2665  }
2666 
2667  if (unformat (input, "%d", &table_id))
2668  ;
2669  else
2670  {
2671  error = clib_error_return (0, "expected table id `%U'",
2672  format_unformat_error, input);
2673  goto done;
2674  }
2675 
2676  {
2677  ip4_main_t *im = &ip4_main;
2678  u32 fib_index;
2679 
2681  table_id);
2682 
2683  //
2684  // FIXME-LATER
2685  // changing an interface's table has consequences for any connecteds
2686  // and adj-fibs already installed.
2687  //
2688  vec_validate (im->fib_index_by_sw_if_index, sw_if_index);
2689  im->fib_index_by_sw_if_index[sw_if_index] = fib_index;
2690  }
2691 
2692 done:
2693  return error;
2694 }
2695 
2696 /*?
2697  * Place the indicated interface into the supplied IPv4 FIB table (also known
2698  * as a VRF). If the FIB table does not exist, this command creates it. To
2699  * display the current IPv4 FIB table, use the command '<em>show ip fib</em>'.
2700  * FIB table will only be displayed if a route has been added to the table, or
2701  * an IP Address is assigned to an interface in the table (which adds a route
2702  * automatically).
2703  *
2704  * @note IP addresses added after setting the interface IP table end up in
2705  * the indicated FIB table. If the IP address is added prior to adding the
2706  * interface to the FIB table, it will NOT be part of the FIB table. Predictable
2707  * but potentially counter-intuitive results occur if you provision interface
2708  * addresses in multiple FIBs. Upon RX, packets will be processed in the last
2709  * IP table ID provisioned. It might be marginally useful to evade source RPF
2710  * drops to put an interface address into multiple FIBs.
2711  *
2712  * @cliexpar
2713  * Example of how to add an interface to an IPv4 FIB table (where 2 is the table-id):
2714  * @cliexcmd{set interface ip table GigabitEthernet2/0/0 2}
2715  ?*/
2716 /* *INDENT-OFF* */
2717 VLIB_CLI_COMMAND (set_interface_ip_table_command, static) =
2718 {
2719  .path = "set interface ip table",
2720  .function = add_del_interface_table,
2721  .short_help = "set interface ip table <interface> <table-id>",
2722 };
2723 /* *INDENT-ON* */
2724 
2725 
2726 static uword
2728  vlib_node_runtime_t * node, vlib_frame_t * frame)
2729 {
2730  ip4_main_t *im = &ip4_main;
2732  u32 n_left_from, n_left_to_next, *from, *to_next;
2733  ip_lookup_next_t next;
2734  u32 cpu_index = os_get_cpu_number ();
2735 
2736  from = vlib_frame_vector_args (frame);
2737  n_left_from = frame->n_vectors;
2738  next = node->cached_next_index;
2739 
2740  while (n_left_from > 0)
2741  {
2742  vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
2743 
2744  while (n_left_from >= 4 && n_left_to_next >= 2)
2745  {
2746  vlib_buffer_t *p0, *p1;
2747  u32 pi0, pi1, lb_index0, lb_index1, wrong_next;
2748  ip_lookup_next_t next0, next1;
2749  ip4_header_t *ip0, *ip1;
2750  u32 fib_index0, fib_index1;
2751  const dpo_id_t *dpo0, *dpo1;
2752  const load_balance_t *lb0, *lb1;
2753 
2754  /* Prefetch next iteration. */
2755  {
2756  vlib_buffer_t *p2, *p3;
2757 
2758  p2 = vlib_get_buffer (vm, from[2]);
2759  p3 = vlib_get_buffer (vm, from[3]);
2760 
2761  vlib_prefetch_buffer_header (p2, LOAD);
2762  vlib_prefetch_buffer_header (p3, LOAD);
2763 
2764  CLIB_PREFETCH (p2->data, sizeof (ip0[0]), LOAD);
2765  CLIB_PREFETCH (p3->data, sizeof (ip0[0]), LOAD);
2766  }
2767 
2768  pi0 = to_next[0] = from[0];
2769  pi1 = to_next[1] = from[1];
2770 
2771  p0 = vlib_get_buffer (vm, pi0);
2772  p1 = vlib_get_buffer (vm, pi1);
2773 
2774  ip0 = vlib_buffer_get_current (p0);
2775  ip1 = vlib_buffer_get_current (p1);
2776 
2777  fib_index0 =
2779  vnet_buffer (p0)->sw_if_index[VLIB_RX]);
2780  fib_index1 =
2782  vnet_buffer (p1)->sw_if_index[VLIB_RX]);
2783  fib_index0 =
2784  (vnet_buffer (p0)->sw_if_index[VLIB_TX] ==
2785  (u32) ~ 0) ? fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
2786  fib_index1 =
2787  (vnet_buffer (p1)->sw_if_index[VLIB_TX] ==
2788  (u32) ~ 0) ? fib_index1 : vnet_buffer (p1)->sw_if_index[VLIB_TX];
2789 
2790  lb_index0 = ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0),
2791  &ip0->dst_address);
2792  lb_index1 = ip4_fib_table_lookup_lb (ip4_fib_get (fib_index1),
2793  &ip1->dst_address);
2794 
2795  lb0 = load_balance_get (lb_index0);
2796  lb1 = load_balance_get (lb_index1);
2797 
2798  ASSERT (lb0->lb_n_buckets > 0);
2799  ASSERT (is_pow2 (lb0->lb_n_buckets));
2800  ASSERT (lb1->lb_n_buckets > 0);
2801  ASSERT (is_pow2 (lb1->lb_n_buckets));
2802 
2803  vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
2804  (ip0, lb0->lb_hash_config);
2805 
2806  vnet_buffer (p1)->ip.flow_hash = ip4_compute_flow_hash
2807  (ip1, lb1->lb_hash_config);
2808 
2809  dpo0 = load_balance_get_bucket_i (lb0,
2810  (vnet_buffer (p0)->ip.flow_hash &
2811  (lb0->lb_n_buckets_minus_1)));
2812  dpo1 = load_balance_get_bucket_i (lb1,
2813  (vnet_buffer (p1)->ip.flow_hash &
2814  (lb1->lb_n_buckets_minus_1)));
2815 
2816  next0 = dpo0->dpoi_next_node;
2817  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
2818  next1 = dpo1->dpoi_next_node;
2819  vnet_buffer (p1)->ip.adj_index[VLIB_TX] = dpo1->dpoi_index;
2820 
2821  if (1) /* $$$$$$ HACK FIXME */
2823  (cm, cpu_index, lb_index0, 1,
2824  vlib_buffer_length_in_chain (vm, p0));
2825  if (1) /* $$$$$$ HACK FIXME */
2827  (cm, cpu_index, lb_index1, 1,
2828  vlib_buffer_length_in_chain (vm, p1));
2829 
2830  from += 2;
2831  to_next += 2;
2832  n_left_to_next -= 2;
2833  n_left_from -= 2;
2834 
2835  wrong_next = (next0 != next) + 2 * (next1 != next);
2836  if (PREDICT_FALSE (wrong_next != 0))
2837  {
2838  switch (wrong_next)
2839  {
2840  case 1:
2841  /* A B A */
2842  to_next[-2] = pi1;
2843  to_next -= 1;
2844  n_left_to_next += 1;
2845  vlib_set_next_frame_buffer (vm, node, next0, pi0);
2846  break;
2847 
2848  case 2:
2849  /* A A B */
2850  to_next -= 1;
2851  n_left_to_next += 1;
2852  vlib_set_next_frame_buffer (vm, node, next1, pi1);
2853  break;
2854 
2855  case 3:
2856  /* A B C */
2857  to_next -= 2;
2858  n_left_to_next += 2;
2859  vlib_set_next_frame_buffer (vm, node, next0, pi0);
2860  vlib_set_next_frame_buffer (vm, node, next1, pi1);
2861  if (next0 == next1)
2862  {
2863  /* A B B */
2864  vlib_put_next_frame (vm, node, next, n_left_to_next);
2865  next = next1;
2866  vlib_get_next_frame (vm, node, next, to_next,
2867  n_left_to_next);
2868  }
2869  }
2870  }
2871  }
2872 
2873  while (n_left_from > 0 && n_left_to_next > 0)
2874  {
2875  vlib_buffer_t *p0;
2876  ip4_header_t *ip0;
2877  u32 pi0, lb_index0;
2878  ip_lookup_next_t next0;
2879  u32 fib_index0;
2880  const dpo_id_t *dpo0;
2881  const load_balance_t *lb0;
2882 
2883  pi0 = from[0];
2884  to_next[0] = pi0;
2885 
2886  p0 = vlib_get_buffer (vm, pi0);
2887 
2888  ip0 = vlib_buffer_get_current (p0);
2889 
2890  fib_index0 = vec_elt (im->fib_index_by_sw_if_index,
2891  vnet_buffer (p0)->sw_if_index[VLIB_RX]);
2892  fib_index0 = (vnet_buffer (p0)->sw_if_index[VLIB_TX] == (u32) ~ 0) ?
2893  fib_index0 : vnet_buffer (p0)->sw_if_index[VLIB_TX];
2894 
2895  lb_index0 = ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0),
2896  &ip0->dst_address);
2897 
2898  lb0 = load_balance_get (lb_index0);
2899 
2900  ASSERT (lb0->lb_n_buckets > 0);
2901  ASSERT (is_pow2 (lb0->lb_n_buckets));
2902 
2903  vnet_buffer (p0)->ip.flow_hash = ip4_compute_flow_hash
2904  (ip0, lb0->lb_hash_config);
2905 
2906  dpo0 = load_balance_get_bucket_i (lb0,
2907  (vnet_buffer (p0)->ip.flow_hash &
2908  (lb0->lb_n_buckets_minus_1)));
2909 
2910  next0 = dpo0->dpoi_next_node;
2911  vnet_buffer (p0)->ip.adj_index[VLIB_TX] = dpo0->dpoi_index;
2912 
2913  if (1) /* $$$$$$ HACK FIXME */
2915  (cm, cpu_index, lb_index0, 1,
2916  vlib_buffer_length_in_chain (vm, p0));
2917 
2918  from += 1;
2919  to_next += 1;
2920  n_left_to_next -= 1;
2921  n_left_from -= 1;
2922 
2923  if (PREDICT_FALSE (next0 != next))
2924  {
2925  n_left_to_next += 1;
2926  vlib_put_next_frame (vm, node, next, n_left_to_next);
2927  next = next0;
2928  vlib_get_next_frame (vm, node, next, to_next, n_left_to_next);
2929  to_next[0] = pi0;
2930  to_next += 1;
2931  n_left_to_next -= 1;
2932  }
2933  }
2934 
2935  vlib_put_next_frame (vm, node, next, n_left_to_next);
2936  }
2937 
2938  if (node->flags & VLIB_NODE_FLAG_TRACE)
2939  ip4_forward_next_trace (vm, node, frame, VLIB_TX);
2940 
2941  return frame->n_vectors;
2942 }
2943 
2945 {
2946 .function = ip4_lookup_multicast,.name =
2947  "ip4-lookup-multicast",.vector_size = sizeof (u32),.sibling_of =
2948  "ip4-lookup",.format_trace = format_ip4_lookup_trace,.n_next_nodes = 0,};
2949 
2952 
2954 {
2955  .function = ip4_drop,.name = "ip4-multicast",.vector_size =
2956  sizeof (u32),.format_trace = format_ip4_forward_next_trace,.n_next_nodes =
2957  1,.next_nodes =
2958  {
2959  [0] = "error-drop",}
2960 ,};
2961 
2962 int
2964 {
2965  ip4_fib_mtrie_t *mtrie0;
2966  ip4_fib_mtrie_leaf_t leaf0;
2967  u32 lbi0;
2968 
2969  mtrie0 = &ip4_fib_get (fib_index0)->mtrie;
2970 
2971  leaf0 = IP4_FIB_MTRIE_LEAF_ROOT;
2972  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 0);
2973  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 1);
2974  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 2);
2975  leaf0 = ip4_fib_mtrie_lookup_step (mtrie0, leaf0, a, 3);
2976 
2977  /* Handle default route. */
2978  leaf0 = (leaf0 == IP4_FIB_MTRIE_LEAF_EMPTY ? mtrie0->default_leaf : leaf0);
2979 
2980  lbi0 = ip4_fib_mtrie_leaf_get_adj_index (leaf0);
2981 
2982  return lbi0 == ip4_fib_table_lookup_lb (ip4_fib_get (fib_index0), a);
2983 }
2984 
2985 static clib_error_t *
2987  unformat_input_t * input, vlib_cli_command_t * cmd)
2988 {
2989  ip4_fib_t *fib;
2990  u32 table_id = 0;
2991  f64 count = 1;
2992  u32 n;
2993  int i;
2994  ip4_address_t ip4_base_address;
2995  u64 errors = 0;
2996 
2997  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
2998  {
2999  if (unformat (input, "table %d", &table_id))
3000  {
3001  /* Make sure the entry exists. */
3002  fib = ip4_fib_get (table_id);
3003  if ((fib) && (fib->index != table_id))
3004  return clib_error_return (0, "<fib-index> %d does not exist",
3005  table_id);
3006  }
3007  else if (unformat (input, "count %f", &count))
3008  ;
3009 
3010  else if (unformat (input, "%U",
3011  unformat_ip4_address, &ip4_base_address))
3012  ;
3013  else
3014  return clib_error_return (0, "unknown input `%U'",
3015  format_unformat_error, input);
3016  }
3017 
3018  n = count;
3019 
3020  for (i = 0; i < n; i++)
3021  {
3022  if (!ip4_lookup_validate (&ip4_base_address, table_id))
3023  errors++;
3024 
3025  ip4_base_address.as_u32 =
3026  clib_host_to_net_u32 (1 +
3027  clib_net_to_host_u32 (ip4_base_address.as_u32));
3028  }
3029 
3030  if (errors)
3031  vlib_cli_output (vm, "%llu errors out of %d lookups\n", errors, n);
3032  else
3033  vlib_cli_output (vm, "No errors in %d lookups\n", n);
3034 
3035  return 0;
3036 }
3037 
3038 /*?
3039  * Perform a lookup of an IPv4 Address (or range of addresses) in the
3040  * given FIB table to determine if there is a conflict with the
3041  * adjacency table. The fib-id can be determined by using the
3042  * '<em>show ip fib</em>' command. If fib-id is not entered, default value
3043  * of 0 is used.
3044  *
3045  * @todo This command uses fib-id, other commands use table-id (not
3046  * just a name, they are different indexes). Would like to change this
3047  * to table-id for consistency.
3048  *
3049  * @cliexpar
3050  * Example of how to run the test lookup command:
3051  * @cliexstart{test lookup 172.16.1.1 table 1 count 2}
3052  * No errors in 2 lookups
3053  * @cliexend
3054 ?*/
3055 /* *INDENT-OFF* */
3056 VLIB_CLI_COMMAND (lookup_test_command, static) =
3057 {
3058  .path = "test lookup",
3059  .short_help = "test lookup <ipv4-addr> [table <fib-id>] [count <nn>]",
3060  .function = test_lookup_command_fn,
3061 };
3062 /* *INDENT-ON* */
3063 
3064 int
3065 vnet_set_ip4_flow_hash (u32 table_id, u32 flow_hash_config)
3066 {
3067  ip4_main_t *im4 = &ip4_main;
3068  ip4_fib_t *fib;
3069  uword *p = hash_get (im4->fib_index_by_table_id, table_id);
3070 
3071  if (p == 0)
3072  return VNET_API_ERROR_NO_SUCH_FIB;
3073 
3074  fib = ip4_fib_get (p[0]);
3075 
3076  fib->flow_hash_config = flow_hash_config;
3077  return 0;
3078 }
3079 
3080 static clib_error_t *
3082  unformat_input_t * input,
3083  vlib_cli_command_t * cmd)
3084 {
3085  int matched = 0;
3086  u32 table_id = 0;
3087  u32 flow_hash_config = 0;
3088  int rv;
3089 
3090  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3091  {
3092  if (unformat (input, "table %d", &table_id))
3093  matched = 1;
3094 #define _(a,v) \
3095  else if (unformat (input, #a)) { flow_hash_config |= v; matched=1;}
3097 #undef _
3098  else
3099  break;
3100  }
3101 
3102  if (matched == 0)
3103  return clib_error_return (0, "unknown input `%U'",
3104  format_unformat_error, input);
3105 
3106  rv = vnet_set_ip4_flow_hash (table_id, flow_hash_config);
3107  switch (rv)
3108  {
3109  case 0:
3110  break;
3111 
3112  case VNET_API_ERROR_NO_SUCH_FIB:
3113  return clib_error_return (0, "no such FIB table %d", table_id);
3114 
3115  default:
3116  clib_warning ("BUG: illegal flow hash config 0x%x", flow_hash_config);
3117  break;
3118  }
3119 
3120  return 0;
3121 }
3122 
3123 /*?
3124  * Configure the set of IPv4 fields used by the flow hash.
3125  *
3126  * @cliexpar
3127  * Example of how to set the flow hash on a given table:
3128  * @cliexcmd{set ip flow-hash table 7 dst sport dport proto}
3129  * Example of display the configured flow hash:
3130  * @cliexstart{show ip fib}
3131  * ipv4-VRF:0, fib_index 0, flow hash: src dst sport dport proto
3132  * 0.0.0.0/0
3133  * unicast-ip4-chain
3134  * [@0]: dpo-load-balance: [index:0 buckets:1 uRPF:0 to:[0:0]]
3135  * [0] [@0]: dpo-drop ip6
3136  * 0.0.0.0/32
3137  * unicast-ip4-chain
3138  * [@0]: dpo-load-balance: [index:1 buckets:1 uRPF:1 to:[0:0]]
3139  * [0] [@0]: dpo-drop ip6
3140  * 224.0.0.0/8
3141  * unicast-ip4-chain
3142  * [@0]: dpo-load-balance: [index:3 buckets:1 uRPF:3 to:[0:0]]
3143  * [0] [@0]: dpo-drop ip6
3144  * 6.0.1.2/32
3145  * unicast-ip4-chain
3146  * [@0]: dpo-load-balance: [index:30 buckets:1 uRPF:29 to:[0:0]]
3147  * [0] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3148  * 7.0.0.1/32
3149  * unicast-ip4-chain
3150  * [@0]: dpo-load-balance: [index:31 buckets:4 uRPF:30 to:[0:0]]
3151  * [0] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3152  * [1] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3153  * [2] [@3]: arp-ipv4: via 6.0.0.2 af_packet0
3154  * [3] [@3]: arp-ipv4: via 6.0.0.1 af_packet0
3155  * 240.0.0.0/8
3156  * unicast-ip4-chain
3157  * [@0]: dpo-load-balance: [index:2 buckets:1 uRPF:2 to:[0:0]]
3158  * [0] [@0]: dpo-drop ip6
3159  * 255.255.255.255/32
3160  * unicast-ip4-chain
3161  * [@0]: dpo-load-balance: [index:4 buckets:1 uRPF:4 to:[0:0]]
3162  * [0] [@0]: dpo-drop ip6
3163  * ipv4-VRF:7, fib_index 1, flow hash: dst sport dport proto
3164  * 0.0.0.0/0
3165  * unicast-ip4-chain
3166  * [@0]: dpo-load-balance: [index:12 buckets:1 uRPF:11 to:[0:0]]
3167  * [0] [@0]: dpo-drop ip6
3168  * 0.0.0.0/32
3169  * unicast-ip4-chain
3170  * [@0]: dpo-load-balance: [index:13 buckets:1 uRPF:12 to:[0:0]]
3171  * [0] [@0]: dpo-drop ip6
3172  * 172.16.1.0/24
3173  * unicast-ip4-chain
3174  * [@0]: dpo-load-balance: [index:17 buckets:1 uRPF:16 to:[0:0]]
3175  * [0] [@4]: ipv4-glean: af_packet0
3176  * 172.16.1.1/32
3177  * unicast-ip4-chain
3178  * [@0]: dpo-load-balance: [index:18 buckets:1 uRPF:17 to:[1:84]]
3179  * [0] [@2]: dpo-receive: 172.16.1.1 on af_packet0
3180  * 172.16.1.2/32
3181  * unicast-ip4-chain
3182  * [@0]: dpo-load-balance: [index:21 buckets:1 uRPF:20 to:[0:0]]
3183  * [0] [@5]: ipv4 via 172.16.1.2 af_packet0: IP4: 02:fe:9e:70:7a:2b -> 26:a5:f6:9c:3a:36
3184  * 172.16.2.0/24
3185  * unicast-ip4-chain
3186  * [@0]: dpo-load-balance: [index:19 buckets:1 uRPF:18 to:[0:0]]
3187  * [0] [@4]: ipv4-glean: af_packet1
3188  * 172.16.2.1/32
3189  * unicast-ip4-chain
3190  * [@0]: dpo-load-balance: [index:20 buckets:1 uRPF:19 to:[0:0]]
3191  * [0] [@2]: dpo-receive: 172.16.2.1 on af_packet1
3192  * 224.0.0.0/8
3193  * unicast-ip4-chain
3194  * [@0]: dpo-load-balance: [index:15 buckets:1 uRPF:14 to:[0:0]]
3195  * [0] [@0]: dpo-drop ip6
3196  * 240.0.0.0/8
3197  * unicast-ip4-chain
3198  * [@0]: dpo-load-balance: [index:14 buckets:1 uRPF:13 to:[0:0]]
3199  * [0] [@0]: dpo-drop ip6
3200  * 255.255.255.255/32
3201  * unicast-ip4-chain
3202  * [@0]: dpo-load-balance: [index:16 buckets:1 uRPF:15 to:[0:0]]
3203  * [0] [@0]: dpo-drop ip6
3204  * @cliexend
3205 ?*/
3206 /* *INDENT-OFF* */
3207 VLIB_CLI_COMMAND (set_ip_flow_hash_command, static) =
3208 {
3209  .path = "set ip flow-hash",
3210  .short_help =
3211  "set ip flow-hash table <table-id> [src] [dst] [sport] [dport] [proto] [reverse]",
3212  .function = set_ip_flow_hash_command_fn,
3213 };
3214 /* *INDENT-ON* */
3215 
3216 int
3218  u32 table_index)
3219 {
3220  vnet_main_t *vnm = vnet_get_main ();
3222  ip4_main_t *ipm = &ip4_main;
3223  ip_lookup_main_t *lm = &ipm->lookup_main;
3225  ip4_address_t *if_addr;
3226 
3227  if (pool_is_free_index (im->sw_interfaces, sw_if_index))
3228  return VNET_API_ERROR_NO_MATCHING_INTERFACE;
3229 
3230  if (table_index != ~0 && pool_is_free_index (cm->tables, table_index))
3231  return VNET_API_ERROR_NO_SUCH_ENTRY;
3232 
3234  lm->classify_table_index_by_sw_if_index[sw_if_index] = table_index;
3235 
3236  if_addr = ip4_interface_first_address (ipm, sw_if_index, NULL);
3237 
3238  if (NULL != if_addr)
3239  {
3240  fib_prefix_t pfx = {
3241  .fp_len = 32,
3242  .fp_proto = FIB_PROTOCOL_IP4,
3243  .fp_addr.ip4 = *if_addr,
3244  };
3245  u32 fib_index;
3246 
3248  sw_if_index);
3249 
3250 
3251  if (table_index != (u32) ~ 0)
3252  {
3253  dpo_id_t dpo = DPO_INVALID;
3254 
3255  dpo_set (&dpo,
3256  DPO_CLASSIFY,
3257  DPO_PROTO_IP4,
3258  classify_dpo_create (DPO_PROTO_IP4, table_index));
3259 
3261  &pfx,
3263  FIB_ENTRY_FLAG_NONE, &dpo);
3264  dpo_reset (&dpo);
3265  }
3266  else
3267  {
3268  fib_table_entry_special_remove (fib_index,
3269  &pfx, FIB_SOURCE_CLASSIFY);
3270  }
3271  }
3272 
3273  return 0;
3274 }
3275 
3276 static clib_error_t *
3278  unformat_input_t * input,
3279  vlib_cli_command_t * cmd)
3280 {
3281  u32 table_index = ~0;
3282  int table_index_set = 0;
3283  u32 sw_if_index = ~0;
3284  int rv;
3285 
3286  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
3287  {
3288  if (unformat (input, "table-index %d", &table_index))
3289  table_index_set = 1;
3290  else if (unformat (input, "intfc %U", unformat_vnet_sw_interface,
3291  vnet_get_main (), &sw_if_index))
3292  ;
3293  else
3294  break;
3295  }
3296 
3297  if (table_index_set == 0)
3298  return clib_error_return (0, "classify table-index must be specified");
3299 
3300  if (sw_if_index == ~0)
3301  return clib_error_return (0, "interface / subif must be specified");
3302 
3303  rv = vnet_set_ip4_classify_intfc (vm, sw_if_index, table_index);
3304 
3305  switch (rv)
3306  {
3307  case 0:
3308  break;
3309 
3310  case VNET_API_ERROR_NO_MATCHING_INTERFACE:
3311  return clib_error_return (0, "No such interface");
3312 
3313  case VNET_API_ERROR_NO_SUCH_ENTRY:
3314  return clib_error_return (0, "No such classifier table");
3315  }
3316  return 0;
3317 }
3318 
3319 /*?
3320  * Assign a classification table to an interface. The classification
3321  * table is created using the '<em>classify table</em>' and '<em>classify session</em>'
3322  * commands. Once the table is create, use this command to filter packets
3323  * on an interface.
3324  *
3325  * @cliexpar
3326  * Example of how to assign a classification table to an interface:
3327  * @cliexcmd{set ip classify intfc GigabitEthernet2/0/0 table-index 1}
3328 ?*/
3329 /* *INDENT-OFF* */
3330 VLIB_CLI_COMMAND (set_ip_classify_command, static) =
3331 {
3332  .path = "set ip classify",
3333  .short_help =
3334  "set ip classify intfc <interface> table-index <classify-idx>",
3335  .function = set_ip_classify_command_fn,
3336 };
3337 /* *INDENT-ON* */
3338 
3339 /*
3340  * fd.io coding-style-patch-verification: ON
3341  *
3342  * Local Variables:
3343  * eval: (c-set-style "gnu")
3344  * End:
3345  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:396
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:459
#define foreach_ip_interface_address(lm, a, sw_if_index, loop, body)
Definition: lookup.h:460
fib_node_index_t fib_table_entry_update_one_path(u32 fib_index, const fib_prefix_t *prefix, fib_source_t source, fib_entry_flag_t flags, fib_protocol_t next_hop_proto, const ip46_address_t *next_hop, u32 next_hop_sw_if_index, u32 next_hop_fib_index, u32 next_hop_weight, mpls_label_t *next_hop_labels, fib_route_path_flags_t path_flags)
Update the entry to have just one path.
Definition: fib_table.c:726
#define vnet_rewrite_one_header(rw0, p0, most_likely_size)
Definition: rewrite.h:253
u16 lb_n_buckets
number of buckets in the load-balance.
Definition: load_balance.h:87
ip_lookup_next_t
Common (IP4/IP6) next index stored in adjacency.
Definition: lookup.h:60
vmrglw vmrglh hi
vlib_combined_counter_main_t lbm_to_counters
Definition: load_balance.h:45
clib_error_t * ip4_add_del_interface_address(vlib_main_t *vm, u32 sw_if_index, ip4_address_t *address, u32 address_length, u32 is_del)
Definition: ip4_forward.c:921
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
ip4_address_t * ip4_interface_first_address(ip4_main_t *im, u32 sw_if_index, ip_interface_address_t **result_ia)
Definition: ip4_forward.c:710
uword vlib_error_drop_buffers(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u32 next_buffer_stride, u32 n_buffers, u32 next_index, u32 drop_error_node, u32 drop_error_code)
Definition: error.c:44
#define CLIB_UNUSED(x)
Definition: clib.h:79
static vlib_node_registration_t ip4_drop_node
(constructor) VLIB_REGISTER_NODE (ip4_drop_node)
Definition: ip4_forward.c:1344
uword unformat(unformat_input_t *i, char *fmt,...)
Definition: unformat.c:966
int vnet_set_ip4_flow_hash(u32 table_id, u32 flow_hash_config)
Definition: ip4_forward.c:3065
static int fib_urpf_check_size(index_t ui)
Data-Plane function to check the size of an uRPF list, (i.e.
static uword ip4_drop(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: ip4_forward.c:1333
ip4_add_del_interface_address_callback_t * add_del_interface_address_callbacks
Functions to call when interface address changes.
Definition: ip4.h:116
a
Definition: bitmap.h:516
ip4_address_t src_address
Definition: ip4_packet.h:163
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static void vlib_set_next_frame_buffer(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 buffer_index)
Definition: node_funcs.h:383
ip_interface_address_t * if_address_pool
Pool of addresses that are assigned to interfaces.
Definition: lookup.h:375
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
static clib_error_t * ip4_add_del_interface_address_internal(vlib_main_t *vm, u32 sw_if_index, ip4_address_t *address, u32 address_length, u32 is_del)
Definition: ip4_forward.c:836
vnet_interface_main_t interface_main
Definition: vnet.h:57
static uword ip4_midchain(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: ip4_forward.c:2623
vlib_node_registration_t ip4_lookup_node
(constructor) VLIB_REGISTER_NODE (ip4_lookup_node)
Definition: ip4_forward.c:511
#define PREDICT_TRUE(x)
Definition: clib.h:98
unformat_function_t unformat_pg_ip4_header
Definition: format.h:91
#define UNFORMAT_END_OF_INPUT
Definition: format.h:143
static int ip4_header_bytes(ip4_header_t *i)
Definition: ip4_packet.h:226
#define NULL
Definition: clib.h:55
static u32 ip4_compute_flow_hash(const ip4_header_t *ip, flow_hash_config_t flow_hash_config)
Definition: ip4.h:272
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:182
IP unicast adjacency.
Definition: lookup.h:188
u32 fib_table_get_index_for_sw_if_index(fib_protocol_t proto, u32 sw_if_index)
Get the index of the FIB bound to the interface.
Definition: fib_table.c:902
static void * clib_random_buffer_get_data(clib_random_buffer_t *b, uword n_bytes)
Definition: random_buffer.h:78
flow_hash_config_t lb_hash_config
the hash config to use when selecting a bucket.
Definition: load_balance.h:122
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:482
static clib_error_t * test_lookup_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: ip4_forward.c:2986
#define vlib_validate_buffer_enqueue_x4(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, bi2, bi3, next0, next1, next2, next3)
Finish enqueueing four buffers forward in the graph.
Definition: buffer_node.h:138
ip4_rewrite_next_t
Definition: ip4_forward.c:2237
ip_lookup_main_t lookup_main
Definition: ip4.h:97
uword ip_csum_t
Definition: ip_packet.h:90
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
static ip_csum_t ip_csum_with_carry(ip_csum_t sum, ip_csum_t x)
Definition: ip_packet.h:93
u32 * fib_index_by_sw_if_index
Table index indexed by software interface.
Definition: ip4.h:105
unformat_function_t unformat_vnet_sw_interface
static uword ip4_arp(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: ip4_forward.c:2104
vlib_error_t * errors
Definition: node.h:419
Definition: fib_entry.h:218
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:100
VNET_FEATURE_ARC_INIT(ip4_unicast, static)
format_function_t format_ip4_address
Definition: format.h:79
u32 neighbor_probe_adj_index
Definition: lookup.h:341
u8 mcast_feature_arc_index
Feature arc indices.
Definition: lookup.h:388
format_function_t format_vnet_sw_if_index_name
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1063
union ip_adjacency_t_::@175 sub_type
#define IP4_FIB_MTRIE_LEAF_EMPTY
Definition: ip4_mtrie.h:54
u32 index
Definition: ip4.h:60
static pg_node_t * pg_get_node(uword node_index)
Definition: pg.h:350
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_physmem_alloc, char *fmt,...)
static ip4_fib_mtrie_leaf_t ip4_fib_mtrie_lookup_step(ip4_fib_mtrie_t *m, ip4_fib_mtrie_leaf_t current_leaf, const ip4_address_t *dst_address, u32 dst_address_byte_index)
Definition: ip4_mtrie.h:164
u8 output_feature_arc_index
Definition: lookup.h:390
vlib_rx_or_tx_t
Definition: defs.h:44
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
#define VNET_BUFFER_LOCALLY_ORIGINATED
Definition: buffer.h:71
static clib_error_t * show_ip_local_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: ip4_forward.c:1868
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:78
format_function_t format_ip_adjacency_packet_data
Definition: format.h:59
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
Definition: buffer_funcs.h:170
ethernet_arp_ip4_over_ethernet_address_t ip4_over_ethernet[2]
Definition: arp_packet.h:136
ip_csum_t ip_incremental_checksum(ip_csum_t sum, void *_data, uword n_bytes)
Definition: ip_checksum.c:43
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:111
void fib_table_entry_special_remove(u32 fib_index, const fib_prefix_t *prefix, fib_source_t source)
Remove a &#39;special&#39; entry from the FIB.
Definition: fib_table.c:399
#define always_inline
Definition: clib.h:84
static uword pow2_mask(uword x)
Definition: clib.h:251
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:194
static void ip4_add_interface_routes(u32 sw_if_index, ip4_main_t *im, u32 fib_index, ip_interface_address_t *a)
Definition: ip4_forward.c:733
u16 lb_n_buckets_minus_1
number of buckets in the load-balance - 1.
Definition: load_balance.h:92
ip4_address_t dst_address
Definition: ip4_packet.h:163
#define IP_BUFFER_L4_CHECKSUM_CORRECT
Definition: buffer.h:50
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
ip4_error_t
Definition: ip4_error.h:79
static uword ip4_rewrite_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_midchain)
Definition: ip4_forward.c:2244
int i32
Definition: types.h:81
u32 ip4_fib_mtrie_leaf_t
Definition: ip4_mtrie.h:52
Aggregrate type for a prefix.
Definition: fib_types.h:145
#define IP4_LOOKUP_NEXT_NODES
Definition: lookup.h:111
#define clib_warning(format, args...)
Definition: error.h:59
vlib_node_registration_t ip4_arp_node
(constructor) VLIB_REGISTER_NODE (ip4_arp_node)
Definition: ip4_forward.c:2124
unsigned long u64
Definition: types.h:89
void ip4_sw_interface_enable_disable(u32 sw_if_index, u32 is_enable)
Definition: ip4_forward.c:807
static void * ip4_next_header(ip4_header_t *i)
Definition: ip4_packet.h:232
uword unformat_user(unformat_input_t *input, unformat_function_t *func,...)
Definition: unformat.c:977
u16 fp_len
The mask length.
Definition: fib_types.h:149
#define vlib_call_init_function(vm, x)
Definition: init.h:161
struct ip_adjacency_t_::@175::@178 glean
IP_LOOKUP_NEXT_GLEAN.
This packet matches an "interface route" and packets need to be passed to ARP to find rewrite string ...
Definition: lookup.h:78
#define foreach_notrace_ip4_arp_error
Definition: ip4_forward.c:2144
static clib_error_t * vnet_feature_init(vlib_main_t *vm)
Definition: feature.c:21
adj_index_t fib_entry_get_adj(fib_node_index_t fib_entry_index)
Definition: fib_entry.c:447
static u32 ip4_fib_mtrie_leaf_get_adj_index(ip4_fib_mtrie_leaf_t n)
Definition: ip4_mtrie.h:76
#define VLIB_BUFFER_NEXT_PRESENT
Definition: buffer.h:97
Definition: fib_entry.h:216
vlib_node_registration_t ip4_rewrite_node
(constructor) VLIB_REGISTER_NODE (ip4_rewrite_node)
Definition: ip4_forward.c:2630
static int ip4_is_fragment(ip4_header_t *i)
Definition: ip4_packet.h:204
unformat_function_t unformat_ip4_address
Definition: format.h:76
vnet_api_error_t api_errno
Definition: vnet.h:77
The identity of a DPO is a combination of its type and its instance number/index of objects of that t...
Definition: dpo.h:138
Definition: fib_entry.h:221
#define hash_get(h, key)
Definition: hash.h:248
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:369
static uword format_get_indent(u8 *s)
Definition: format.h:72
uword * fib_index_by_table_id
Hash table mapping table id to fib index.
Definition: ip4.h:112
vlib_node_registration_t ip4_input_node
Global ip4 input node.
Definition: ip4_input.c:416
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:82
u32 * classify_table_index_by_sw_if_index
First table index to use for this interface, ~0 => none.
Definition: lookup.h:385
ip46_address_t fp_addr
The address type is not deriveable from the fp_addr member.
Definition: fib_types.h:168
static vlib_node_registration_t ip4_local_node
(constructor) VLIB_REGISTER_NODE (ip4_local_node)
Definition: ip4_forward.c:1841
index_t classify_dpo_create(dpo_proto_t proto, u32 classify_table_index)
Definition: classify_dpo.c:43
clib_error_t * arp_notrace_init(vlib_main_t *vm)
Definition: ip4_forward.c:2151
dpo_type_t dpoi_type
the type
Definition: dpo.h:142
static const dpo_id_t * load_balance_get_bucket_i(const load_balance_t *lb, u32 bucket)
Definition: load_balance.h:194
void ip4_register_protocol(u32 protocol, u32 node_index)
Definition: ip4_forward.c:1856
static uword ip4_arp_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int is_glean)
Definition: ip4_forward.c:1908
uword os_get_cpu_number(void)
Definition: unix-misc.c:224
static uword ip4_punt(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: ip4_forward.c:1339
void ip4_forward_next_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vlib_rx_or_tx_t which_adj_index)
Definition: ip4_forward.c:1226
u8 * ip_enabled_by_sw_if_index
Definition: ip4.h:108
static uword ip4_glean(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: ip4_forward.c:2110
The FIB DPO provieds;.
Definition: load_balance.h:83
ip4_main_t ip4_main
Global ip4 main structure.
Definition: ip4_forward.c:1099
#define PREDICT_FALSE(x)
Definition: clib.h:97
u8 local_next_by_ip_protocol[256]
Table mapping ip protocol to ip[46]-local node next index.
Definition: lookup.h:408
static uword ip4_load_balance(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: ip4_forward.c:520
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:196
load_balance_main_t load_balance_main
The one instance of load-balance main.
Definition: load_balance.c:55
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:216
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:350
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:576
vlib_combined_counter_main_t adjacency_counters
Adjacency packet counters.
Definition: adj.c:30
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:121
static vlib_node_registration_t ip4_punt_node
(constructor) VLIB_REGISTER_NODE (ip4_punt_node)
Definition: ip4_forward.c:1355
static u8 * format_ip4_lookup_trace(u8 *s, va_list *args)
Definition: ip4_forward.c:1190
clib_error_t * ip4_probe_neighbor(vlib_main_t *vm, ip4_address_t *dst, u32 sw_if_index)
Definition: ip4_forward.c:2170
static u8 * format_ip4_rewrite_trace(u8 *s, va_list *args)
Definition: ip4_forward.c:1206
static uword ip4_policer_classify(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: node_funcs.c:841
static void vlib_buffer_copy_trace_flag(vlib_main_t *vm, vlib_buffer_t *b, u32 bi_target)
Definition: trace_funcs.h:134
u16 n_vectors
Definition: node.h:344
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:300
void fib_table_entry_delete(u32 fib_index, const fib_prefix_t *prefix, fib_source_t source)
Delete a FIB entry.
Definition: fib_table.c:808
void icmp4_error_set_vnet_buffer(vlib_buffer_t *b, u8 type, u8 code, u32 data)
Definition: icmp4.c:431
ip4_add_del_interface_address_function_t * function
Definition: ip4.h:82
static ip4_fib_t * ip4_fib_get(u32 index)
Get the FIB at the given index.
Definition: ip4_fib.h:71
static clib_error_t * add_del_interface_table(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: ip4_forward.c:2651
Definition: ip4.h:48
struct ip_adjacency_t_::@175::@176 nbr
IP_LOOKUP_NEXT_ARP/IP_LOOKUP_NEXT_REWRITE.
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:88
#define clib_memcpy(a, b, c)
Definition: string.h:69
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:207
int ip4_lookup_validate(ip4_address_t *a, u32 fib_index0)
Definition: ip4_forward.c:2963
static clib_error_t * set_ip_flow_hash_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: ip4_forward.c:3081
unformat_function_t * unformat_edit
Definition: pg.h:307
static vlib_node_registration_t ip4_multicast_node
(constructor) VLIB_REGISTER_NODE (ip4_multicast_node)
Definition: ip4_forward.c:2953
u8 packet_data[64-1 *sizeof(u32)]
Definition: ip4_forward.c:1172
u32 fib_node_index_t
A typedef of a node index.
Definition: fib_types.h:28
void ip_lookup_init(ip_lookup_main_t *lm, u32 is_ip6)
Definition: lookup.c:172
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
Definition: pool.h:211
#define ARRAY_LEN(x)
Definition: clib.h:59
This packet matches an "incomplete adjacency" and packets need to be passed to ARP to find rewrite st...
Definition: lookup.h:73
void dpo_set(dpo_id_t *dpo, dpo_type_t type, dpo_proto_t proto, index_t index)
Set/create a DPO ID The DPO will be locked.
Definition: dpo.c:154
vlib_combined_counter_main_t lbm_via_counters
Definition: load_balance.h:46
static void ip4_addr_fib_init(ip4_address_fib_t *addr_fib, ip4_address_t *address, u32 fib_index)
Definition: ip4_packet.h:66
static clib_error_t * set_ip_classify_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: ip4_forward.c:3277
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
static clib_error_t * ip4_sw_interface_add_del(vnet_main_t *vnm, u32 sw_if_index, u32 is_add)
Definition: ip4_forward.c:1080
struct _vnet_classify_main vnet_classify_main_t
Definition: vnet_classify.h:69
#define foreach_flow_hash_bit
Definition: lookup.h:149
fib_node_index_t fib_table_entry_special_dpo_add(u32 fib_index, const fib_prefix_t *prefix, fib_source_t source, fib_entry_flag_t flags, const dpo_id_t *dpo)
Add a &#39;special&#39; entry to the FIB that links to the DPO passed A special entry is an entry that the FI...
Definition: fib_table.c:288
vlib_packet_template_t ip4_arp_request_packet_template
Template used to generate IP4 ARP packets.
Definition: ip4.h:119
u16 cached_next_index
Definition: node.h:463
vlib_node_registration_t ip4_load_balance_node
(constructor) VLIB_REGISTER_NODE (ip4_load_balance_node)
Definition: ip4_forward.c:700
#define VNET_SW_INTERFACE_FLAG_ADMIN_UP
Definition: interface.h:528
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 cpu_index, u32 index, u32 packet_increment, u32 byte_increment)
Increment a combined counter.
Definition: counter.h:241
#define ASSERT(truth)
#define IP4_FIB_MTRIE_LEAF_ROOT
Definition: ip4_mtrie.h:55
index_t lb_urpf
This is the index of the uRPF list for this LB.
Definition: load_balance.h:117
format_function_t format_ip4_header
Definition: format.h:86
unsigned int u32
Definition: types.h:88
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
#define vnet_buffer(b)
Definition: buffer.h:361
u32 ip4_tcp_udp_validate_checksum(vlib_main_t *vm, vlib_buffer_t *p0)
Definition: ip4_forward.c:1423
#define LOG2_IP_BUFFER_L4_CHECKSUM_CORRECT
Definition: buffer.h:48
static load_balance_t * load_balance_get(index_t lbi)
Definition: load_balance.h:185
void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
#define hash_v3_finalize32(a, b, c)
Definition: hash.h:540
IPv4 main type.
Definition: ip4.h:95
Classify.
Definition: fib_entry.h:44
vlib_node_registration_t ip4_glean_node
(constructor) VLIB_REGISTER_NODE (ip4_glean_node)
Definition: ip4_forward.c:2134
VLIB_NODE_FUNCTION_MULTIARCH(ip4_lookup_node, ip4_lookup)
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:117
u32 fib_table_find_or_create_and_lock(fib_protocol_t proto, u32 table_id)
Get the index of the FIB for a Table-ID.
Definition: fib_table.c:964
vlib_node_registration_t ip4_midchain_node
(constructor) VLIB_REGISTER_NODE (ip4_midchain_node)
Definition: ip4_forward.c:2642
static uword ip4_inacl(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: ip_input_acl.c:386
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:259
vnet_classify_main_t vnet_classify_main
Definition: vnet_classify.c:22
Route added as a result of interface configuration.
Definition: fib_entry.h:50
clib_error_t * ip4_lookup_init(vlib_main_t *vm)
Definition: ip4_forward.c:1102
u32 flow_hash_config_t
A flow hash configuration is a mask of the flow hash options.
Definition: lookup.h:160
#define VLIB_BUFFER_IS_TRACED
Definition: buffer.h:95
#define VNET_FEATURES(...)
Definition: feature.h:363
static uword is_pow2(uword x)
Definition: clib.h:266
static uword ip4_local(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: ip4_forward.c:1449
u64 uword
Definition: types.h:112
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:55
#define vec_elt(v, i)
Get vector value at index i.
#define IP_BUFFER_L4_CHECKSUM_COMPUTED
Definition: buffer.h:49
u8 ucast_feature_arc_index
Definition: lookup.h:389
u16 ip4_tcp_udp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip4_header_t *ip0)
Definition: ip4_forward.c:1368
Definition: defs.h:47
unsigned short u16
Definition: types.h:57
index_t dpoi_index
the index of objects of that type
Definition: dpo.h:154
VNET_FEATURE_INIT(ip4_flow_classify, static)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
unsigned char u8
Definition: types.h:56
ip_lookup_next_t lookup_next_index
Definition: lookup.h:199
Definition: fib_entry.h:217
vnet_sw_interface_t * sw_interfaces
Definition: interface.h:606
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:253
ip4_fib_mtrie_t mtrie
Definition: ip4.h:54
u8 * format_ip4_forward_next_trace(u8 *s, va_list *args)
Definition: ip4_forward.c:1177
#define DPO_INVALID
An initialiser for DPOs declared on the stack.
Definition: dpo.h:165
clib_error_t * ip_interface_address_add_del(ip_lookup_main_t *lm, u32 sw_if_index, void *addr_fib, u32 address_length, u32 is_del, u32 *result_if_address_index)
Definition: lookup.c:59
A collection of combined counters.
Definition: counter.h:212
static uword ip4_flow_classify(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
flow_hash_config_t flow_hash_config
Definition: ip4.h:63
#define clib_mem_unaligned(pointer, type)
Definition: types.h:155
static void ip4_del_interface_routes(ip4_main_t *im, u32 fib_index, ip4_address_t *address, u32 address_length)
Definition: ip4_forward.c:787
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
static uword ip4_lookup_multicast(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: ip4_forward.c:2727
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:170
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:169
static uword ip4_source_and_port_range_check_tx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:143
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:418
int vnet_set_ip4_classify_intfc(vlib_main_t *vm, u32 sw_if_index, u32 table_index)
Definition: ip4_forward.c:3217
u8 data[0]
Packet data.
Definition: buffer.h:158
This packet is to be rewritten and forwarded to the next processing node.
Definition: lookup.h:83
void dpo_reset(dpo_id_t *dpo)
reset a DPO ID The DPO will be unlocked.
Definition: dpo.c:191
#define vec_foreach(var, vec)
Vector iterator.
ip_local_next_t
Definition: lookup.h:355
index_t ip4_fib_table_lookup_lb(ip4_fib_t *fib, const ip4_address_t *addr)
Definition: ip4_fib.c:261
static uword ip4_drop_or_punt(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, ip4_error_t error_code)
Definition: ip4_forward.c:1313
u16 dpoi_next_node
The next VLIB node to follow.
Definition: dpo.h:150
#define clib_error_return(e, args...)
Definition: error.h:111
VNET_SW_INTERFACE_ADD_DEL_FUNCTION(ip4_sw_interface_add_del)
struct _unformat_input_t unformat_input_t
static void * ip_interface_address_get_address(ip_lookup_main_t *lm, ip_interface_address_t *a)
Definition: lookup.h:453
static uword ip4_lookup(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
IPv4 lookup node.
Definition: ip4_forward.c:500
static uword ip4_rewrite(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
IPv4 rewrite node.
Definition: ip4_forward.c:2616
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:187
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:445
#define vnet_rewrite_two_headers(rw0, rw1, p0, p1, most_likely_size)
Definition: rewrite.h:258
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:85
static int ip4_src_address_for_packet(ip_lookup_main_t *lm, u32 sw_if_index, ip4_address_t *src)
Definition: ip4.h:179
#define BITS(x)
Definition: clib.h:58
static uword ip4_vxlan_bypass(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: decap.c:996
static vlib_node_registration_t ip4_lookup_multicast_node
(constructor) VLIB_REGISTER_NODE (ip4_lookup_multicast_node)
Definition: ip4_forward.c:2944
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
static uword ip4_source_and_port_range_check_rx(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
static u16 ip4_header_checksum(ip4_header_t *i)
Definition: ip4_packet.h:238
clib_random_buffer_t random_buffer
Definition: main.h:153
Definition: pg.h:304
static u16 ip_csum_fold(ip_csum_t c)
Definition: ip_packet.h:145
static char * ip4_arp_error_strings[]
Definition: ip4_forward.c:2115
Definition: defs.h:46
static uword ip4_lookup_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int lookup_for_responses_to_locally_received_packets)
Definition: ip4_forward.c:68
int vnet_feature_enable_disable(const char *arc_name, const char *node_name, u32 sw_if_index, int enable_disable, void *feature_config, u32 n_feature_config_bytes)
Definition: feature.c:238
static ip_adjacency_t * ip_get_adjacency(ip_lookup_main_t *lm, u32 adj_index)
Definition: lookup.h:415
format_function_t format_ip_adjacency
Definition: format.h:58
struct ip_adjacency_t_::@175::@177 midchain
IP_LOOKUP_NEXT_MIDCHAIN.
u32 fib_masks[33]
Definition: ip4.h:102
ip4_fib_mtrie_leaf_t default_leaf
Definition: ip4_mtrie.h:145
static_always_inline void vnet_feature_arc_start(u8 arc, u32 sw_if_index, u32 *next0, vlib_buffer_t *b0)
Definition: feature.h:203
static ip4_address_t * ip4_interface_address_matching_destination(ip4_main_t *im, ip4_address_t *dst, u32 sw_if_index, ip_interface_address_t **result_ia)
Definition: ip4.h:201
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:109