FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
node.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * ethernet_node.c: ethernet packet processing
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <vlib/vlib.h>
41 #include <vnet/pg/pg.h>
42 #include <vnet/ethernet/ethernet.h>
44 #include <vnet/devices/pipe/pipe.h>
45 #include <vppinfra/sparse_vec.h>
46 #include <vnet/l2/l2_bvi.h>
47 
48 #define foreach_ethernet_input_next \
49  _ (PUNT, "error-punt") \
50  _ (DROP, "error-drop") \
51  _ (LLC, "llc-input") \
52  _ (IP4_INPUT, "ip4-input") \
53  _ (IP4_INPUT_NCS, "ip4-input-no-checksum")
54 
55 typedef enum
56 {
57 #define _(s,n) ETHERNET_INPUT_NEXT_##s,
59 #undef _
62 
63 typedef struct
64 {
65  u8 packet_data[32];
69 
70 static u8 *
71 format_ethernet_input_trace (u8 * s, va_list * va)
72 {
73  CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
74  CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
76  u32 indent = format_get_indent (s);
77 
78  if (t->frame_flags)
79  {
80  s = format (s, "frame: flags 0x%x", t->frame_flags);
82  s = format (s, ", hw-if-index %u, sw-if-index %u",
84  s = format (s, "\n%U", format_white_space, indent);
85  }
86  s = format (s, "%U", format_ethernet_header, t->packet_data);
87 
88  return s;
89 }
90 
92 
93 typedef enum
94 {
99 
100 
101 // Parse the ethernet header to extract vlan tags and innermost ethertype
104  vlib_buffer_t * b0,
105  u16 * type,
106  u16 * orig_type,
107  u16 * outer_id, u16 * inner_id, u32 * match_flags)
108 {
109  u8 vlan_count;
110 
111  if (variant == ETHERNET_INPUT_VARIANT_ETHERNET
112  || variant == ETHERNET_INPUT_VARIANT_NOT_L2)
113  {
114  ethernet_header_t *e0;
115 
116  e0 = (void *) (b0->data + b0->current_data);
117 
118  vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
119  b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
120 
121  vlib_buffer_advance (b0, sizeof (e0[0]));
122 
123  *type = clib_net_to_host_u16 (e0->type);
124  }
125  else if (variant == ETHERNET_INPUT_VARIANT_ETHERNET_TYPE)
126  {
127  // here when prior node was LLC/SNAP processing
128  u16 *e0;
129 
130  e0 = (void *) (b0->data + b0->current_data);
131 
132  vlib_buffer_advance (b0, sizeof (e0[0]));
133 
134  *type = clib_net_to_host_u16 (e0[0]);
135  }
136 
137  // save for distinguishing between dot1q and dot1ad later
138  *orig_type = *type;
139 
140  // default the tags to 0 (used if there is no corresponding tag)
141  *outer_id = 0;
142  *inner_id = 0;
143 
145  vlan_count = 0;
146 
147  // check for vlan encaps
148  if (ethernet_frame_is_tagged (*type))
149  {
151  u16 tag;
152 
154 
155  h0 = (void *) (b0->data + b0->current_data);
156 
157  tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
158 
159  *outer_id = tag & 0xfff;
160  if (0 == *outer_id)
161  *match_flags &= ~SUBINT_CONFIG_MATCH_1_TAG;
162 
163  *type = clib_net_to_host_u16 (h0->type);
164 
165  vlib_buffer_advance (b0, sizeof (h0[0]));
166  vlan_count = 1;
167 
168  if (*type == ETHERNET_TYPE_VLAN)
169  {
170  // Double tagged packet
172 
173  h0 = (void *) (b0->data + b0->current_data);
174 
175  tag = clib_net_to_host_u16 (h0->priority_cfi_and_id);
176 
177  *inner_id = tag & 0xfff;
178 
179  *type = clib_net_to_host_u16 (h0->type);
180 
181  vlib_buffer_advance (b0, sizeof (h0[0]));
182  vlan_count = 2;
183  if (*type == ETHERNET_TYPE_VLAN)
184  {
185  // More than double tagged packet
187 
188  vlib_buffer_advance (b0, sizeof (h0[0]));
189  vlan_count = 3; // "unknown" number, aka, 3-or-more
190  }
191  }
192  }
193  ethernet_buffer_set_vlan_count (b0, vlan_count);
194 }
195 
196 // Determine the subinterface for this packet, given the result of the
197 // vlan table lookups and vlan header parsing. Check the most specific
198 // matches first.
201  vlib_buffer_t * b0,
202  u32 match_flags,
203  main_intf_t * main_intf,
204  vlan_intf_t * vlan_intf,
205  qinq_intf_t * qinq_intf,
206  u32 * new_sw_if_index, u8 * error0, u32 * is_l2)
207 {
208  u32 matched;
209 
210  matched = eth_identify_subint (hi, match_flags, main_intf, vlan_intf,
211  qinq_intf, new_sw_if_index, error0, is_l2);
212 
213  if (matched)
214  {
215 
216  // Perform L3 my-mac filter
217  // A unicast packet arriving on an L3 interface must have a dmac matching the interface mac.
218  // This is required for promiscuous mode, else we will forward packets we aren't supposed to.
219  if (!(*is_l2))
220  {
221  ethernet_header_t *e0;
222  e0 = (void *) (b0->data + vnet_buffer (b0)->l2_hdr_offset);
223 
224  if (!(ethernet_address_cast (e0->dst_address)))
225  {
226  if (!eth_mac_equal ((u8 *) e0, hi->hw_address))
227  {
228  *error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
229  }
230  }
231  }
232 
233  // Check for down subinterface
234  *error0 = (*new_sw_if_index) != ~0 ? (*error0) : ETHERNET_ERROR_DOWN;
235  }
236 }
237 
240  ethernet_input_variant_t variant,
241  u32 is_l20,
242  u32 type0, vlib_buffer_t * b0, u8 * error0, u8 * next0)
243 {
244  vnet_buffer (b0)->l3_hdr_offset = b0->current_data;
245  b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
246 
247  if (PREDICT_FALSE (*error0 != ETHERNET_ERROR_NONE))
248  {
249  // some error occurred
250  *next0 = ETHERNET_INPUT_NEXT_DROP;
251  }
252  else if (is_l20)
253  {
254  // record the L2 len and reset the buffer so the L2 header is preserved
255  u32 eth_start = vnet_buffer (b0)->l2_hdr_offset;
256  vnet_buffer (b0)->l2.l2_len = b0->current_data - eth_start;
257  *next0 = em->l2_next;
258  ASSERT (vnet_buffer (b0)->l2.l2_len ==
260  vlib_buffer_advance (b0, -(vnet_buffer (b0)->l2.l2_len));
261 
262  // check for common IP/MPLS ethertypes
263  }
264  else if (type0 == ETHERNET_TYPE_IP4)
265  {
266  *next0 = em->l3_next.input_next_ip4;
267  }
268  else if (type0 == ETHERNET_TYPE_IP6)
269  {
270  *next0 = em->l3_next.input_next_ip6;
271  }
272  else if (type0 == ETHERNET_TYPE_MPLS)
273  {
274  *next0 = em->l3_next.input_next_mpls;
275 
276  }
277  else if (em->redirect_l3)
278  {
279  // L3 Redirect is on, the cached common next nodes will be
280  // pointing to the redirect node, catch the uncommon types here
281  *next0 = em->redirect_l3_next;
282  }
283  else
284  {
285  // uncommon ethertype, check table
286  u32 i0;
287  i0 = sparse_vec_index (em->l3_next.input_next_by_type, type0);
288  *next0 = vec_elt (em->l3_next.input_next_by_type, i0);
289  *error0 =
290  i0 ==
291  SPARSE_VEC_INVALID_INDEX ? ETHERNET_ERROR_UNKNOWN_TYPE : *error0;
292 
293  // The table is not populated with LLC values, so check that now.
294  // If variant is variant_ethernet then we came from LLC processing. Don't
295  // go back there; drop instead using by keeping the drop/bad table result.
296  if ((type0 < 0x600) && (variant == ETHERNET_INPUT_VARIANT_ETHERNET))
297  {
298  *next0 = ETHERNET_INPUT_NEXT_LLC;
299  }
300  }
301 }
302 
303 
304 /* following vector code relies on following assumptions */
305 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_data, 0);
306 STATIC_ASSERT_OFFSET_OF (vlib_buffer_t, current_length, 2);
309  STRUCT_OFFSET_OF (vnet_buffer_opaque_t, l3_hdr_offset) - 2,
310  "l3_hdr_offset must follow l2_hdr_offset");
311 
314 {
315  i16 adv = sizeof (ethernet_header_t);
316  u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
317  VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
318 
319 #ifdef CLIB_HAVE_VEC256
320  /* to reduce number of small loads/stores we are loading first 64 bits
321  of each buffer metadata into 256-bit register so we can advance
322  current_data, current_length and flags.
323  Observed saving of this code is ~2 clocks per packet */
324  u64x4 r, radv;
325 
326  /* vector if signed 16 bit integers used in signed vector add operation
327  to advnce current_data and current_length */
328  u32x8 flags4 = { 0, flags, 0, flags, 0, flags, 0, flags };
329  i16x16 adv4 = {
330  adv, -adv, 0, 0, adv, -adv, 0, 0,
331  adv, -adv, 0, 0, adv, -adv, 0, 0
332  };
333 
334  /* load 4 x 64 bits */
335  r = u64x4_gather (b[0], b[1], b[2], b[3]);
336 
337  /* set flags */
338  r |= (u64x4) flags4;
339 
340  /* advance buffer */
341  radv = (u64x4) ((i16x16) r + adv4);
342 
343  /* write 4 x 64 bits */
344  u64x4_scatter (is_l3 ? radv : r, b[0], b[1], b[2], b[3]);
345 
346  /* use old current_data as l2_hdr_offset and new current_data as
347  l3_hdr_offset */
348  r = (u64x4) u16x16_blend (r, radv << 16, 0xaa);
349 
350  /* store both l2_hdr_offset and l3_hdr_offset in single store operation */
351  u32x8_scatter_one ((u32x8) r, 0, &vnet_buffer (b[0])->l2_hdr_offset);
352  u32x8_scatter_one ((u32x8) r, 2, &vnet_buffer (b[1])->l2_hdr_offset);
353  u32x8_scatter_one ((u32x8) r, 4, &vnet_buffer (b[2])->l2_hdr_offset);
354  u32x8_scatter_one ((u32x8) r, 6, &vnet_buffer (b[3])->l2_hdr_offset);
355 
356  if (is_l3)
357  {
358  ASSERT (b[0]->current_data == vnet_buffer (b[0])->l3_hdr_offset);
359  ASSERT (b[1]->current_data == vnet_buffer (b[1])->l3_hdr_offset);
360  ASSERT (b[2]->current_data == vnet_buffer (b[2])->l3_hdr_offset);
361  ASSERT (b[3]->current_data == vnet_buffer (b[3])->l3_hdr_offset);
362 
363  ASSERT (b[0]->current_data - vnet_buffer (b[0])->l2_hdr_offset == adv);
364  ASSERT (b[1]->current_data - vnet_buffer (b[1])->l2_hdr_offset == adv);
365  ASSERT (b[2]->current_data - vnet_buffer (b[2])->l2_hdr_offset == adv);
366  ASSERT (b[3]->current_data - vnet_buffer (b[3])->l2_hdr_offset == adv);
367  }
368  else
369  {
370  ASSERT (b[0]->current_data == vnet_buffer (b[0])->l2_hdr_offset);
371  ASSERT (b[1]->current_data == vnet_buffer (b[1])->l2_hdr_offset);
372  ASSERT (b[2]->current_data == vnet_buffer (b[2])->l2_hdr_offset);
373  ASSERT (b[3]->current_data == vnet_buffer (b[3])->l2_hdr_offset);
374 
375  ASSERT (b[0]->current_data - vnet_buffer (b[0])->l3_hdr_offset == -adv);
376  ASSERT (b[1]->current_data - vnet_buffer (b[1])->l3_hdr_offset == -adv);
377  ASSERT (b[2]->current_data - vnet_buffer (b[2])->l3_hdr_offset == -adv);
378  ASSERT (b[3]->current_data - vnet_buffer (b[3])->l3_hdr_offset == -adv);
379  }
380 
381 #else
382  vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
383  vnet_buffer (b[1])->l2_hdr_offset = b[1]->current_data;
384  vnet_buffer (b[2])->l2_hdr_offset = b[2]->current_data;
385  vnet_buffer (b[3])->l2_hdr_offset = b[3]->current_data;
386  vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
387  vnet_buffer (b[1])->l3_hdr_offset = b[1]->current_data + adv;
388  vnet_buffer (b[2])->l3_hdr_offset = b[2]->current_data + adv;
389  vnet_buffer (b[3])->l3_hdr_offset = b[3]->current_data + adv;
390 
391  if (is_l3)
392  {
393  vlib_buffer_advance (b[0], adv);
394  vlib_buffer_advance (b[1], adv);
395  vlib_buffer_advance (b[2], adv);
396  vlib_buffer_advance (b[3], adv);
397  }
398 
399  b[0]->flags |= flags;
400  b[1]->flags |= flags;
401  b[2]->flags |= flags;
402  b[3]->flags |= flags;
403 #endif
404 
405  if (!is_l3)
406  {
407  vnet_buffer (b[0])->l2.l2_len = adv;
408  vnet_buffer (b[1])->l2.l2_len = adv;
409  vnet_buffer (b[2])->l2.l2_len = adv;
410  vnet_buffer (b[3])->l2.l2_len = adv;
411  }
412 }
413 
416 {
417  i16 adv = sizeof (ethernet_header_t);
418  u32 flags = VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
419  VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
420 
421  vnet_buffer (b[0])->l2_hdr_offset = b[0]->current_data;
422  vnet_buffer (b[0])->l3_hdr_offset = b[0]->current_data + adv;
423 
424  if (is_l3)
425  vlib_buffer_advance (b[0], adv);
426  b[0]->flags |= flags;
427  if (!is_l3)
428  vnet_buffer (b[0])->l2.l2_len = adv;
429 }
430 
431 
434  u64 * dmacs, int offset, int dmac_check)
435 {
437  e = vlib_buffer_get_current (b[offset]);
438 #ifdef CLIB_HAVE_VEC128
439  u64x2 r = u64x2_load_unaligned (((u8 *) & e->type) - 6);
440  etype[offset] = ((u16x8) r)[3];
441  tags[offset] = r[1];
442 #else
443  etype[offset] = e->type;
444  tags[offset] = *(u64 *) (e + 1);
445 #endif
446 
447  if (dmac_check)
448  dmacs[offset] = *(u64 *) e;
449 }
450 
453 {
455 
456  return (etype < 0x600) ? ETHERNET_INPUT_NEXT_LLC :
459 }
460 
461 typedef struct
462 {
463  u64 tag, mask;
465  u16 type, len, next;
467  u8 err, n_tags;
468  u64 n_packets, n_bytes;
470 
474 {
475  if (l->n_packets == 0 || l->sw_if_index == ~0)
476  return;
477 
478  if (l->adv > 0)
479  l->n_bytes += l->n_packets * l->len;
480 
484  l->n_packets, l->n_bytes);
485 }
486 
490  u64 tag, u16 * next, vlib_buffer_t * b,
491  eth_input_tag_lookup_t * l, u8 dmac_bad, int is_dot1ad,
492  int main_is_l3, int check_dmac)
493 {
495 
496  if ((tag ^ l->tag) & l->mask)
497  {
499  vlan_intf_t *vif;
500  qinq_intf_t *qif;
501  vlan_table_t *vlan_table;
502  qinq_table_t *qinq_table;
503  u16 *t = (u16 *) & tag;
504  u16 vlan1 = clib_net_to_host_u16 (t[0]) & 0xFFF;
505  u16 vlan2 = clib_net_to_host_u16 (t[2]) & 0xFFF;
506  u32 matched, is_l2, new_sw_if_index;
507 
508  vlan_table = vec_elt_at_index (em->vlan_pool, is_dot1ad ?
509  mif->dot1ad_vlans : mif->dot1q_vlans);
510  vif = &vlan_table->vlans[vlan1];
511  qinq_table = vec_elt_at_index (em->qinq_pool, vif->qinqs);
512  qif = &qinq_table->vlans[vlan2];
513  l->err = ETHERNET_ERROR_NONE;
514  l->type = clib_net_to_host_u16 (t[1]);
515 
516  if (l->type == ETHERNET_TYPE_VLAN)
517  {
518  l->type = clib_net_to_host_u16 (t[3]);
519  l->n_tags = 2;
521  SUBINT_CONFIG_MATCH_2_TAG, mif, vif,
522  qif, &new_sw_if_index, &l->err,
523  &is_l2);
524  }
525  else
526  {
527  l->n_tags = 1;
528  if (vlan1 == 0)
529  {
530  new_sw_if_index = hi->sw_if_index;
531  l->err = ETHERNET_ERROR_NONE;
532  matched = 1;
533  is_l2 = main_is_l3 == 0;
534  }
535  else
538  vif, qif, &new_sw_if_index,
539  &l->err, &is_l2);
540  }
541 
542  if (l->sw_if_index != new_sw_if_index)
543  {
544  eth_input_update_if_counters (vm, vnm, l);
545  l->n_packets = 0;
546  l->n_bytes = 0;
547  l->sw_if_index = new_sw_if_index;
548  }
549  l->tag = tag;
550  l->mask = (l->n_tags == 2) ?
551  clib_net_to_host_u64 (0xffffffffffffffff) :
552  clib_net_to_host_u64 (0xffffffff00000000);
553 
554  if (matched && l->sw_if_index == ~0)
555  l->err = ETHERNET_ERROR_DOWN;
556 
557  l->len = sizeof (ethernet_header_t) +
558  l->n_tags * sizeof (ethernet_vlan_header_t);
559  if (main_is_l3)
560  l->adv = is_l2 ? -(int) sizeof (ethernet_header_t) :
561  l->n_tags * sizeof (ethernet_vlan_header_t);
562  else
563  l->adv = is_l2 ? 0 : l->len;
564 
565  if (PREDICT_FALSE (l->err != ETHERNET_ERROR_NONE))
566  l->next = ETHERNET_INPUT_NEXT_DROP;
567  else if (is_l2)
568  l->next = em->l2_next;
569  else if (l->type == ETHERNET_TYPE_IP4)
570  l->next = em->l3_next.input_next_ip4;
571  else if (l->type == ETHERNET_TYPE_IP6)
572  l->next = em->l3_next.input_next_ip6;
573  else if (l->type == ETHERNET_TYPE_MPLS)
574  l->next = em->l3_next.input_next_mpls;
575  else if (em->redirect_l3)
576  l->next = em->redirect_l3_next;
577  else
578  {
579  l->next = eth_input_next_by_type (l->type);
580  if (l->next == ETHERNET_INPUT_NEXT_PUNT)
581  l->err = ETHERNET_ERROR_UNKNOWN_TYPE;
582  }
583  }
584 
585  if (check_dmac && l->adv > 0 && dmac_bad)
586  {
587  l->err = ETHERNET_ERROR_L3_MAC_MISMATCH;
588  next[0] = ETHERNET_INPUT_NEXT_PUNT;
589  }
590  else
591  next[0] = l->next;
592 
593  vlib_buffer_advance (b, l->adv);
594  vnet_buffer (b)->l2.l2_len = l->len;
595  vnet_buffer (b)->l3_hdr_offset = vnet_buffer (b)->l2_hdr_offset + l->len;
596 
597  if (l->err == ETHERNET_ERROR_NONE)
598  {
599  vnet_buffer (b)->sw_if_index[VLIB_RX] = l->sw_if_index;
601  }
602  else
603  b->error = node->errors[l->err];
604 
605  /* update counters */
606  l->n_packets += 1;
607  l->n_bytes += vlib_buffer_length_in_chain (vm, b);
608 }
609 
610 /* process frame of buffers, store ethertype into array and update
611  buffer metadata fields depending on interface being l2 or l3 assuming that
612  packets are untagged. For tagged packets those fields are updated later.
613  Optionally store Destionation MAC address and tag data into arrays
614  for further processing */
615 
617  "VLIB_FRAME_SIZE must be power of 8");
621  u32 * buffer_indices, u32 n_packets, int main_is_l3,
622  int ip4_cksum_ok, int dmac_check)
623 {
625  u16 nexts[VLIB_FRAME_SIZE], *next;
626  u16 etypes[VLIB_FRAME_SIZE], *etype = etypes;
627  u64 dmacs[VLIB_FRAME_SIZE], *dmac = dmacs;
628  u8 dmacs_bad[VLIB_FRAME_SIZE];
629  u64 tags[VLIB_FRAME_SIZE], *tag = tags;
630  u16 slowpath_indices[VLIB_FRAME_SIZE];
631  u16 n_slowpath, i;
632  u16 next_ip4, next_ip6, next_mpls, next_l2;
633  u16 et_ip4 = clib_host_to_net_u16 (ETHERNET_TYPE_IP4);
634  u16 et_ip6 = clib_host_to_net_u16 (ETHERNET_TYPE_IP6);
635  u16 et_mpls = clib_host_to_net_u16 (ETHERNET_TYPE_MPLS);
636  u16 et_vlan = clib_host_to_net_u16 (ETHERNET_TYPE_VLAN);
637  u16 et_dot1ad = clib_host_to_net_u16 (ETHERNET_TYPE_DOT1AD);
638  i32 n_left = n_packets;
639  vlib_buffer_t *b[20];
640  u32 *from;
641 
642  from = buffer_indices;
643 
644  while (n_left >= 20)
645  {
646  vlib_buffer_t **ph = b + 16, **pd = b + 8;
647  vlib_get_buffers (vm, from, b, 4);
648  vlib_get_buffers (vm, from + 8, pd, 4);
649  vlib_get_buffers (vm, from + 16, ph, 4);
650 
651  vlib_prefetch_buffer_header (ph[0], LOAD);
652  vlib_prefetch_buffer_data (pd[0], LOAD);
653  eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
654 
655  vlib_prefetch_buffer_header (ph[1], LOAD);
656  vlib_prefetch_buffer_data (pd[1], LOAD);
657  eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
658 
659  vlib_prefetch_buffer_header (ph[2], LOAD);
660  vlib_prefetch_buffer_data (pd[2], LOAD);
661  eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
662 
663  vlib_prefetch_buffer_header (ph[3], LOAD);
664  vlib_prefetch_buffer_data (pd[3], LOAD);
665  eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
666 
667  eth_input_adv_and_flags_x4 (b, main_is_l3);
668 
669  /* next */
670  n_left -= 4;
671  etype += 4;
672  tag += 4;
673  dmac += 4;
674  from += 4;
675  }
676  while (n_left >= 4)
677  {
678  vlib_get_buffers (vm, from, b, 4);
679  eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
680  eth_input_get_etype_and_tags (b, etype, tag, dmac, 1, dmac_check);
681  eth_input_get_etype_and_tags (b, etype, tag, dmac, 2, dmac_check);
682  eth_input_get_etype_and_tags (b, etype, tag, dmac, 3, dmac_check);
683  eth_input_adv_and_flags_x4 (b, main_is_l3);
684 
685  /* next */
686  n_left -= 4;
687  etype += 4;
688  tag += 4;
689  dmac += 4;
690  from += 4;
691  }
692  while (n_left)
693  {
694  vlib_get_buffers (vm, from, b, 1);
695  eth_input_get_etype_and_tags (b, etype, tag, dmac, 0, dmac_check);
696  eth_input_adv_and_flags_x1 (b, main_is_l3);
697 
698  /* next */
699  n_left -= 1;
700  etype += 1;
701  tag += 1;
702  dmac += 4;
703  from += 1;
704  }
705 
706  if (dmac_check)
707  {
708  u64 mask = clib_net_to_host_u64 (0xFFFFFFFFFFFF0000);
709  u64 igbit = clib_net_to_host_u64 (0x0100000000000000);
710  u64 hwaddr = (*(u64 *) hi->hw_address) & mask;
711  u64 *dmac = dmacs;
712  u8 *dmac_bad = dmacs_bad;
713 
714  n_left = n_packets;
715 
716 #ifdef CLIB_HAVE_VEC256
717  u64x4 igbit4 = u64x4_splat (igbit);
718  u64x4 mask4 = u64x4_splat (mask);
719  u64x4 hwaddr4 = u64x4_splat (hwaddr);
720  while (n_left >= 0)
721  {
722  u64x4 r0, r1;
723  r0 = u64x4_load_unaligned (dmac + 0) & mask4;
724  r1 = u64x4_load_unaligned (dmac + 4) & mask4;
725 
726  r0 = (r0 != hwaddr4) & ((r0 & igbit4) == 0);
727  r1 = (r1 != hwaddr4) & ((r1 & igbit4) == 0);
728 
729  *(u32 *) (dmac_bad + 0) = u8x32_msb_mask ((u8x32) (r0));
730  *(u32 *) (dmac_bad + 4) = u8x32_msb_mask ((u8x32) (r1));
731 
732  /* next */
733  dmac += 8;
734  dmac_bad += 8;
735  n_left -= 8;
736  }
737 #else
738  while (n_left > 0)
739  {
740  u64 r0, r1, r2, r3;
741  r0 = dmac[0] & mask;
742  r1 = dmac[1] & mask;
743  r2 = dmac[2] & mask;
744  r3 = dmac[3] & mask;
745 
746  r0 = (r0 != hwaddr) && ((r0 & igbit) == 0);
747  r1 = (r1 != hwaddr) && ((r1 & igbit) == 0);
748  r2 = (r2 != hwaddr) && ((r2 & igbit) == 0);
749  r3 = (r3 != hwaddr) && ((r3 & igbit) == 0);
750 
751  dmac_bad[0] = r0;
752  dmac_bad[1] = r1;
753  dmac_bad[2] = r2;
754  dmac_bad[3] = r3;
755 
756  /* next */
757  dmac += 4;
758  dmac_bad += 4;
759  n_left -= 4;
760  }
761 #endif
762  }
763 
764  next_ip4 = em->l3_next.input_next_ip4;
765  next_ip6 = em->l3_next.input_next_ip6;
766  next_mpls = em->l3_next.input_next_mpls;
767  next_l2 = em->l2_next;
768 
769  if (next_ip4 == ETHERNET_INPUT_NEXT_IP4_INPUT && ip4_cksum_ok)
770  next_ip4 = ETHERNET_INPUT_NEXT_IP4_INPUT_NCS;
771 
772 #ifdef CLIB_HAVE_VEC256
773  u16x16 et16_ip4 = u16x16_splat (et_ip4);
774  u16x16 et16_ip6 = u16x16_splat (et_ip6);
775  u16x16 et16_mpls = u16x16_splat (et_mpls);
776  u16x16 et16_vlan = u16x16_splat (et_vlan);
777  u16x16 et16_dot1ad = u16x16_splat (et_dot1ad);
778  u16x16 next16_ip4 = u16x16_splat (next_ip4);
779  u16x16 next16_ip6 = u16x16_splat (next_ip6);
780  u16x16 next16_mpls = u16x16_splat (next_mpls);
781  u16x16 next16_l2 = u16x16_splat (next_l2);
782  u16x16 zero = { 0 };
783  u16x16 stairs = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 };
784 #endif
785 
786  etype = etypes;
787  n_left = n_packets;
788  next = nexts;
789  n_slowpath = 0;
790  i = 0;
791 
792  /* fastpath - in l3 mode hadles ip4, ip6 and mpls packets, other packets
793  are considered as slowpath, in l2 mode all untagged packets are
794  considered as fastpath */
795  while (n_left > 0)
796  {
797 #ifdef CLIB_HAVE_VEC256
798  if (n_left >= 16)
799  {
800  u16x16 r = zero;
801  u16x16 e16 = u16x16_load_unaligned (etype);
802  if (main_is_l3)
803  {
804  r += (e16 == et16_ip4) & next16_ip4;
805  r += (e16 == et16_ip6) & next16_ip6;
806  r += (e16 == et16_mpls) & next16_mpls;
807  }
808  else
809  r = ((e16 != et16_vlan) & (e16 != et16_dot1ad)) & next16_l2;
810  u16x16_store_unaligned (r, next);
811 
812  if (!u16x16_is_all_zero (r == zero))
813  {
814  if (u16x16_is_all_zero (r))
815  {
816  u16x16_store_unaligned (u16x16_splat (i) + stairs,
817  slowpath_indices + n_slowpath);
818  n_slowpath += 16;
819  }
820  else
821  {
822  for (int j = 0; j < 16; j++)
823  if (next[j] == 0)
824  slowpath_indices[n_slowpath++] = i + j;
825  }
826  }
827 
828  etype += 16;
829  next += 16;
830  n_left -= 16;
831  i += 16;
832  continue;
833  }
834 #endif
835  if (main_is_l3 && etype[0] == et_ip4)
836  next[0] = next_ip4;
837  else if (main_is_l3 && etype[0] == et_ip6)
838  next[0] = next_ip6;
839  else if (main_is_l3 && etype[0] == et_mpls)
840  next[0] = next_mpls;
841  else if (main_is_l3 == 0 &&
842  etype[0] != et_vlan && etype[0] != et_dot1ad)
843  next[0] = next_l2;
844  else
845  {
846  next[0] = 0;
847  slowpath_indices[n_slowpath++] = i;
848  }
849 
850  etype += 1;
851  next += 1;
852  n_left -= 1;
853  i += 1;
854  }
855 
856  if (n_slowpath)
857  {
858  vnet_main_t *vnm = vnet_get_main ();
859  n_left = n_slowpath;
860  u16 *si = slowpath_indices;
861  u32 last_unknown_etype = ~0;
862  u32 last_unknown_next = ~0;
863  eth_input_tag_lookup_t dot1ad_lookup, dot1q_lookup = {
864  .mask = -1LL,
865  .tag = tags[si[0]] ^ -1LL,
866  .sw_if_index = ~0
867  };
868 
869  clib_memcpy_fast (&dot1ad_lookup, &dot1q_lookup, sizeof (dot1q_lookup));
870 
871  while (n_left)
872  {
873  i = si[0];
874  u16 etype = etypes[i];
875 
876  if (etype == et_vlan)
877  {
878  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
879  eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
880  &dot1q_lookup, dmacs_bad[i], 0,
881  main_is_l3, dmac_check);
882 
883  }
884  else if (etype == et_dot1ad)
885  {
886  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
887  eth_input_tag_lookup (vm, vnm, node, hi, tags[i], nexts + i, b,
888  &dot1ad_lookup, dmacs_bad[i], 1,
889  main_is_l3, dmac_check);
890  }
891  else
892  {
893  /* untagged packet with not well known etyertype */
894  if (last_unknown_etype != etype)
895  {
896  last_unknown_etype = etype;
897  etype = clib_host_to_net_u16 (etype);
898  last_unknown_next = eth_input_next_by_type (etype);
899  }
900  if (dmac_check && main_is_l3 && dmacs_bad[i])
901  {
902  vlib_buffer_t *b = vlib_get_buffer (vm, buffer_indices[i]);
903  b->error = node->errors[ETHERNET_ERROR_L3_MAC_MISMATCH];
904  nexts[i] = ETHERNET_INPUT_NEXT_PUNT;
905  }
906  else
907  nexts[i] = last_unknown_next;
908  }
909 
910  /* next */
911  n_left--;
912  si++;
913  }
914 
915  eth_input_update_if_counters (vm, vnm, &dot1q_lookup);
916  eth_input_update_if_counters (vm, vnm, &dot1ad_lookup);
917  }
918 
919  vlib_buffer_enqueue_to_next (vm, node, buffer_indices, nexts, n_packets);
920 }
921 
924  vnet_hw_interface_t * hi, u32 * from, u32 n_pkts,
925  int ip4_cksum_ok)
926 {
929  ei = pool_elt_at_index (em->interfaces, hi->hw_instance);
931  subint_config_t *subint0 = &intf0->untagged_subint;
932 
933  int main_is_l3 = (subint0->flags & SUBINT_CONFIG_L2) == 0;
934  int promisc = (ei->flags & ETHERNET_INTERFACE_FLAG_ACCEPT_ALL) != 0;
935 
936  if (main_is_l3)
937  {
938  /* main interface is L3, we dont expect tagged packets and interface
939  is not in promisc node, so we dont't need to check DMAC */
940  int is_l3 = 1;
941 
942  if (promisc == 0)
943  eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3,
944  ip4_cksum_ok, 0);
945  else
946  /* subinterfaces and promisc mode so DMAC check is needed */
947  eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3,
948  ip4_cksum_ok, 1);
949  return;
950  }
951  else
952  {
953  /* untagged packets are treated as L2 */
954  int is_l3 = 0;
955  eth_input_process_frame (vm, node, hi, from, n_pkts, is_l3,
956  ip4_cksum_ok, 1);
957  return;
958  }
959 }
960 
963  vlib_frame_t * from_frame)
964 {
965  u32 *from, n_left;
966  if ((node->flags & VLIB_NODE_FLAG_TRACE) == 0)
967  return;
968 
969  from = vlib_frame_vector_args (from_frame);
970  n_left = from_frame->n_vectors;
971 
972  while (n_left)
973  {
975  vlib_buffer_t *b0 = vlib_get_buffer (vm, from[0]);
976 
977  if (b0->flags & VLIB_BUFFER_IS_TRACED)
978  {
979  t0 = vlib_add_trace (vm, node, b0, sizeof (ethernet_input_trace_t));
981  sizeof (t0->packet_data));
982  t0->frame_flags = from_frame->flags;
984  vlib_frame_scalar_args (from_frame),
985  sizeof (ethernet_input_frame_t));
986  }
987  from += 1;
988  n_left -= 1;
989  }
990 }
991 
994  vlib_node_runtime_t * node,
995  u32 * from, u32 n_packets,
996  ethernet_input_variant_t variant)
997 {
998  vnet_main_t *vnm = vnet_get_main ();
1000  vlib_node_runtime_t *error_node;
1001  u32 n_left_from, next_index, *to_next;
1002  u32 stats_sw_if_index, stats_n_packets, stats_n_bytes;
1003  u32 thread_index = vm->thread_index;
1004  u32 cached_sw_if_index = ~0;
1005  u32 cached_is_l2 = 0; /* shut up gcc */
1006  vnet_hw_interface_t *hi = NULL; /* used for main interface only */
1007 
1008  if (variant != ETHERNET_INPUT_VARIANT_ETHERNET)
1009  error_node = vlib_node_get_runtime (vm, ethernet_input_node.index);
1010  else
1011  error_node = node;
1012 
1013  n_left_from = n_packets;
1014 
1015  next_index = node->cached_next_index;
1016  stats_sw_if_index = node->runtime_data[0];
1017  stats_n_packets = stats_n_bytes = 0;
1018 
1019  while (n_left_from > 0)
1020  {
1021  u32 n_left_to_next;
1022 
1023  vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
1024 
1025  while (n_left_from >= 4 && n_left_to_next >= 2)
1026  {
1027  u32 bi0, bi1;
1028  vlib_buffer_t *b0, *b1;
1029  u8 next0, next1, error0, error1;
1030  u16 type0, orig_type0, type1, orig_type1;
1031  u16 outer_id0, inner_id0, outer_id1, inner_id1;
1032  u32 match_flags0, match_flags1;
1033  u32 old_sw_if_index0, new_sw_if_index0, len0, old_sw_if_index1,
1034  new_sw_if_index1, len1;
1035  vnet_hw_interface_t *hi0, *hi1;
1036  main_intf_t *main_intf0, *main_intf1;
1037  vlan_intf_t *vlan_intf0, *vlan_intf1;
1038  qinq_intf_t *qinq_intf0, *qinq_intf1;
1039  u32 is_l20, is_l21;
1040  ethernet_header_t *e0, *e1;
1041 
1042  /* Prefetch next iteration. */
1043  {
1044  vlib_buffer_t *b2, *b3;
1045 
1046  b2 = vlib_get_buffer (vm, from[2]);
1047  b3 = vlib_get_buffer (vm, from[3]);
1048 
1049  vlib_prefetch_buffer_header (b2, STORE);
1050  vlib_prefetch_buffer_header (b3, STORE);
1051 
1052  CLIB_PREFETCH (b2->data, sizeof (ethernet_header_t), LOAD);
1053  CLIB_PREFETCH (b3->data, sizeof (ethernet_header_t), LOAD);
1054  }
1055 
1056  bi0 = from[0];
1057  bi1 = from[1];
1058  to_next[0] = bi0;
1059  to_next[1] = bi1;
1060  from += 2;
1061  to_next += 2;
1062  n_left_to_next -= 2;
1063  n_left_from -= 2;
1064 
1065  b0 = vlib_get_buffer (vm, bi0);
1066  b1 = vlib_get_buffer (vm, bi1);
1067 
1068  error0 = error1 = ETHERNET_ERROR_NONE;
1069  e0 = vlib_buffer_get_current (b0);
1070  type0 = clib_net_to_host_u16 (e0->type);
1071  e1 = vlib_buffer_get_current (b1);
1072  type1 = clib_net_to_host_u16 (e1->type);
1073 
1074  /* Set the L2 header offset for all packets */
1075  vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1076  vnet_buffer (b1)->l2_hdr_offset = b1->current_data;
1077  b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1078  b1->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1079 
1080  /* Speed-path for the untagged case */
1083  type1)))
1084  {
1085  main_intf_t *intf0;
1086  subint_config_t *subint0;
1087  u32 sw_if_index0, sw_if_index1;
1088 
1089  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1090  sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1091  is_l20 = cached_is_l2;
1092 
1093  /* This is probably wholly unnecessary */
1094  if (PREDICT_FALSE (sw_if_index0 != sw_if_index1))
1095  goto slowpath;
1096 
1097  /* Now sw_if_index0 == sw_if_index1 */
1098  if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1099  {
1100  cached_sw_if_index = sw_if_index0;
1101  hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1102  intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1103  subint0 = &intf0->untagged_subint;
1104  cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1105  }
1106 
1107  if (PREDICT_TRUE (is_l20 != 0))
1108  {
1109  vnet_buffer (b0)->l3_hdr_offset =
1110  vnet_buffer (b0)->l2_hdr_offset +
1111  sizeof (ethernet_header_t);
1112  vnet_buffer (b1)->l3_hdr_offset =
1113  vnet_buffer (b1)->l2_hdr_offset +
1114  sizeof (ethernet_header_t);
1115  b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1116  b1->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1117  next0 = em->l2_next;
1118  vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1119  next1 = em->l2_next;
1120  vnet_buffer (b1)->l2.l2_len = sizeof (ethernet_header_t);
1121  }
1122  else
1123  {
1124  if (!ethernet_address_cast (e0->dst_address) &&
1125  (hi->hw_address != 0) &&
1126  !eth_mac_equal ((u8 *) e0, hi->hw_address))
1127  error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1128  if (!ethernet_address_cast (e1->dst_address) &&
1129  (hi->hw_address != 0) &&
1130  !eth_mac_equal ((u8 *) e1, hi->hw_address))
1131  error1 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1132  vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1133  determine_next_node (em, variant, 0, type0, b0,
1134  &error0, &next0);
1135  vlib_buffer_advance (b1, sizeof (ethernet_header_t));
1136  determine_next_node (em, variant, 0, type1, b1,
1137  &error1, &next1);
1138  }
1139  goto ship_it01;
1140  }
1141 
1142  /* Slow-path for the tagged case */
1143  slowpath:
1144  parse_header (variant,
1145  b0,
1146  &type0,
1147  &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1148 
1149  parse_header (variant,
1150  b1,
1151  &type1,
1152  &orig_type1, &outer_id1, &inner_id1, &match_flags1);
1153 
1154  old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1155  old_sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
1156 
1158  vnm,
1159  old_sw_if_index0,
1160  orig_type0,
1161  outer_id0,
1162  inner_id0,
1163  &hi0,
1164  &main_intf0, &vlan_intf0, &qinq_intf0);
1165 
1167  vnm,
1168  old_sw_if_index1,
1169  orig_type1,
1170  outer_id1,
1171  inner_id1,
1172  &hi1,
1173  &main_intf1, &vlan_intf1, &qinq_intf1);
1174 
1175  identify_subint (hi0,
1176  b0,
1177  match_flags0,
1178  main_intf0,
1179  vlan_intf0,
1180  qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1181 
1182  identify_subint (hi1,
1183  b1,
1184  match_flags1,
1185  main_intf1,
1186  vlan_intf1,
1187  qinq_intf1, &new_sw_if_index1, &error1, &is_l21);
1188 
1189  // Save RX sw_if_index for later nodes
1190  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1191  error0 !=
1192  ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1193  vnet_buffer (b1)->sw_if_index[VLIB_RX] =
1194  error1 !=
1195  ETHERNET_ERROR_NONE ? old_sw_if_index1 : new_sw_if_index1;
1196 
1197  // Check if there is a stat to take (valid and non-main sw_if_index for pkt 0 or pkt 1)
1198  if (((new_sw_if_index0 != ~0)
1199  && (new_sw_if_index0 != old_sw_if_index0))
1200  || ((new_sw_if_index1 != ~0)
1201  && (new_sw_if_index1 != old_sw_if_index1)))
1202  {
1203 
1204  len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1205  - vnet_buffer (b0)->l2_hdr_offset;
1206  len1 = vlib_buffer_length_in_chain (vm, b1) + b1->current_data
1207  - vnet_buffer (b1)->l2_hdr_offset;
1208 
1209  stats_n_packets += 2;
1210  stats_n_bytes += len0 + len1;
1211 
1212  if (PREDICT_FALSE
1213  (!(new_sw_if_index0 == stats_sw_if_index
1214  && new_sw_if_index1 == stats_sw_if_index)))
1215  {
1216  stats_n_packets -= 2;
1217  stats_n_bytes -= len0 + len1;
1218 
1219  if (new_sw_if_index0 != old_sw_if_index0
1220  && new_sw_if_index0 != ~0)
1222  interface_main.combined_sw_if_counters
1223  +
1225  thread_index,
1226  new_sw_if_index0, 1,
1227  len0);
1228  if (new_sw_if_index1 != old_sw_if_index1
1229  && new_sw_if_index1 != ~0)
1231  interface_main.combined_sw_if_counters
1232  +
1234  thread_index,
1235  new_sw_if_index1, 1,
1236  len1);
1237 
1238  if (new_sw_if_index0 == new_sw_if_index1)
1239  {
1240  if (stats_n_packets > 0)
1241  {
1245  thread_index,
1246  stats_sw_if_index,
1247  stats_n_packets, stats_n_bytes);
1248  stats_n_packets = stats_n_bytes = 0;
1249  }
1250  stats_sw_if_index = new_sw_if_index0;
1251  }
1252  }
1253  }
1254 
1255  if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1256  is_l20 = is_l21 = 0;
1257 
1258  determine_next_node (em, variant, is_l20, type0, b0, &error0,
1259  &next0);
1260  determine_next_node (em, variant, is_l21, type1, b1, &error1,
1261  &next1);
1262 
1263  ship_it01:
1264  b0->error = error_node->errors[error0];
1265  b1->error = error_node->errors[error1];
1266 
1267  // verify speculative enqueue
1268  vlib_validate_buffer_enqueue_x2 (vm, node, next_index, to_next,
1269  n_left_to_next, bi0, bi1, next0,
1270  next1);
1271  }
1272 
1273  while (n_left_from > 0 && n_left_to_next > 0)
1274  {
1275  u32 bi0;
1276  vlib_buffer_t *b0;
1277  u8 error0, next0;
1278  u16 type0, orig_type0;
1279  u16 outer_id0, inner_id0;
1280  u32 match_flags0;
1281  u32 old_sw_if_index0, new_sw_if_index0, len0;
1282  vnet_hw_interface_t *hi0;
1283  main_intf_t *main_intf0;
1284  vlan_intf_t *vlan_intf0;
1285  qinq_intf_t *qinq_intf0;
1286  ethernet_header_t *e0;
1287  u32 is_l20;
1288 
1289  // Prefetch next iteration
1290  if (n_left_from > 1)
1291  {
1292  vlib_buffer_t *p2;
1293 
1294  p2 = vlib_get_buffer (vm, from[1]);
1295  vlib_prefetch_buffer_header (p2, STORE);
1297  }
1298 
1299  bi0 = from[0];
1300  to_next[0] = bi0;
1301  from += 1;
1302  to_next += 1;
1303  n_left_from -= 1;
1304  n_left_to_next -= 1;
1305 
1306  b0 = vlib_get_buffer (vm, bi0);
1307 
1308  error0 = ETHERNET_ERROR_NONE;
1309  e0 = vlib_buffer_get_current (b0);
1310  type0 = clib_net_to_host_u16 (e0->type);
1311 
1312  /* Set the L2 header offset for all packets */
1313  vnet_buffer (b0)->l2_hdr_offset = b0->current_data;
1314  b0->flags |= VNET_BUFFER_F_L2_HDR_OFFSET_VALID;
1315 
1316  /* Speed-path for the untagged case */
1318  && !ethernet_frame_is_tagged (type0)))
1319  {
1320  main_intf_t *intf0;
1321  subint_config_t *subint0;
1322  u32 sw_if_index0;
1323 
1324  sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1325  is_l20 = cached_is_l2;
1326 
1327  if (PREDICT_FALSE (cached_sw_if_index != sw_if_index0))
1328  {
1329  cached_sw_if_index = sw_if_index0;
1330  hi = vnet_get_sup_hw_interface (vnm, sw_if_index0);
1331  intf0 = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1332  subint0 = &intf0->untagged_subint;
1333  cached_is_l2 = is_l20 = subint0->flags & SUBINT_CONFIG_L2;
1334  }
1335 
1336 
1337  if (PREDICT_TRUE (is_l20 != 0))
1338  {
1339  vnet_buffer (b0)->l3_hdr_offset =
1340  vnet_buffer (b0)->l2_hdr_offset +
1341  sizeof (ethernet_header_t);
1342  b0->flags |= VNET_BUFFER_F_L3_HDR_OFFSET_VALID;
1343  next0 = em->l2_next;
1344  vnet_buffer (b0)->l2.l2_len = sizeof (ethernet_header_t);
1345  }
1346  else
1347  {
1348  if (!ethernet_address_cast (e0->dst_address) &&
1349  (hi->hw_address != 0) &&
1350  !eth_mac_equal ((u8 *) e0, hi->hw_address))
1351  error0 = ETHERNET_ERROR_L3_MAC_MISMATCH;
1352  vlib_buffer_advance (b0, sizeof (ethernet_header_t));
1353  determine_next_node (em, variant, 0, type0, b0,
1354  &error0, &next0);
1355  }
1356  goto ship_it0;
1357  }
1358 
1359  /* Slow-path for the tagged case */
1360  parse_header (variant,
1361  b0,
1362  &type0,
1363  &orig_type0, &outer_id0, &inner_id0, &match_flags0);
1364 
1365  old_sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
1366 
1368  vnm,
1369  old_sw_if_index0,
1370  orig_type0,
1371  outer_id0,
1372  inner_id0,
1373  &hi0,
1374  &main_intf0, &vlan_intf0, &qinq_intf0);
1375 
1376  identify_subint (hi0,
1377  b0,
1378  match_flags0,
1379  main_intf0,
1380  vlan_intf0,
1381  qinq_intf0, &new_sw_if_index0, &error0, &is_l20);
1382 
1383  // Save RX sw_if_index for later nodes
1384  vnet_buffer (b0)->sw_if_index[VLIB_RX] =
1385  error0 !=
1386  ETHERNET_ERROR_NONE ? old_sw_if_index0 : new_sw_if_index0;
1387 
1388  // Increment subinterface stats
1389  // Note that interface-level counters have already been incremented
1390  // prior to calling this function. Thus only subinterface counters
1391  // are incremented here.
1392  //
1393  // Interface level counters include packets received on the main
1394  // interface and all subinterfaces. Subinterface level counters
1395  // include only those packets received on that subinterface
1396  // Increment stats if the subint is valid and it is not the main intf
1397  if ((new_sw_if_index0 != ~0)
1398  && (new_sw_if_index0 != old_sw_if_index0))
1399  {
1400 
1401  len0 = vlib_buffer_length_in_chain (vm, b0) + b0->current_data
1402  - vnet_buffer (b0)->l2_hdr_offset;
1403 
1404  stats_n_packets += 1;
1405  stats_n_bytes += len0;
1406 
1407  // Batch stat increments from the same subinterface so counters
1408  // don't need to be incremented for every packet.
1409  if (PREDICT_FALSE (new_sw_if_index0 != stats_sw_if_index))
1410  {
1411  stats_n_packets -= 1;
1412  stats_n_bytes -= len0;
1413 
1414  if (new_sw_if_index0 != ~0)
1418  thread_index, new_sw_if_index0, 1, len0);
1419  if (stats_n_packets > 0)
1420  {
1424  thread_index,
1425  stats_sw_if_index, stats_n_packets, stats_n_bytes);
1426  stats_n_packets = stats_n_bytes = 0;
1427  }
1428  stats_sw_if_index = new_sw_if_index0;
1429  }
1430  }
1431 
1432  if (variant == ETHERNET_INPUT_VARIANT_NOT_L2)
1433  is_l20 = 0;
1434 
1435  determine_next_node (em, variant, is_l20, type0, b0, &error0,
1436  &next0);
1437 
1438  ship_it0:
1439  b0->error = error_node->errors[error0];
1440 
1441  // verify speculative enqueue
1442  vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
1443  to_next, n_left_to_next,
1444  bi0, next0);
1445  }
1446 
1447  vlib_put_next_frame (vm, node, next_index, n_left_to_next);
1448  }
1449 
1450  // Increment any remaining batched stats
1451  if (stats_n_packets > 0)
1452  {
1456  thread_index, stats_sw_if_index, stats_n_packets, stats_n_bytes);
1457  node->runtime_data[0] = stats_sw_if_index;
1458  }
1459 }
1460 
1462  vlib_node_runtime_t * node,
1463  vlib_frame_t * frame)
1464 {
1465  vnet_main_t *vnm = vnet_get_main ();
1466  u32 *from = vlib_frame_vector_args (frame);
1467  u32 n_packets = frame->n_vectors;
1468 
1469  ethernet_input_trace (vm, node, frame);
1470 
1471  if (frame->flags & ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX)
1472  {
1474  int ip4_cksum_ok = (frame->flags & ETH_INPUT_FRAME_F_IP4_CKSUM_OK) != 0;
1476  eth_input_single_int (vm, node, hi, from, n_packets, ip4_cksum_ok);
1477  }
1478  else
1479  ethernet_input_inline (vm, node, from, n_packets,
1481  return n_packets;
1482 }
1483 
1485  vlib_node_runtime_t * node,
1486  vlib_frame_t * from_frame)
1487 {
1488  u32 *from = vlib_frame_vector_args (from_frame);
1489  u32 n_packets = from_frame->n_vectors;
1490  ethernet_input_trace (vm, node, from_frame);
1491  ethernet_input_inline (vm, node, from, n_packets,
1493  return n_packets;
1494 }
1495 
1497  vlib_node_runtime_t * node,
1498  vlib_frame_t * from_frame)
1499 {
1500  u32 *from = vlib_frame_vector_args (from_frame);
1501  u32 n_packets = from_frame->n_vectors;
1502  ethernet_input_trace (vm, node, from_frame);
1503  ethernet_input_inline (vm, node, from, n_packets,
1505  return n_packets;
1506 }
1507 
1508 
1509 // Return the subinterface config struct for the given sw_if_index
1510 // Also return via parameter the appropriate match flags for the
1511 // configured number of tags.
1512 // On error (unsupported or not ethernet) return 0.
1513 static subint_config_t *
1515  u32 sw_if_index,
1516  u32 * flags, u32 * unsupported)
1517 {
1520  vnet_sw_interface_t *si;
1521  main_intf_t *main_intf;
1522  vlan_table_t *vlan_table;
1523  qinq_table_t *qinq_table;
1524  subint_config_t *subint = 0;
1525 
1526  hi = vnet_get_sup_hw_interface (vnm, sw_if_index);
1527 
1528  if (!hi || (hi->hw_class_index != ethernet_hw_interface_class.index))
1529  {
1530  *unsupported = 0;
1531  goto done; // non-ethernet interface
1532  }
1533 
1534  // ensure there's an entry for the main intf (shouldn't really be necessary)
1535  vec_validate (em->main_intfs, hi->hw_if_index);
1536  main_intf = vec_elt_at_index (em->main_intfs, hi->hw_if_index);
1537 
1538  // Locate the subint for the given ethernet config
1539  si = vnet_get_sw_interface (vnm, sw_if_index);
1540 
1541  if (si->type == VNET_SW_INTERFACE_TYPE_P2P)
1542  {
1543  p2p_ethernet_main_t *p2pm = &p2p_main;
1544  u32 p2pe_sw_if_index =
1546  if (p2pe_sw_if_index == ~0)
1547  {
1548  pool_get (p2pm->p2p_subif_pool, subint);
1549  si->p2p.pool_index = subint - p2pm->p2p_subif_pool;
1550  }
1551  else
1552  subint = vec_elt_at_index (p2pm->p2p_subif_pool, si->p2p.pool_index);
1553  *flags = SUBINT_CONFIG_P2P;
1554  }
1555  else if (si->type == VNET_SW_INTERFACE_TYPE_PIPE)
1556  {
1557  pipe_t *pipe;
1558 
1559  pipe = pipe_get (sw_if_index);
1560  subint = &pipe->subint;
1561  *flags = SUBINT_CONFIG_P2P;
1562  }
1563  else if (si->sub.eth.flags.default_sub)
1564  {
1565  subint = &main_intf->default_subint;
1566  *flags = SUBINT_CONFIG_MATCH_1_TAG |
1568  }
1569  else if ((si->sub.eth.flags.no_tags) || (si->sub.eth.raw_flags == 0))
1570  {
1571  // if no flags are set then this is a main interface
1572  // so treat as untagged
1573  subint = &main_intf->untagged_subint;
1574  *flags = SUBINT_CONFIG_MATCH_0_TAG;
1575  }
1576  else
1577  {
1578  // one or two tags
1579  // first get the vlan table
1580  if (si->sub.eth.flags.dot1ad)
1581  {
1582  if (main_intf->dot1ad_vlans == 0)
1583  {
1584  // Allocate a vlan table from the pool
1585  pool_get (em->vlan_pool, vlan_table);
1586  main_intf->dot1ad_vlans = vlan_table - em->vlan_pool;
1587  }
1588  else
1589  {
1590  // Get ptr to existing vlan table
1591  vlan_table =
1592  vec_elt_at_index (em->vlan_pool, main_intf->dot1ad_vlans);
1593  }
1594  }
1595  else
1596  { // dot1q
1597  if (main_intf->dot1q_vlans == 0)
1598  {
1599  // Allocate a vlan table from the pool
1600  pool_get (em->vlan_pool, vlan_table);
1601  main_intf->dot1q_vlans = vlan_table - em->vlan_pool;
1602  }
1603  else
1604  {
1605  // Get ptr to existing vlan table
1606  vlan_table =
1607  vec_elt_at_index (em->vlan_pool, main_intf->dot1q_vlans);
1608  }
1609  }
1610 
1611  if (si->sub.eth.flags.one_tag)
1612  {
1613  *flags = si->sub.eth.flags.exact_match ?
1617 
1618  if (si->sub.eth.flags.outer_vlan_id_any)
1619  {
1620  // not implemented yet
1621  *unsupported = 1;
1622  goto done;
1623  }
1624  else
1625  {
1626  // a single vlan, a common case
1627  subint =
1628  &vlan_table->vlans[si->sub.eth.
1629  outer_vlan_id].single_tag_subint;
1630  }
1631 
1632  }
1633  else
1634  {
1635  // Two tags
1636  *flags = si->sub.eth.flags.exact_match ?
1639 
1640  if (si->sub.eth.flags.outer_vlan_id_any
1641  && si->sub.eth.flags.inner_vlan_id_any)
1642  {
1643  // not implemented yet
1644  *unsupported = 1;
1645  goto done;
1646  }
1647 
1648  if (si->sub.eth.flags.inner_vlan_id_any)
1649  {
1650  // a specific outer and "any" inner
1651  // don't need a qinq table for this
1652  subint =
1653  &vlan_table->vlans[si->sub.eth.
1654  outer_vlan_id].inner_any_subint;
1655  if (si->sub.eth.flags.exact_match)
1656  {
1657  *flags = SUBINT_CONFIG_MATCH_2_TAG;
1658  }
1659  else
1660  {
1661  *flags = SUBINT_CONFIG_MATCH_2_TAG |
1663  }
1664  }
1665  else
1666  {
1667  // a specific outer + specifc innner vlan id, a common case
1668 
1669  // get the qinq table
1670  if (vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs == 0)
1671  {
1672  // Allocate a qinq table from the pool
1673  pool_get (em->qinq_pool, qinq_table);
1674  vlan_table->vlans[si->sub.eth.outer_vlan_id].qinqs =
1675  qinq_table - em->qinq_pool;
1676  }
1677  else
1678  {
1679  // Get ptr to existing qinq table
1680  qinq_table =
1682  vlan_table->vlans[si->sub.
1683  eth.outer_vlan_id].
1684  qinqs);
1685  }
1686  subint = &qinq_table->vlans[si->sub.eth.inner_vlan_id].subint;
1687  }
1688  }
1689  }
1690 
1691 done:
1692  return subint;
1693 }
1694 
1695 static clib_error_t *
1697 {
1698  subint_config_t *subint;
1699  u32 dummy_flags;
1700  u32 dummy_unsup;
1701  clib_error_t *error = 0;
1702 
1703  // Find the config for this subinterface
1704  subint =
1705  ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags,
1706  &dummy_unsup);
1707 
1708  if (subint == 0)
1709  {
1710  // not implemented yet or not ethernet
1711  goto done;
1712  }
1713 
1714  subint->sw_if_index =
1715  ((flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) ? sw_if_index : ~0);
1716 
1717 done:
1718  return error;
1719 }
1720 
1722 
1723 
1724 #ifndef CLIB_MARCH_VARIANT
1725 // Set the L2/L3 mode for the subinterface
1726 void
1728 {
1729  subint_config_t *subint;
1730  u32 dummy_flags;
1731  u32 dummy_unsup;
1732  int is_port;
1733  vnet_sw_interface_t *sw = vnet_get_sw_interface (vnm, sw_if_index);
1734 
1735  is_port = !(sw->type == VNET_SW_INTERFACE_TYPE_SUB);
1736 
1737  // Find the config for this subinterface
1738  subint =
1739  ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags,
1740  &dummy_unsup);
1741 
1742  if (subint == 0)
1743  {
1744  // unimplemented or not ethernet
1745  goto done;
1746  }
1747 
1748  // Double check that the config we found is for our interface (or the interface is down)
1749  ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
1750 
1751  if (l2)
1752  {
1753  subint->flags |= SUBINT_CONFIG_L2;
1754  if (is_port)
1755  subint->flags |=
1758  }
1759  else
1760  {
1761  subint->flags &= ~SUBINT_CONFIG_L2;
1762  if (is_port)
1763  subint->flags &=
1766  }
1767 
1768 done:
1769  return;
1770 }
1771 
1772 /*
1773  * Set the L2/L3 mode for the subinterface regardless of port
1774  */
1775 void
1777  u32 sw_if_index, u32 l2)
1778 {
1779  subint_config_t *subint;
1780  u32 dummy_flags;
1781  u32 dummy_unsup;
1782 
1783  /* Find the config for this subinterface */
1784  subint =
1785  ethernet_sw_interface_get_config (vnm, sw_if_index, &dummy_flags,
1786  &dummy_unsup);
1787 
1788  if (subint == 0)
1789  {
1790  /* unimplemented or not ethernet */
1791  goto done;
1792  }
1793 
1794  /*
1795  * Double check that the config we found is for our interface (or the
1796  * interface is down)
1797  */
1798  ASSERT ((subint->sw_if_index == sw_if_index) | (subint->sw_if_index == ~0));
1799 
1800  if (l2)
1801  {
1802  subint->flags |= SUBINT_CONFIG_L2;
1803  }
1804  else
1805  {
1806  subint->flags &= ~SUBINT_CONFIG_L2;
1807  }
1808 
1809 done:
1810  return;
1811 }
1812 #endif
1813 
1814 static clib_error_t *
1816  u32 sw_if_index, u32 is_create)
1817 {
1818  clib_error_t *error = 0;
1819  subint_config_t *subint;
1820  u32 match_flags;
1821  u32 unsupported = 0;
1822 
1823  // Find the config for this subinterface
1824  subint =
1825  ethernet_sw_interface_get_config (vnm, sw_if_index, &match_flags,
1826  &unsupported);
1827 
1828  if (subint == 0)
1829  {
1830  // not implemented yet or not ethernet
1831  if (unsupported)
1832  {
1833  // this is the NYI case
1834  error = clib_error_return (0, "not implemented yet");
1835  }
1836  goto done;
1837  }
1838 
1839  if (!is_create)
1840  {
1841  subint->flags = 0;
1842  return error;
1843  }
1844 
1845  // Initialize the subint
1846  if (subint->flags & SUBINT_CONFIG_VALID)
1847  {
1848  // Error vlan already in use
1849  error = clib_error_return (0, "vlan is already in use");
1850  }
1851  else
1852  {
1853  // Note that config is L3 by default
1854  subint->flags = SUBINT_CONFIG_VALID | match_flags;
1855  subint->sw_if_index = ~0; // because interfaces are initially down
1856  }
1857 
1858 done:
1859  return error;
1860 }
1861 
1863 
1864 static char *ethernet_error_strings[] = {
1865 #define ethernet_error(n,c,s) s,
1866 #include "error.def"
1867 #undef ethernet_error
1868 };
1869 
1870 /* *INDENT-OFF* */
1872  .name = "ethernet-input",
1873  /* Takes a vector of packets. */
1874  .vector_size = sizeof (u32),
1875  .scalar_size = sizeof (ethernet_input_frame_t),
1876  .n_errors = ETHERNET_N_ERROR,
1877  .error_strings = ethernet_error_strings,
1878  .n_next_nodes = ETHERNET_INPUT_N_NEXT,
1879  .next_nodes = {
1880 #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
1882 #undef _
1883  },
1884  .format_buffer = format_ethernet_header_with_length,
1885  .format_trace = format_ethernet_input_trace,
1886  .unformat_buffer = unformat_ethernet_header,
1887 };
1888 
1890  .name = "ethernet-input-type",
1891  /* Takes a vector of packets. */
1892  .vector_size = sizeof (u32),
1893  .n_next_nodes = ETHERNET_INPUT_N_NEXT,
1894  .next_nodes = {
1895 #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
1897 #undef _
1898  },
1899 };
1900 
1902  .name = "ethernet-input-not-l2",
1903  /* Takes a vector of packets. */
1904  .vector_size = sizeof (u32),
1905  .n_next_nodes = ETHERNET_INPUT_N_NEXT,
1906  .next_nodes = {
1907 #define _(s,n) [ETHERNET_INPUT_NEXT_##s] = n,
1909 #undef _
1910  },
1911 };
1912 /* *INDENT-ON* */
1913 
1914 #ifndef CLIB_MARCH_VARIANT
1915 void
1917  vnet_hw_interface_t * hi, u32 enable)
1918 {
1919  // Insure all packets go to ethernet-input (i.e. untagged ipv4 packets
1920  // don't go directly to ip4-input)
1922  (vnm, hi->hw_if_index, enable ? ethernet_input_node.index : ~0);
1923 }
1924 
1925 
1926 /*
1927  * Initialization and registration for the next_by_ethernet structure
1928  */
1929 
1930 clib_error_t *
1932 {
1934  ( /* elt bytes */ sizeof (l3_next->input_next_by_type[0]),
1935  /* bits in index */ BITS (((ethernet_header_t *) 0)->type));
1936 
1938  ETHERNET_INPUT_NEXT_DROP);
1940  ETHERNET_INPUT_NEXT_PUNT);
1941  l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_DROP] =
1943  l3_next->sparse_index_by_input_next_index[ETHERNET_INPUT_NEXT_PUNT] =
1945 
1946  /*
1947  * Make sure we don't wipe out an ethernet registration by mistake
1948  * Can happen if init function ordering constraints are missing.
1949  */
1950  if (CLIB_DEBUG > 0)
1951  {
1954  }
1955 
1956  return 0;
1957 }
1958 
1959 // Add an ethertype -> next index mapping to the structure
1960 clib_error_t *
1962  u32 ethertype, u32 next_index)
1963 {
1964  u32 i;
1965  u16 *n;
1967 
1968  if (CLIB_DEBUG > 0)
1969  {
1972  }
1973 
1974  /* Setup ethernet type -> next index sparse vector mapping. */
1975  n = sparse_vec_validate (l3_next->input_next_by_type, ethertype);
1976  n[0] = next_index;
1977 
1978  /* Rebuild next index -> sparse index inverse mapping when sparse vector
1979  is updated. */
1980  vec_validate (l3_next->sparse_index_by_input_next_index, next_index);
1981  for (i = 1; i < vec_len (l3_next->input_next_by_type); i++)
1982  l3_next->
1983  sparse_index_by_input_next_index[l3_next->input_next_by_type[i]] = i;
1984 
1985  // do not allow the cached next index's to be updated if L3
1986  // redirect is enabled, as it will have overwritten them
1987  if (!em->redirect_l3)
1988  {
1989  // Cache common ethertypes directly
1990  if (ethertype == ETHERNET_TYPE_IP4)
1991  {
1992  l3_next->input_next_ip4 = next_index;
1993  }
1994  else if (ethertype == ETHERNET_TYPE_IP6)
1995  {
1996  l3_next->input_next_ip6 = next_index;
1997  }
1998  else if (ethertype == ETHERNET_TYPE_MPLS)
1999  {
2000  l3_next->input_next_mpls = next_index;
2001  }
2002  }
2003  return 0;
2004 }
2005 
2006 
2007 static clib_error_t *
2009 {
2011  __attribute__ ((unused)) vlan_table_t *invalid_vlan_table;
2012  __attribute__ ((unused)) qinq_table_t *invalid_qinq_table;
2013 
2017 
2019 
2020  // Initialize pools and vector for vlan parsing
2021  vec_validate (em->main_intfs, 10); // 10 main interfaces
2022  pool_alloc (em->vlan_pool, 10);
2023  pool_alloc (em->qinq_pool, 1);
2024 
2025  // The first vlan pool will always be reserved for an invalid table
2026  pool_get (em->vlan_pool, invalid_vlan_table); // first id = 0
2027  // The first qinq pool will always be reserved for an invalid table
2028  pool_get (em->qinq_pool, invalid_qinq_table); // first id = 0
2029 
2030  return 0;
2031 }
2032 
2034 
2035 void
2037  ethernet_type_t type, u32 node_index)
2038 {
2041  u32 i;
2042 
2043  {
2045  if (error)
2046  clib_error_report (error);
2047  }
2048 
2049  ti = ethernet_get_type_info (em, type);
2050  ti->node_index = node_index;
2051  ti->next_index = vlib_node_add_next (vm,
2052  ethernet_input_node.index, node_index);
2053  i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2054  ASSERT (i == ti->next_index);
2055 
2056  i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2057  ASSERT (i == ti->next_index);
2058 
2059  // Add the L3 node for this ethertype to the next nodes structure
2060  next_by_ethertype_register (&em->l3_next, type, ti->next_index);
2061 
2062  // Call the registration functions for other nodes that want a mapping
2063  l2bvi_register_input_type (vm, type, node_index);
2064 }
2065 
2066 void
2068 {
2070  u32 i;
2071 
2072  em->l2_next =
2073  vlib_node_add_next (vm, ethernet_input_node.index, node_index);
2074 
2075  /*
2076  * Even if we never use these arcs, we have to align the next indices...
2077  */
2078  i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2079 
2080  ASSERT (i == em->l2_next);
2081 
2082  i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2083  ASSERT (i == em->l2_next);
2084 }
2085 
2086 // Register a next node for L3 redirect, and enable L3 redirect
2087 void
2089 {
2091  u32 i;
2092 
2093  em->redirect_l3 = 1;
2095  ethernet_input_node.index,
2096  node_index);
2097  /*
2098  * Change the cached next nodes to the redirect node
2099  */
2103 
2104  /*
2105  * Even if we never use these arcs, we have to align the next indices...
2106  */
2107  i = vlib_node_add_next (vm, ethernet_input_type_node.index, node_index);
2108 
2109  ASSERT (i == em->redirect_l3_next);
2110 
2111  i = vlib_node_add_next (vm, ethernet_input_not_l2_node.index, node_index);
2112 
2113  ASSERT (i == em->redirect_l3_next);
2114 }
2115 #endif
2116 
2117 /*
2118  * fd.io coding-style-patch-verification: ON
2119  *
2120  * Local Variables:
2121  * eval: (c-set-style "gnu")
2122  * End:
2123  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
subint_config_t subint
Definition: ethernet.h:266
vmrglw vmrglh hi
VNET_SW_INTERFACE_ADMIN_UP_DOWN_FUNCTION(bond_sw_interface_up_down)
static_always_inline void identify_subint(vnet_hw_interface_t *hi, vlib_buffer_t *b0, u32 match_flags, main_intf_t *main_intf, vlan_intf_t *vlan_intf, qinq_intf_t *qinq_intf, u32 *new_sw_if_index, u8 *error0, u32 *is_l2)
Definition: node.c:200
u32 flags
Definition: vhost_user.h:115
#define CLIB_UNUSED(x)
Definition: clib.h:82
STATIC_ASSERT_OFFSET_OF(vlib_buffer_t, current_data, 0)
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
Definition: counter.h:220
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:545
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
#define pool_alloc(P, N)
Allocate N more free elements to pool (unspecified alignment).
Definition: pool.h:341
vnet_interface_main_t interface_main
Definition: vnet.h:56
vnet_p2p_sub_interface_t p2p
Definition: interface.h:729
#define PREDICT_TRUE(x)
Definition: clib.h:112
unsigned long u64
Definition: types.h:89
int vnet_hw_interface_rx_redirect_to_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: interface.c:1176
u16x16 u64x4 static_always_inline u32 u8x32_msb_mask(u8x32 v)
Definition: vector_avx2.h:108
void ethernet_register_l2_input(vlib_main_t *vm, u32 node_index)
Definition: node.c:2067
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static_always_inline void eth_input_tag_lookup(vlib_main_t *vm, vnet_main_t *vnm, vlib_node_runtime_t *node, vnet_hw_interface_t *hi, u64 tag, u16 *next, vlib_buffer_t *b, eth_input_tag_lookup_t *l, u8 dmac_bad, int is_dot1ad, int main_is_l3, int check_dmac)
Definition: node.c:488
#define NULL
Definition: clib.h:58
ethernet_type_t
Definition: packet.h:45
#define SUBINT_CONFIG_MATCH_0_TAG
Definition: ethernet.h:227
#define ethernet_buffer_header_size(b)
Determine the size of the Ethernet headers of the current frame in the buffer.
Definition: ethernet.h:453
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 thread_index
Definition: main.h:179
static subint_config_t * ethernet_sw_interface_get_config(vnet_main_t *vnm, u32 sw_if_index, u32 *flags, u32 *unsupported)
Definition: node.c:1514
ethernet_input_variant_t
Definition: node.c:93
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
main_intf_t * main_intfs
Definition: ethernet.h:314
#define STRUCT_OFFSET_OF(t, f)
Definition: clib.h:65
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
subint_config_t inner_any_subint
Definition: ethernet.h:255
u8 packet_data[32]
Definition: node.c:65
#define VLIB_NODE_FN(node)
Definition: node.h:201
vlib_error_t * errors
Vector of errors for this node.
Definition: node.h:493
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:267
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:236
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1092
unsigned char u8
Definition: types.h:56
static_always_inline void u64x4_scatter(u64x4 r, void *p0, void *p1, void *p2, void *p3)
Definition: vector_avx2.h:216
static clib_error_t * ethernet_sw_interface_add_del(vnet_main_t *vnm, u32 sw_if_index, u32 is_create)
Definition: node.c:1815
static u32 eth_mac_equal(const u8 *mac1, const u8 *mac2)
Definition: ethernet.h:570
struct vnet_sub_interface_t::@198::@199::@201 flags
epu16_epi64 epu8_epi64 epu8_epi64 epi16_epi64 i16x16
Definition: vector_avx2.h:129
vlib_node_registration_t ethernet_input_node
(constructor) VLIB_REGISTER_NODE (ethernet_input_node)
Definition: node.c:1871
ethernet_main_t ethernet_main
Definition: init.c:45
represenation of a pipe interface
Definition: pipe.h:24
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
#define static_always_inline
Definition: clib.h:99
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
static uword ethernet_address_cast(u8 *a)
Definition: packet.h:67
static_always_inline u16 eth_input_next_by_type(u16 etype)
Definition: node.c:452
u32 sw_if_index
Definition: vxlan_gbp.api:37
void ethernet_set_rx_redirect(vnet_main_t *vnm, vnet_hw_interface_t *hi, u32 enable)
Definition: node.c:1916
clib_error_t * next_by_ethertype_init(next_by_ethertype_t *l3_next)
Definition: node.c:1931
#define ETH_INPUT_FRAME_F_SINGLE_SW_IF_IDX
Definition: ethernet.h:51
vlib_combined_counter_main_t * combined_sw_if_counters
Definition: interface.h:839
u8 dst_address[6]
Definition: packet.h:55
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:113
#define sparse_vec_validate(v, i)
Definition: sparse_vec.h:227
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
Definition: buffer.h:188
subint_config_t default_subint
Definition: ethernet.h:247
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
unsigned int u32
Definition: types.h:88
#define vlib_call_init_function(vm, x)
Definition: init.h:260
#define VLIB_FRAME_SIZE
Definition: node.h:401
static char * ethernet_error_strings[]
Definition: node.c:1864
ethernet_input_next_t
Definition: node.c:55
epu8_epi32 epu16_epi32 u64x2
Definition: vector_sse42.h:640
static clib_error_t * ethernet_input_init(vlib_main_t *vm)
Definition: node.c:2008
u16 dot1q_vlans
Definition: ethernet.h:248
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
subint_config_t single_tag_subint
Definition: ethernet.h:254
vnet_sub_interface_t sub
Definition: interface.h:726
u32 * sparse_index_by_input_next_index
Definition: ethernet.h:282
static ethernet_type_info_t * ethernet_get_type_info(ethernet_main_t *em, ethernet_type_t type)
Definition: ethernet.h:338
unsigned short u16
Definition: types.h:57
static_always_inline u64x4 u64x4_gather(void *p0, void *p1, void *p2, void *p3)
Definition: vector_avx2.h:195
#define ETH_INPUT_FRAME_F_IP4_CKSUM_OK
Definition: ethernet.h:54
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:214
static_always_inline void eth_input_process_frame(vlib_main_t *vm, vlib_node_runtime_t *node, vnet_hw_interface_t *hi, u32 *buffer_indices, u32 n_packets, int main_is_l3, int ip4_cksum_ok, int dmac_check)
Definition: node.c:619
vlib_node_registration_t ethernet_input_type_node
(constructor) VLIB_REGISTER_NODE (ethernet_input_type_node)
Definition: node.c:1889
#define PREDICT_FALSE(x)
Definition: clib.h:111
#define ethernet_buffer_set_vlan_count(b, v)
Sets the number of VLAN headers in the current Ethernet frame in the buffer.
Definition: ethernet.h:426
u16 dot1ad_vlans
Definition: ethernet.h:249
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
Definition: buffer_node.h:70
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
Definition: buffer_node.h:218
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
Definition: node_funcs.h:338
#define SUBINT_CONFIG_MATCH_3_TAG
Definition: ethernet.h:230
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:139
vnet_hw_interface_class_t ethernet_hw_interface_class
u8 next_by_ethertype_register_called
Definition: ethernet.h:326
subint_config_t untagged_subint
Definition: ethernet.h:246
u8 len
Definition: ip_types.api:49
static_always_inline void u32x8_scatter_one(u32x8 r, int index, void *p)
Definition: vector_avx2.h:245
u32 p2p_ethernet_lookup(u32 parent_if_index, u8 *client_mac)
Definition: p2p_ethernet.c:35
u8 * format_ethernet_header_with_length(u8 *s, va_list *args)
Definition: format.c:97
#define SUBINT_CONFIG_VALID
Definition: ethernet.h:231
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
static_always_inline void eth_input_single_int(vlib_main_t *vm, vlib_node_runtime_t *node, vnet_hw_interface_t *hi, u32 *from, u32 n_pkts, int ip4_cksum_ok)
Definition: node.c:923
qinq_intf_t vlans[ETHERNET_N_VLAN]
Definition: ethernet.h:271
u16 n_vectors
Definition: node.h:420
VNET_SW_INTERFACE_ADD_DEL_FUNCTION(ethernet_sw_interface_add_del)
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:79
vlib_main_t * vm
Definition: buffer.c:301
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
Definition: buffer_node.h:332
struct vnet_sub_interface_t::@198 eth
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:174
#define vlib_prefetch_buffer_data(b, type)
Definition: buffer.h:189
static u32 eth_identify_subint(vnet_hw_interface_t *hi, u32 match_flags, main_intf_t *main_intf, vlan_intf_t *vlan_intf, qinq_intf_t *qinq_intf, u32 *new_sw_if_index, u8 *error0, u32 *is_l2)
Definition: ethernet.h:519
static void * vlib_frame_scalar_args(vlib_frame_t *f)
Get pointer to frame scalar data.
Definition: node_funcs.h:258
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:452
#define SUBINT_CONFIG_MATCH_1_TAG
Definition: ethernet.h:228
static_always_inline void parse_header(ethernet_input_variant_t variant, vlib_buffer_t *b0, u16 *type, u16 *orig_type, u16 *outer_id, u16 *inner_id, u32 *match_flags)
Definition: node.c:103
static_always_inline void eth_input_adv_and_flags_x1(vlib_buffer_t **b, int is_l3)
Definition: node.c:415
static void eth_vlan_table_lookups(ethernet_main_t *em, vnet_main_t *vnm, u32 port_sw_if_index0, u16 first_ethertype, u16 outer_id, u16 inner_id, vnet_hw_interface_t **hi, main_intf_t **main_intf, vlan_intf_t **vlan_intf, qinq_intf_t **qinq_intf)
Definition: ethernet.h:480
signed int i32
Definition: types.h:77
static uword sparse_vec_index(void *v, uword sparse_index)
Definition: sparse_vec.h:157
static_always_inline void ethernet_input_inline(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *from, u32 n_packets, ethernet_input_variant_t variant)
Definition: node.c:993
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:537
#define SUBINT_CONFIG_L2
Definition: ethernet.h:232
vlan_table_t * vlan_pool
Definition: ethernet.h:317
#define ASSERT(truth)
u8 * format_ethernet_header(u8 *s, va_list *args)
Definition: format.c:178
u32 redirect_l3_next
Definition: ethernet.h:302
void ethernet_register_l3_redirect(vlib_main_t *vm, u32 node_index)
Definition: node.c:2088
u16 flags
Definition: node.h:411
void ethernet_sw_interface_set_l2_mode_noport(vnet_main_t *vnm, u32 sw_if_index, u32 l2)
Definition: node.c:1776
static_always_inline int ethernet_frame_is_tagged(u16 type)
Definition: ethernet.h:111
static_always_inline void determine_next_node(ethernet_main_t *em, ethernet_input_variant_t variant, u32 is_l20, u32 type0, vlib_buffer_t *b0, u8 *error0, u8 *next0)
Definition: node.c:239
void ethernet_register_input_type(vlib_main_t *vm, ethernet_type_t type, u32 node_index)
Definition: node.c:2036
#define clib_error_report(e)
Definition: error.h:113
static u8 * format_ethernet_input_trace(u8 *s, va_list *va)
Definition: node.c:71
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:233
STATIC_ASSERT(STRUCT_OFFSET_OF(vnet_buffer_opaque_t, l2_hdr_offset)==STRUCT_OFFSET_OF(vnet_buffer_opaque_t, l3_hdr_offset) - 2, "l3_hdr_offset must follow l2_hdr_offset")
vlan_intf_t vlans[ETHERNET_N_VLAN]
Definition: ethernet.h:261
#define SUBINT_CONFIG_MATCH_2_TAG
Definition: ethernet.h:229
clib_error_t * next_by_ethertype_register(next_by_ethertype_t *l3_next, u32 ethertype, u32 next_index)
Definition: node.c:1961
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
Definition: trace_funcs.h:57
#define vec_elt(v, i)
Get vector value at index i.
subint_config_t subint
Sub-interface config.
Definition: pipe.h:30
vlib_node_registration_t ethernet_input_not_l2_node
(constructor) VLIB_REGISTER_NODE (ethernet_input_not_l2_node)
Definition: node.c:1901
struct _vlib_node_registration vlib_node_registration_t
template key/value backing page structure
Definition: bihash_doc.h:44
#define foreach_ethernet_input_next
Definition: node.c:48
#define SUBINT_CONFIG_P2P
Definition: ethernet.h:233
void l2bvi_register_input_type(vlib_main_t *vm, ethernet_type_t type, u32 node_index)
Definition: l2_bvi.c:27
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
pipe_t * pipe_get(u32 sw_if_index)
Get the pipe instnace based on one end.
Definition: pipe.c:95
static_always_inline int ethernet_frame_is_any_tagged_x2(u16 type0, u16 type1)
Definition: ethernet.h:125
static_always_inline void eth_input_adv_and_flags_x4(vlib_buffer_t **b, int is_l3)
Definition: node.c:313
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
static clib_error_t * ethernet_init(vlib_main_t *vm)
Definition: init.c:83
void ethernet_sw_interface_set_l2_mode(vnet_main_t *vnm, u32 sw_if_index, u32 l2)
Definition: node.c:1727
struct clib_bihash_value offset
template key/value backing page structure
u64x4
Definition: vector_avx2.h:121
#define vnet_buffer(b)
Definition: buffer.h:368
p2p_ethernet_main_t p2p_main
Definition: p2p_ethernet.c:23
vnet_sw_interface_type_t type
Definition: interface.h:704
qinq_table_t * qinq_pool
Definition: ethernet.h:320
u8 data[0]
Packet data.
Definition: buffer.h:176
#define u16x16_blend(v1, v2, mask)
Definition: vector_avx2.h:191
u16 flags
Copy of main node flags.
Definition: node.h:531
next_by_ethertype_t l3_next
Definition: ethernet.h:295
subint_config_t * p2p_subif_pool
Definition: p2p_ethernet.h:31
static clib_error_t * ethernet_sw_interface_up_down(vnet_main_t *vnm, u32 sw_if_index, u32 flags)
Definition: node.c:1696
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:145
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:326
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static void * sparse_vec_new(uword elt_bytes, uword sparse_index_bits)
Definition: sparse_vec.h:71
ethernet_interface_t * interfaces
Definition: ethernet.h:305
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:117
static_always_inline void eth_input_update_if_counters(vlib_main_t *vm, vnet_main_t *vnm, eth_input_tag_lookup_t *l)
Definition: node.c:472
epu16_epi64 u16x16
Definition: vector_avx2.h:123
ethernet_input_frame_t frame_data
Definition: node.c:67
#define BITS(x)
Definition: clib.h:61
static_always_inline void eth_input_get_etype_and_tags(vlib_buffer_t **b, u16 *etype, u64 *tags, u64 *dmacs, int offset, int dmac_check)
Definition: node.c:433
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:62
static_always_inline void ethernet_input_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
Definition: node.c:962
static void ethernet_setup_node(vlib_main_t *vm, u32 node_index)
Definition: ethernet.h:398
Definition: defs.h:46
uword unformat_ethernet_header(unformat_input_t *input, va_list *args)
Definition: format.c:290
u16 * input_next_by_type
Definition: ethernet.h:281
#define SPARSE_VEC_INVALID_INDEX
Definition: sparse_vec.h:68
signed short i16
Definition: types.h:46