FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
ip4_mtrie.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * ip/ip4_fib.h: ip4 mtrie fib
17  *
18  * Copyright (c) 2012 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <vnet/ip/ip.h>
41 #include <vnet/ip/ip4_mtrie.h>
42 #include <vnet/fib/ip4_fib.h>
43 
44 
45 /**
46  * Global pool of IPv4 8bit PLYs
47  */
49 
52 {
53  /*
54  * It's 'non-empty' if the length of the leaf stored is greater than the
55  * length of a leaf in the covering ply. i.e. the leaf is more specific
56  * than it's would be cover in the covering ply
57  */
59  return (1);
60  return (0);
61 }
62 
65 {
67  l = 1 + 2 * adj_index;
68  ASSERT (ip4_fib_mtrie_leaf_get_adj_index (l) == adj_index);
69  return l;
70 }
71 
74 {
75  return (n & 1) == 0;
76 }
77 
80 {
82  return n >> 1;
83 }
84 
87 {
89  l = 0 + 2 * i;
91  return l;
92 }
93 
94 #ifndef __ALTIVEC__
95 #define PLY_X4_SPLAT_INIT(init_x4, init) \
96  init_x4 = u32x4_splat (init);
97 #else
98 #define PLY_X4_SPLAT_INIT(init_x4, init) \
99 { \
100  u32x4_union_t y; \
101  y.as_u32[0] = init; \
102  y.as_u32[1] = init; \
103  y.as_u32[2] = init; \
104  y.as_u32[3] = init; \
105  init_x4 = y.as_u32x4; \
106 }
107 #endif
108 
109 #ifdef CLIB_HAVE_VEC128
110 #define PLY_INIT_LEAVES(p) \
111 { \
112  u32x4 *l, init_x4; \
113  \
114  PLY_X4_SPLAT_INIT(init_x4, init); \
115  for (l = p->leaves_as_u32x4; \
116  l < p->leaves_as_u32x4 + ARRAY_LEN (p->leaves_as_u32x4); \
117  l += 4) \
118  { \
119  l[0] = init_x4; \
120  l[1] = init_x4; \
121  l[2] = init_x4; \
122  l[3] = init_x4; \
123  } \
124 }
125 #else
126 #define PLY_INIT_LEAVES(p) \
127 { \
128  u32 *l; \
129  \
130  for (l = p->leaves; l < p->leaves + ARRAY_LEN (p->leaves); l += 4) \
131  { \
132  l[0] = init; \
133  l[1] = init; \
134  l[2] = init; \
135  l[3] = init; \
136  } \
137 }
138 #endif
139 
140 #define PLY_INIT(p, init, prefix_len, ply_base_len) \
141 { \
142  /* \
143  * A leaf is 'empty' if it represents a leaf from the covering PLY \
144  * i.e. if the prefix length of the leaf is less than or equal to \
145  * the prefix length of the PLY \
146  */ \
147  p->n_non_empty_leafs = (prefix_len > ply_base_len ? \
148  ARRAY_LEN (p->leaves) : 0); \
149  clib_memset (p->dst_address_bits_of_leaves, prefix_len, \
150  sizeof (p->dst_address_bits_of_leaves)); \
151  p->dst_address_bits_base = ply_base_len; \
152  \
153  /* Initialize leaves. */ \
154  PLY_INIT_LEAVES(p); \
155 }
156 
157 static void
159  ip4_fib_mtrie_leaf_t init, uword prefix_len, u32 ply_base_len)
160 {
161  PLY_INIT (p, init, prefix_len, ply_base_len);
162 }
163 
164 static void
166  ip4_fib_mtrie_leaf_t init, uword prefix_len)
167 {
168  clib_memset (p->dst_address_bits_of_leaves, prefix_len,
169  sizeof (p->dst_address_bits_of_leaves));
170  PLY_INIT_LEAVES (p);
171 }
172 
175  ip4_fib_mtrie_leaf_t init_leaf,
176  u32 leaf_prefix_len, u32 ply_base_len)
177 {
179  void *old_heap;
180  /* Get cache aligned ply. */
181 
183  pool_get_aligned (ip4_ply_pool, p, CLIB_CACHE_LINE_BYTES);
184  clib_mem_set_heap (old_heap);
185 
186  ply_8_init (p, init_leaf, leaf_prefix_len, ply_base_len);
187  return ip4_fib_mtrie_leaf_set_next_ply_index (p - ip4_ply_pool);
188 }
189 
192 {
194 
195  return pool_elt_at_index (ip4_ply_pool, n);
196 }
197 
198 void
200 {
201  /* the root ply is embedded so there is nothing to do,
202  * the assumption being that the IP4 FIB table has emptied the trie
203  * before deletion.
204  */
205 #if CLIB_DEBUG > 0
206  int i;
207  for (i = 0; i < ARRAY_LEN (m->root_ply.leaves); i++)
208  {
210  }
211 #endif
212 }
213 
214 void
216 {
218 }
219 
220 typedef struct
221 {
228 
229 static void
231  ip4_fib_mtrie_8_ply_t * ply,
232  ip4_fib_mtrie_leaf_t new_leaf,
233  uword new_leaf_dst_address_bits)
234 {
235  ip4_fib_mtrie_leaf_t old_leaf;
236  uword i;
237 
239 
240  for (i = 0; i < ARRAY_LEN (ply->leaves); i++)
241  {
242  old_leaf = ply->leaves[i];
243 
244  /* Recurse into sub plies. */
245  if (!ip4_fib_mtrie_leaf_is_terminal (old_leaf))
246  {
247  ip4_fib_mtrie_8_ply_t *sub_ply =
248  get_next_ply_for_leaf (m, old_leaf);
249  set_ply_with_more_specific_leaf (m, sub_ply, new_leaf,
250  new_leaf_dst_address_bits);
251  }
252 
253  /* Replace less specific terminal leaves with new leaf. */
254  else if (new_leaf_dst_address_bits >=
256  {
257  clib_atomic_store_rel_n (&ply->leaves[i], new_leaf);
258  ply->dst_address_bits_of_leaves[i] = new_leaf_dst_address_bits;
260  }
261  }
262 }
263 
264 static void
267  u32 old_ply_index, u32 dst_address_byte_index)
268 {
269  ip4_fib_mtrie_leaf_t old_leaf, new_leaf;
270  i32 n_dst_bits_next_plies;
271  u8 dst_byte;
272  ip4_fib_mtrie_8_ply_t *old_ply;
273 
274  old_ply = pool_elt_at_index (ip4_ply_pool, old_ply_index);
275 
276  ASSERT (a->dst_address_length <= 32);
277  ASSERT (dst_address_byte_index < ARRAY_LEN (a->dst_address.as_u8));
278 
279  /* how many bits of the destination address are in the next PLY */
280  n_dst_bits_next_plies =
281  a->dst_address_length - BITS (u8) * (dst_address_byte_index + 1);
282 
283  dst_byte = a->dst_address.as_u8[dst_address_byte_index];
284 
285  /* Number of bits next plies <= 0 => insert leaves this ply. */
286  if (n_dst_bits_next_plies <= 0)
287  {
288  /* The mask length of the address to insert maps to this ply */
289  uword old_leaf_is_terminal;
290  u32 i, n_dst_bits_this_ply;
291 
292  /* The number of bits, and hence slots/buckets, we will fill */
293  n_dst_bits_this_ply = clib_min (8, -n_dst_bits_next_plies);
294  ASSERT ((a->dst_address.as_u8[dst_address_byte_index] &
295  pow2_mask (n_dst_bits_this_ply)) == 0);
296 
297  /* Starting at the value of the byte at this section of the v4 address
298  * fill the buckets/slots of the ply */
299  for (i = dst_byte; i < dst_byte + (1 << n_dst_bits_this_ply); i++)
300  {
301  ip4_fib_mtrie_8_ply_t *new_ply;
302 
303  old_leaf = old_ply->leaves[i];
304  old_leaf_is_terminal = ip4_fib_mtrie_leaf_is_terminal (old_leaf);
305 
306  if (a->dst_address_length >= old_ply->dst_address_bits_of_leaves[i])
307  {
308  /* The new leaf is more or equally specific than the one currently
309  * occupying the slot */
311 
312  if (old_leaf_is_terminal)
313  {
314  /* The current leaf is terminal, we can replace it with
315  * the new one */
316  old_ply->n_non_empty_leafs -=
317  ip4_fib_mtrie_leaf_is_non_empty (old_ply, i);
318 
319  old_ply->dst_address_bits_of_leaves[i] =
321  clib_atomic_store_rel_n (&old_ply->leaves[i], new_leaf);
322 
323  old_ply->n_non_empty_leafs +=
324  ip4_fib_mtrie_leaf_is_non_empty (old_ply, i);
325  ASSERT (old_ply->n_non_empty_leafs <=
326  ARRAY_LEN (old_ply->leaves));
327  }
328  else
329  {
330  /* Existing leaf points to another ply. We need to place
331  * new_leaf into all more specific slots. */
332  new_ply = get_next_ply_for_leaf (m, old_leaf);
333  set_ply_with_more_specific_leaf (m, new_ply, new_leaf,
334  a->dst_address_length);
335  }
336  }
337  else if (!old_leaf_is_terminal)
338  {
339  /* The current leaf is less specific and not termial (i.e. a ply),
340  * recurse on down the trie */
341  new_ply = get_next_ply_for_leaf (m, old_leaf);
342  set_leaf (m, a, new_ply - ip4_ply_pool,
343  dst_address_byte_index + 1);
344  }
345  /*
346  * else
347  * the route we are adding is less specific than the leaf currently
348  * occupying this slot. leave it there
349  */
350  }
351  }
352  else
353  {
354  /* The address to insert requires us to move down at a lower level of
355  * the trie - recurse on down */
356  ip4_fib_mtrie_8_ply_t *new_ply;
357  u8 ply_base_len;
358 
359  ply_base_len = 8 * (dst_address_byte_index + 1);
360 
361  old_leaf = old_ply->leaves[dst_byte];
362 
363  if (ip4_fib_mtrie_leaf_is_terminal (old_leaf))
364  {
365  /* There is a leaf occupying the slot. Replace it with a new ply */
366  old_ply->n_non_empty_leafs -=
367  ip4_fib_mtrie_leaf_is_non_empty (old_ply, dst_byte);
368 
369  new_leaf =
370  ply_create (m, old_leaf,
371  old_ply->dst_address_bits_of_leaves[dst_byte],
372  ply_base_len);
373  new_ply = get_next_ply_for_leaf (m, new_leaf);
374 
375  /* Refetch since ply_create may move pool. */
376  old_ply = pool_elt_at_index (ip4_ply_pool, old_ply_index);
377 
378  clib_atomic_store_rel_n (&old_ply->leaves[dst_byte], new_leaf);
379  old_ply->dst_address_bits_of_leaves[dst_byte] = ply_base_len;
380 
381  old_ply->n_non_empty_leafs +=
382  ip4_fib_mtrie_leaf_is_non_empty (old_ply, dst_byte);
383  ASSERT (old_ply->n_non_empty_leafs >= 0);
384  }
385  else
386  new_ply = get_next_ply_for_leaf (m, old_leaf);
387 
388  set_leaf (m, a, new_ply - ip4_ply_pool, dst_address_byte_index + 1);
389  }
390 }
391 
392 static void
395 {
396  ip4_fib_mtrie_leaf_t old_leaf, new_leaf;
397  ip4_fib_mtrie_16_ply_t *old_ply;
398  i32 n_dst_bits_next_plies;
399  u16 dst_byte;
400 
401  old_ply = &m->root_ply;
402 
403  ASSERT (a->dst_address_length <= 32);
404 
405  /* how many bits of the destination address are in the next PLY */
406  n_dst_bits_next_plies = a->dst_address_length - BITS (u16);
407 
408  dst_byte = a->dst_address.as_u16[0];
409 
410  /* Number of bits next plies <= 0 => insert leaves this ply. */
411  if (n_dst_bits_next_plies <= 0)
412  {
413  /* The mask length of the address to insert maps to this ply */
414  uword old_leaf_is_terminal;
415  u32 i, n_dst_bits_this_ply;
416 
417  /* The number of bits, and hence slots/buckets, we will fill */
418  n_dst_bits_this_ply = 16 - a->dst_address_length;
419  ASSERT ((clib_host_to_net_u16 (a->dst_address.as_u16[0]) &
420  pow2_mask (n_dst_bits_this_ply)) == 0);
421 
422  /* Starting at the value of the byte at this section of the v4 address
423  * fill the buckets/slots of the ply */
424  for (i = 0; i < (1 << n_dst_bits_this_ply); i++)
425  {
426  ip4_fib_mtrie_8_ply_t *new_ply;
427  u16 slot;
428 
429  slot = clib_net_to_host_u16 (dst_byte);
430  slot += i;
431  slot = clib_host_to_net_u16 (slot);
432 
433  old_leaf = old_ply->leaves[slot];
434  old_leaf_is_terminal = ip4_fib_mtrie_leaf_is_terminal (old_leaf);
435 
436  if (a->dst_address_length >=
437  old_ply->dst_address_bits_of_leaves[slot])
438  {
439  /* The new leaf is more or equally specific than the one currently
440  * occupying the slot */
442 
443  if (old_leaf_is_terminal)
444  {
445  /* The current leaf is terminal, we can replace it with
446  * the new one */
447  old_ply->dst_address_bits_of_leaves[slot] =
449  clib_atomic_store_rel_n (&old_ply->leaves[slot], new_leaf);
450  }
451  else
452  {
453  /* Existing leaf points to another ply. We need to place
454  * new_leaf into all more specific slots. */
455  new_ply = get_next_ply_for_leaf (m, old_leaf);
456  set_ply_with_more_specific_leaf (m, new_ply, new_leaf,
457  a->dst_address_length);
458  }
459  }
460  else if (!old_leaf_is_terminal)
461  {
462  /* The current leaf is less specific and not termial (i.e. a ply),
463  * recurse on down the trie */
464  new_ply = get_next_ply_for_leaf (m, old_leaf);
465  set_leaf (m, a, new_ply - ip4_ply_pool, 2);
466  }
467  /*
468  * else
469  * the route we are adding is less specific than the leaf currently
470  * occupying this slot. leave it there
471  */
472  }
473  }
474  else
475  {
476  /* The address to insert requires us to move down at a lower level of
477  * the trie - recurse on down */
478  ip4_fib_mtrie_8_ply_t *new_ply;
479  u8 ply_base_len;
480 
481  ply_base_len = 16;
482 
483  old_leaf = old_ply->leaves[dst_byte];
484 
485  if (ip4_fib_mtrie_leaf_is_terminal (old_leaf))
486  {
487  /* There is a leaf occupying the slot. Replace it with a new ply */
488  new_leaf =
489  ply_create (m, old_leaf,
490  old_ply->dst_address_bits_of_leaves[dst_byte],
491  ply_base_len);
492  new_ply = get_next_ply_for_leaf (m, new_leaf);
493 
494  clib_atomic_store_rel_n (&old_ply->leaves[dst_byte], new_leaf);
495  old_ply->dst_address_bits_of_leaves[dst_byte] = ply_base_len;
496  }
497  else
498  new_ply = get_next_ply_for_leaf (m, old_leaf);
499 
500  set_leaf (m, a, new_ply - ip4_ply_pool, 2);
501  }
502 }
503 
504 static uword
507  ip4_fib_mtrie_8_ply_t * old_ply, u32 dst_address_byte_index)
508 {
509  ip4_fib_mtrie_leaf_t old_leaf, del_leaf;
510  i32 n_dst_bits_next_plies;
511  i32 i, n_dst_bits_this_ply, old_leaf_is_terminal;
512  u8 dst_byte;
513 
514  ASSERT (a->dst_address_length <= 32);
515  ASSERT (dst_address_byte_index < ARRAY_LEN (a->dst_address.as_u8));
516 
517  n_dst_bits_next_plies =
518  a->dst_address_length - BITS (u8) * (dst_address_byte_index + 1);
519 
520  dst_byte = a->dst_address.as_u8[dst_address_byte_index];
521  if (n_dst_bits_next_plies < 0)
522  dst_byte &= ~pow2_mask (-n_dst_bits_next_plies);
523 
524  n_dst_bits_this_ply =
525  n_dst_bits_next_plies <= 0 ? -n_dst_bits_next_plies : 0;
526  n_dst_bits_this_ply = clib_min (8, n_dst_bits_this_ply);
527 
529 
530  for (i = dst_byte; i < dst_byte + (1 << n_dst_bits_this_ply); i++)
531  {
532  old_leaf = old_ply->leaves[i];
533  old_leaf_is_terminal = ip4_fib_mtrie_leaf_is_terminal (old_leaf);
534 
535  if (old_leaf == del_leaf
536  || (!old_leaf_is_terminal
537  && unset_leaf (m, a, get_next_ply_for_leaf (m, old_leaf),
538  dst_address_byte_index + 1)))
539  {
540  old_ply->n_non_empty_leafs -=
541  ip4_fib_mtrie_leaf_is_non_empty (old_ply, i);
542 
543  clib_atomic_store_rel_n (&old_ply->leaves[i],
545  (a->cover_adj_index));
547 
548  old_ply->n_non_empty_leafs +=
549  ip4_fib_mtrie_leaf_is_non_empty (old_ply, i);
550 
551  ASSERT (old_ply->n_non_empty_leafs >= 0);
552  if (old_ply->n_non_empty_leafs == 0 && dst_address_byte_index > 0)
553  {
554  pool_put (ip4_ply_pool, old_ply);
555  /* Old ply was deleted. */
556  return 1;
557  }
558 #if CLIB_DEBUG > 0
559  else if (dst_address_byte_index)
560  {
561  int ii, count = 0;
562  for (ii = 0; ii < ARRAY_LEN (old_ply->leaves); ii++)
563  {
564  count += ip4_fib_mtrie_leaf_is_non_empty (old_ply, ii);
565  }
566  ASSERT (count);
567  }
568 #endif
569  }
570  }
571 
572  /* Old ply was not deleted. */
573  return 0;
574 }
575 
576 static void
579 {
580  ip4_fib_mtrie_leaf_t old_leaf, del_leaf;
581  i32 n_dst_bits_next_plies;
582  i32 i, n_dst_bits_this_ply, old_leaf_is_terminal;
583  u16 dst_byte;
584  ip4_fib_mtrie_16_ply_t *old_ply;
585 
586  ASSERT (a->dst_address_length <= 32);
587 
588  old_ply = &m->root_ply;
589  n_dst_bits_next_plies = a->dst_address_length - BITS (u16);
590 
591  dst_byte = a->dst_address.as_u16[0];
592 
593  n_dst_bits_this_ply = (n_dst_bits_next_plies <= 0 ?
594  (16 - a->dst_address_length) : 0);
595 
597 
598  /* Starting at the value of the byte at this section of the v4 address
599  * fill the buckets/slots of the ply */
600  for (i = 0; i < (1 << n_dst_bits_this_ply); i++)
601  {
602  u16 slot;
603 
604  slot = clib_net_to_host_u16 (dst_byte);
605  slot += i;
606  slot = clib_host_to_net_u16 (slot);
607 
608  old_leaf = old_ply->leaves[slot];
609  old_leaf_is_terminal = ip4_fib_mtrie_leaf_is_terminal (old_leaf);
610 
611  if (old_leaf == del_leaf
612  || (!old_leaf_is_terminal
613  && unset_leaf (m, a, get_next_ply_for_leaf (m, old_leaf), 2)))
614  {
615  clib_atomic_store_rel_n (&old_ply->leaves[slot],
617  (a->cover_adj_index));
619  }
620  }
621 }
622 
623 void
625  const ip4_address_t * dst_address,
626  u32 dst_address_length, u32 adj_index)
627 {
629  ip4_main_t *im = &ip4_main;
630 
631  /* Honor dst_address_length. Fib masks are in network byte order */
632  a.dst_address.as_u32 = (dst_address->as_u32 &
633  im->fib_masks[dst_address_length]);
634  a.dst_address_length = dst_address_length;
635  a.adj_index = adj_index;
636 
637  set_root_leaf (m, &a);
638 }
639 
640 void
642  const ip4_address_t * dst_address,
643  u32 dst_address_length,
644  u32 adj_index,
645  u32 cover_address_length, u32 cover_adj_index)
646 {
648  ip4_main_t *im = &ip4_main;
649 
650  /* Honor dst_address_length. Fib masks are in network byte order */
651  a.dst_address.as_u32 = (dst_address->as_u32 &
652  im->fib_masks[dst_address_length]);
653  a.dst_address_length = dst_address_length;
654  a.adj_index = adj_index;
655  a.cover_adj_index = cover_adj_index;
656  a.cover_address_length = cover_address_length;
657 
658  /* the top level ply is never removed */
659  unset_root_leaf (m, &a);
660 }
661 
662 /* Returns number of bytes of memory used by mtrie. */
663 static uword
665 {
666  uword bytes, i;
667 
668  bytes = sizeof (p[0]);
669  for (i = 0; i < ARRAY_LEN (p->leaves); i++)
670  {
671  ip4_fib_mtrie_leaf_t l = p->leaves[i];
673  bytes += mtrie_ply_memory_usage (m, get_next_ply_for_leaf (m, l));
674  }
675 
676  return bytes;
677 }
678 
679 /* Returns number of bytes of memory used by mtrie. */
680 uword
682 {
683  uword bytes, i;
684 
685  bytes = sizeof (*m);
686  for (i = 0; i < ARRAY_LEN (m->root_ply.leaves); i++)
687  {
690  bytes += mtrie_ply_memory_usage (m, get_next_ply_for_leaf (m, l));
691  }
692 
693  return bytes;
694 }
695 
696 static u8 *
697 format_ip4_fib_mtrie_leaf (u8 * s, va_list * va)
698 {
699  ip4_fib_mtrie_leaf_t l = va_arg (*va, ip4_fib_mtrie_leaf_t);
700 
702  s = format (s, "lb-index %d", ip4_fib_mtrie_leaf_get_adj_index (l));
703  else
704  s = format (s, "next ply %d", ip4_fib_mtrie_leaf_get_next_ply_index (l));
705  return s;
706 }
707 
708 #define FORMAT_PLY(s, _p, _a, _i, _base_address, _ply_max_len, _indent) \
709 ({ \
710  u32 a, ia_length; \
711  ip4_address_t ia; \
712  ip4_fib_mtrie_leaf_t _l = p->leaves[(_i)]; \
713  \
714  a = (_base_address) + ((_a) << (32 - (_ply_max_len))); \
715  ia.as_u32 = clib_host_to_net_u32 (a); \
716  ia_length = (_p)->dst_address_bits_of_leaves[(_i)]; \
717  s = format (s, "\n%U%U %U", \
718  format_white_space, (_indent) + 4, \
719  format_ip4_address_and_length, &ia, ia_length, \
720  format_ip4_fib_mtrie_leaf, _l); \
721  \
722  if (ip4_fib_mtrie_leaf_is_next_ply (_l)) \
723  s = format (s, "\n%U", \
724  format_ip4_fib_mtrie_ply, m, a, (_indent) + 8, \
725  ip4_fib_mtrie_leaf_get_next_ply_index (_l)); \
726  s; \
727 })
728 
729 static u8 *
730 format_ip4_fib_mtrie_ply (u8 * s, va_list * va)
731 {
732  ip4_fib_mtrie_t *m = va_arg (*va, ip4_fib_mtrie_t *);
733  u32 base_address = va_arg (*va, u32);
734  u32 indent = va_arg (*va, u32);
735  u32 ply_index = va_arg (*va, u32);
737  int i;
738 
739  p = pool_elt_at_index (ip4_ply_pool, ply_index);
740  s = format (s, "%Uply index %d, %d non-empty leaves",
741  format_white_space, indent, ply_index, p->n_non_empty_leafs);
742 
743  for (i = 0; i < ARRAY_LEN (p->leaves); i++)
744  {
746  {
747  s = FORMAT_PLY (s, p, i, i, base_address,
748  p->dst_address_bits_base + 8, indent);
749  }
750  }
751 
752  return s;
753 }
754 
755 u8 *
756 format_ip4_fib_mtrie (u8 * s, va_list * va)
757 {
758  ip4_fib_mtrie_t *m = va_arg (*va, ip4_fib_mtrie_t *);
759  int verbose = va_arg (*va, int);
761  u32 base_address = 0;
762  int i;
763 
764  s = format (s, "%d plies, memory usage %U\n",
765  pool_elts (ip4_ply_pool),
767  s = format (s, "root-ply");
768  p = &m->root_ply;
769 
770  if (verbose)
771  {
772  s = format (s, "root-ply");
773  p = &m->root_ply;
774 
775  for (i = 0; i < ARRAY_LEN (p->leaves); i++)
776  {
777  u16 slot;
778 
779  slot = clib_host_to_net_u16 (i);
780 
781  if (p->dst_address_bits_of_leaves[slot] > 0)
782  {
783  s = FORMAT_PLY (s, p, i, slot, base_address, 16, 0);
784  }
785  }
786  }
787 
788  return s;
789 }
790 
791 /** Default heap size for the IPv4 mtries */
792 #define IP4_FIB_DEFAULT_MTRIE_HEAP_SIZE (32<<20)
793 #ifndef MAP_HUGE_SHIFT
794 #define MAP_HUGE_SHIFT 26
795 #endif
796 
797 static clib_error_t *
799 {
801  ip4_main_t *im = &ip4_main;
802  clib_error_t *error = NULL;
803  uword *old_heap;
804 
805  if (im->mtrie_heap_size == 0)
807 
808 again:
809  if (im->mtrie_hugetlb)
810  {
811  void *rv;
812  int mmap_flags, mmap_flags_huge;
813  uword htlb_pagesize = clib_mem_get_default_hugepage_size ();
814  if (htlb_pagesize == 0)
815  {
816  clib_warning ("WARNING: htlb pagesize == 0");
817  im->mtrie_hugetlb = 0;
818  goto again;
819  }
820  /* Round the allocation request to an even number of huge pages */
821  im->mtrie_heap_size = (im->mtrie_heap_size + (htlb_pagesize - 1)) &
822  ~(htlb_pagesize - 1);
823  mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS;
824  mmap_flags_huge = (mmap_flags | MAP_HUGETLB | MAP_LOCKED |
825  min_log2 (htlb_pagesize) << MAP_HUGE_SHIFT);
826  rv = mmap (0, im->mtrie_heap_size,
827  PROT_READ | PROT_WRITE, mmap_flags_huge, -1, 0);
828  if (rv == MAP_FAILED)
829  {
830  /* Failure when running as root should be logged... */
831  if (geteuid () == 0)
832  clib_warning ("ip4 mtrie htlb map failed: not enough pages?");
833  im->mtrie_hugetlb = 0;
834  goto again;
835  }
836  if (mlock (rv, im->mtrie_heap_size))
837  clib_warning ("WARNING: couldn't lock mtrie heap at %llx", rv);
839  1 /* locked */ );
840  }
841  else
842  {
843  im->mtrie_mheap = create_mspace (im->mtrie_heap_size, 1 /* locked */ );
844  }
845 
846  /* Burn one ply so index 0 is taken */
848  pool_get (ip4_ply_pool, p);
849  clib_mem_set_heap (old_heap);
850 
851  return (error);
852 }
853 
855 
856 /*
857  * fd.io coding-style-patch-verification: ON
858  *
859  * Local Variables:
860  * eval: (c-set-style "gnu")
861  * End:
862  */
u8 count
Definition: dhcp.api:208
int mtrie_hugetlb
Use hugetlb pages for the Mtries.
Definition: ip4.h:171
static ip4_fib_mtrie_8_ply_t * get_next_ply_for_leaf(ip4_fib_mtrie_t *m, ip4_fib_mtrie_leaf_t l)
Definition: ip4_mtrie.c:191
#define clib_min(x, y)
Definition: clib.h:327
#define CLIB_UNUSED(x)
Definition: clib.h:87
static ip4_fib_mtrie_leaf_t ply_create(ip4_fib_mtrie_t *m, ip4_fib_mtrie_leaf_t init_leaf, u32 leaf_prefix_len, u32 ply_base_len)
Definition: ip4_mtrie.c:174
a
Definition: bitmap.h:538
The mutiway-TRIE.
Definition: ip4_mtrie.h:129
#define PLY_INIT_LEAVES(p)
Definition: ip4_mtrie.c:126
static void set_leaf(ip4_fib_mtrie_t *m, const ip4_fib_mtrie_set_unset_leaf_args_t *a, u32 old_ply_index, u32 dst_address_byte_index)
Definition: ip4_mtrie.c:265
ip4_fib_mtrie_8_ply_t * ip4_ply_pool
Global pool of IPv4 8bit PLYs.
Definition: ip4_mtrie.c:48
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
ip4_fib_mtrie_leaf_t leaves[256]
Definition: ip4_mtrie.h:93
static void unset_root_leaf(ip4_fib_mtrie_t *m, const ip4_fib_mtrie_set_unset_leaf_args_t *a)
Definition: ip4_mtrie.c:577
vlib_main_t * vm
Definition: in2out_ed.c:1582
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static u32 ip4_fib_mtrie_leaf_is_non_empty(ip4_fib_mtrie_8_ply_t *p, u8 dst_byte)
Definition: ip4_mtrie.c:51
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:252
unsigned char u8
Definition: types.h:56
u8 dst_address_bits_of_leaves[PLY_16_SIZE]
Prefix length for terminal leaves.
Definition: ip4_mtrie.h:80
static uword min_log2(uword x)
Definition: clib.h:161
#define IP4_FIB_MTRIE_LEAF_EMPTY
Definition: ip4_mtrie.h:54
One ply of the 4 ply mtrie fib.
Definition: ip4_mtrie.h:86
DLMALLOC_EXPORT mspace create_mspace_with_base(void *base, size_t capacity, int locked)
static void set_ply_with_more_specific_leaf(ip4_fib_mtrie_t *m, ip4_fib_mtrie_8_ply_t *ply, ip4_fib_mtrie_leaf_t new_leaf, uword new_leaf_dst_address_bits)
Definition: ip4_mtrie.c:230
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
#define FORMAT_PLY(s, _p, _a, _i, _base_address, _ply_max_len, _indent)
Definition: ip4_mtrie.c:708
static uword pow2_mask(uword x)
Definition: clib.h:237
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
void ip4_mtrie_free(ip4_fib_mtrie_t *m)
Free an mtrie, It must be emty when free&#39;d.
Definition: ip4_mtrie.c:199
u32 ip4_fib_mtrie_leaf_t
Definition: ip4_mtrie.h:52
u8 * format_memory_size(u8 *s, va_list *va)
Definition: std-formats.c:209
ip4_fib_mtrie_leaf_t leaves[PLY_16_SIZE]
Definition: ip4_mtrie.h:70
unsigned int u32
Definition: types.h:88
#define IP4_FIB_DEFAULT_MTRIE_HEAP_SIZE
Default heap size for the IPv4 mtries.
Definition: ip4_mtrie.c:792
static u32 ip4_fib_mtrie_leaf_get_adj_index(ip4_fib_mtrie_leaf_t n)
From the stored slot value extract the LB index value.
Definition: ip4_mtrie.h:192
void ip4_mtrie_init(ip4_fib_mtrie_t *m)
Initialise an mtrie.
Definition: ip4_mtrie.c:215
static u8 * format_ip4_fib_mtrie_leaf(u8 *s, va_list *va)
Definition: ip4_mtrie.c:697
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:534
u16 as_u16[2]
Definition: ip4_packet.h:56
u8 * format_ip4_fib_mtrie(u8 *s, va_list *va)
Definition: ip4_mtrie.c:756
uword mtrie_heap_size
Heapsize for the Mtries.
Definition: ip4.h:168
unsigned short u16
Definition: types.h:57
static void set_root_leaf(ip4_fib_mtrie_t *m, const ip4_fib_mtrie_set_unset_leaf_args_t *a)
Definition: ip4_mtrie.c:393
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:302
static uword mtrie_ply_memory_usage(ip4_fib_mtrie_t *m, ip4_fib_mtrie_8_ply_t *p)
Definition: ip4_mtrie.c:664
#define always_inline
Definition: ipsec.h:28
#define MAP_HUGE_SHIFT
Definition: ip4_mtrie.c:794
void * mtrie_mheap
The memory heap for the mtries.
Definition: ip4.h:174
#define pool_get_aligned(P, E, A)
Allocate an object E from a pool P with alignment A.
Definition: pool.h:246
u8 dst_address_bits_of_leaves[256]
Prefix length for leaves/ply.
Definition: ip4_mtrie.h:103
u8 slot
Definition: pci_types.api:22
DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked)
uword ip4_fib_mtrie_memory_usage(ip4_fib_mtrie_t *m)
return the memory used by the table
Definition: ip4_mtrie.c:681
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
ip4_fib_mtrie_16_ply_t root_ply
Embed the PLY with the mtrie struct.
Definition: ip4_mtrie.h:136
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:268
#define clib_warning(format, args...)
Definition: error.h:59
#define ARRAY_LEN(x)
Definition: clib.h:67
#define PLY_INIT(p, init, prefix_len, ply_base_len)
Definition: ip4_mtrie.c:140
signed int i32
Definition: types.h:77
static clib_error_t * ip4_mtrie_module_init(vlib_main_t *vm)
Definition: ip4_mtrie.c:798
#define ASSERT(truth)
IPv4 main type.
Definition: ip4.h:106
i32 n_non_empty_leafs
Number of non-empty leafs (whether terminal or not).
Definition: ip4_mtrie.h:108
uword clib_mem_get_default_hugepage_size(void)
Definition: mem.c:57
static void init(void)
Definition: client.c:105
static u32 ip4_fib_mtrie_leaf_is_next_ply(ip4_fib_mtrie_leaf_t n)
Definition: ip4_mtrie.c:73
#define clib_atomic_store_rel_n(a, b)
Definition: atomics.h:49
i32 dst_address_bits_base
The length of the ply&#39;s covering prefix.
Definition: ip4_mtrie.h:115
void ip4_fib_mtrie_route_add(ip4_fib_mtrie_t *m, const ip4_address_t *dst_address, u32 dst_address_length, u32 adj_index)
Add a route/entry to the mtrie.
Definition: ip4_mtrie.c:624
u64 uword
Definition: types.h:112
static ip4_fib_mtrie_leaf_t ip4_fib_mtrie_leaf_set_adj_index(u32 adj_index)
Definition: ip4_mtrie.c:64
ip4_main_t ip4_main
Global ip4 main structure.
Definition: ip4_forward.c:1144
static u32 ip4_fib_mtrie_leaf_get_next_ply_index(ip4_fib_mtrie_leaf_t n)
Definition: ip4_mtrie.c:79
void ip4_fib_mtrie_route_del(ip4_fib_mtrie_t *m, const ip4_address_t *dst_address, u32 dst_address_length, u32 adj_index, u32 cover_address_length, u32 cover_adj_index)
remove a route/entry to the mtrie
Definition: ip4_mtrie.c:641
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static uword unset_leaf(ip4_fib_mtrie_t *m, const ip4_fib_mtrie_set_unset_leaf_args_t *a, ip4_fib_mtrie_8_ply_t *old_ply, u32 dst_address_byte_index)
Definition: ip4_mtrie.c:505
#define BITS(x)
Definition: clib.h:66
static u8 * format_ip4_fib_mtrie_ply(u8 *s, va_list *va)
Definition: ip4_mtrie.c:730
static void ply_16_init(ip4_fib_mtrie_16_ply_t *p, ip4_fib_mtrie_leaf_t init, uword prefix_len)
Definition: ip4_mtrie.c:165
static u32 ip4_fib_mtrie_leaf_is_terminal(ip4_fib_mtrie_leaf_t n)
Is the leaf terminal (i.e.
Definition: ip4_mtrie.h:183
static void ply_8_init(ip4_fib_mtrie_8_ply_t *p, ip4_fib_mtrie_leaf_t init, uword prefix_len, u32 ply_base_len)
Definition: ip4_mtrie.c:158
u32 fib_masks[33]
Definition: ip4.h:119
static ip4_fib_mtrie_leaf_t ip4_fib_mtrie_leaf_set_next_ply_index(u32 i)
Definition: ip4_mtrie.c:86
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128