FD.io VPP  v19.04-6-g6f05f72
Vector Packet Processing
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * main.c: main vector processing loop
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <math.h>
41 #include <vppinfra/format.h>
42 #include <vlib/vlib.h>
43 #include <vlib/threads.h>
45 
46 #include <vlib/unix/unix.h>
47 #include <vlib/unix/cj.h>
48 
50 
51 /* Actually allocate a few extra slots of vector data to support
52  speculative vector enqueues which overflow vector data in next frame. */
53 #define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4)
54 
56 
58 vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes)
59 {
60  u32 n_bytes;
61 
62  /* Make room for vlib_frame_t plus scalar arguments. */
63  n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes);
64 
65  /* Make room for vector arguments.
66  Allocate a few extra slots of vector data to support
67  speculative vector enqueues which overflow vector data in next frame. */
68 #define VLIB_FRAME_SIZE_EXTRA 4
69  n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes;
70 
71  /* Magic number is first 32bit number after vector data.
72  Used to make sure that vector data is never overrun. */
73 #define VLIB_FRAME_MAGIC (0xabadc0ed)
74  n_bytes += sizeof (u32);
75 
76  /* Pad to cache line. */
77  n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES);
78 
79  return n_bytes;
80 }
81 
84 {
85  void *p = f;
86 
88 
90 
91  return p;
92 }
93 
94 static inline vlib_frame_size_t *
96  u32 n_scalar_bytes, u32 n_vector_bytes)
97 {
98 #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
99  uword key = (n_scalar_bytes << 16) | n_vector_bytes;
100  uword *p, i;
101 
102  p = hash_get (nm->frame_size_hash, key);
103  if (p)
104  i = p[0];
105  else
106  {
107  i = vec_len (nm->frame_sizes);
108  vec_validate (nm->frame_sizes, i);
109  hash_set (nm->frame_size_hash, key, i);
110  }
111 
112  return vec_elt_at_index (nm->frame_sizes, i);
113 #else
114  ASSERT (vlib_frame_bytes (n_scalar_bytes, n_vector_bytes)
115  == (vlib_frame_bytes (0, 4)));
116  return vec_elt_at_index (nm->frame_sizes, 0);
117 #endif
118 }
119 
120 static u32
122  u32 frame_flags)
123 {
124  vlib_node_main_t *nm = &vm->node_main;
125  vlib_frame_size_t *fs;
126  vlib_node_t *to_node;
127  vlib_frame_t *f;
128  u32 fi, l, n, scalar_size, vector_size;
129 
130  to_node = vlib_get_node (vm, to_node_index);
131 
132  scalar_size = to_node->scalar_size;
133  vector_size = to_node->vector_size;
134 
135  fs = get_frame_size_info (nm, scalar_size, vector_size);
136  n = vlib_frame_bytes (scalar_size, vector_size);
137  if ((l = vec_len (fs->free_frame_indices)) > 0)
138  {
139  /* Allocate from end of free list. */
140  fi = fs->free_frame_indices[l - 1];
141  f = vlib_get_frame_no_check (vm, fi);
142  _vec_len (fs->free_frame_indices) = l - 1;
143  }
144  else
145  {
147  fi = vlib_frame_index_no_check (vm, f);
148  }
149 
150  /* Poison frame when debugging. */
151  if (CLIB_DEBUG > 0)
152  clib_memset (f, 0xfe, n);
153 
154  /* Insert magic number. */
155  {
156  u32 *magic;
157 
158  magic = vlib_frame_find_magic (f, to_node);
159  *magic = VLIB_FRAME_MAGIC;
160  }
161 
162  f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
163  f->n_vectors = 0;
164  f->scalar_size = scalar_size;
165  f->vector_size = vector_size;
166  f->flags = 0;
167 
168  fs->n_alloc_frames += 1;
169 
170  return fi;
171 }
172 
173 /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX.
174  Returns frame index. */
175 static u32
177  u32 to_next_index)
178 {
179  vlib_node_t *from_node;
180 
181  from_node = vlib_get_node (vm, from_node_runtime->node_index);
182  ASSERT (to_next_index < vec_len (from_node->next_nodes));
183 
184  return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index],
185  /* frame_flags */ 0);
186 }
187 
188 vlib_frame_t *
190 {
191  u32 fi = vlib_frame_alloc_to_node (vm, to_node_index,
192  /* frame_flags */
194  return vlib_get_frame (vm, fi);
195 }
196 
197 void
199 {
201  vlib_node_t *to_node;
202 
203  if (f->n_vectors == 0)
204  return;
205 
206  to_node = vlib_get_node (vm, to_node_index);
207 
208  vec_add2 (vm->node_main.pending_frames, p, 1);
209 
211  p->frame_index = vlib_frame_index (vm, f);
212  p->node_runtime_index = to_node->runtime_index;
214 }
215 
216 /* Free given frame. */
217 void
219 {
220  vlib_node_main_t *nm = &vm->node_main;
221  vlib_node_t *node;
222  vlib_frame_size_t *fs;
223  u32 frame_index;
224 
226 
227  node = vlib_get_node (vm, r->node_index);
228  fs = get_frame_size_info (nm, node->scalar_size, node->vector_size);
229 
230  frame_index = vlib_frame_index (vm, f);
231 
233 
234  /* No next frames may point to freed frame. */
235  if (CLIB_DEBUG > 0)
236  {
237  vlib_next_frame_t *nf;
239  ASSERT (nf->frame_index != frame_index);
240  }
241 
243 
244  vec_add1 (fs->free_frame_indices, frame_index);
245  ASSERT (fs->n_alloc_frames > 0);
246  fs->n_alloc_frames -= 1;
247 }
248 
249 static clib_error_t *
251  unformat_input_t * input, vlib_cli_command_t * cmd)
252 {
253  vlib_node_main_t *nm = &vm->node_main;
254  vlib_frame_size_t *fs;
255 
256  vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free");
257  vec_foreach (fs, nm->frame_sizes)
258  {
259  u32 n_alloc = fs->n_alloc_frames;
260  u32 n_free = vec_len (fs->free_frame_indices);
261 
262  if (n_alloc + n_free > 0)
263  vlib_cli_output (vm, "%=6d%=12d%=12d",
264  fs - nm->frame_sizes, n_alloc, n_free);
265  }
266 
267  return 0;
268 }
269 
270 /* *INDENT-OFF* */
271 VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
272  .path = "show vlib frame-allocation",
273  .short_help = "Show node dispatch frame statistics",
274  .function = show_frame_stats,
275 };
276 /* *INDENT-ON* */
277 
278 /* Change ownership of enqueue rights to given next node. */
279 static void
281  vlib_node_runtime_t * node_runtime,
282  u32 next_index)
283 {
284  vlib_node_main_t *nm = &vm->node_main;
285  vlib_next_frame_t *next_frame;
286  vlib_node_t *node, *next_node;
287 
288  node = vec_elt (nm->nodes, node_runtime->node_index);
289 
290  /* Only internal & input nodes are allowed to call other nodes. */
292  || node->type == VLIB_NODE_TYPE_INPUT
293  || node->type == VLIB_NODE_TYPE_PROCESS);
294 
295  ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes);
296 
297  next_frame =
298  vlib_node_runtime_get_next_frame (vm, node_runtime, next_index);
299  next_node = vec_elt (nm->nodes, node->next_nodes[next_index]);
300 
301  if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX)
302  {
303  /* Get frame from previous owner. */
304  vlib_next_frame_t *owner_next_frame;
305  vlib_next_frame_t tmp;
306 
307  owner_next_frame =
309  next_node->owner_node_index,
310  next_node->owner_next_index);
311 
312  /* Swap target next frame with owner's. */
313  tmp = owner_next_frame[0];
314  owner_next_frame[0] = next_frame[0];
315  next_frame[0] = tmp;
316 
317  /*
318  * If next_frame is already pending, we have to track down
319  * all pending frames and fix their next_frame_index fields.
320  */
321  if (next_frame->flags & VLIB_FRAME_PENDING)
322  {
324  if (next_frame->frame_index != ~0)
325  {
326  vec_foreach (p, nm->pending_frames)
327  {
328  if (p->frame_index == next_frame->frame_index)
329  {
330  p->next_frame_index =
331  next_frame - vm->node_main.next_frames;
332  }
333  }
334  }
335  }
336  }
337  else
338  {
339  /* No previous owner. Take ownership. */
340  next_frame->flags |= VLIB_FRAME_OWNER;
341  }
342 
343  /* Record new owner. */
344  next_node->owner_node_index = node->index;
345  next_node->owner_next_index = next_index;
346 
347  /* Now we should be owner. */
348  ASSERT (next_frame->flags & VLIB_FRAME_OWNER);
349 }
350 
351 /* Make sure that magic number is still there.
352  Otherwise, it is likely that caller has overrun frame arguments. */
353 always_inline void
355  vlib_frame_t * f, vlib_node_t * n, uword next_index)
356 {
357  vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]);
358  u32 *magic = vlib_frame_find_magic (f, next_node);
359  ASSERT (VLIB_FRAME_MAGIC == magic[0]);
360 }
361 
362 vlib_frame_t *
364  vlib_node_runtime_t * node,
365  u32 next_index, u32 allocate_new_next_frame)
366 {
367  vlib_frame_t *f;
368  vlib_next_frame_t *nf;
369  u32 n_used;
370 
371  nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
372 
373  /* Make sure this next frame owns right to enqueue to destination frame. */
374  if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER)))
375  vlib_next_frame_change_ownership (vm, node, next_index);
376 
377  /* ??? Don't need valid flag: can use frame_index == ~0 */
379  {
380  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
382  }
383 
384  f = vlib_get_frame (vm, nf->frame_index);
385 
386  /* Has frame been removed from pending vector (e.g. finished dispatching)?
387  If so we can reuse frame. */
388  if ((nf->flags & VLIB_FRAME_PENDING)
389  && !(f->frame_flags & VLIB_FRAME_PENDING))
390  {
391  nf->flags &= ~VLIB_FRAME_PENDING;
392  f->n_vectors = 0;
393  f->flags = 0;
394  }
395 
396  /* Allocate new frame if current one is marked as no-append or
397  it is already full. */
398  n_used = f->n_vectors;
399  if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0) ||
401  {
402  /* Old frame may need to be freed after dispatch, since we'll have
403  two redundant frames from node -> next node. */
405  {
406  vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame_index);
408  }
409 
410  /* Allocate new frame to replace full one. */
411  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
412  f = vlib_get_frame (vm, nf->frame_index);
413  n_used = f->n_vectors;
414  }
415 
416  /* Should have free vectors in frame now. */
417  ASSERT (n_used < VLIB_FRAME_SIZE);
418 
419  if (CLIB_DEBUG > 0)
420  {
421  validate_frame_magic (vm, f,
422  vlib_get_node (vm, node->node_index), next_index);
423  }
424 
425  return f;
426 }
427 
428 static void
430  vlib_node_runtime_t * rt,
431  u32 next_index, u32 n_vectors_left)
432 {
433  vlib_node_main_t *nm = &vm->node_main;
434  vlib_next_frame_t *nf;
435  vlib_frame_t *f;
436  vlib_node_runtime_t *next_rt;
437  vlib_node_t *next_node;
438  u32 n_before, n_after;
439 
440  nf = vlib_node_runtime_get_next_frame (vm, rt, next_index);
441  f = vlib_get_frame (vm, nf->frame_index);
442 
443  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
444  n_after = VLIB_FRAME_SIZE - n_vectors_left;
445  n_before = f->n_vectors;
446 
447  ASSERT (n_after >= n_before);
448 
450  nf->node_runtime_index);
451  next_node = vlib_get_node (vm, next_rt->node_index);
452  if (n_after > 0 && next_node->validate_frame)
453  {
454  u8 *msg = next_node->validate_frame (vm, rt, f);
455  if (msg)
456  {
457  clib_warning ("%v", msg);
458  ASSERT (0);
459  }
460  vec_free (msg);
461  }
462 }
463 
464 void
467  u32 next_index, u32 n_vectors_left)
468 {
469  vlib_node_main_t *nm = &vm->node_main;
470  vlib_next_frame_t *nf;
471  vlib_frame_t *f;
472  u32 n_vectors_in_frame;
473 
474  if (CLIB_DEBUG > 0)
475  vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
476 
477  nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
478  f = vlib_get_frame (vm, nf->frame_index);
479 
480  /* Make sure that magic number is still there. Otherwise, caller
481  has overrun frame meta data. */
482  if (CLIB_DEBUG > 0)
483  {
484  vlib_node_t *node = vlib_get_node (vm, r->node_index);
485  validate_frame_magic (vm, f, node, next_index);
486  }
487 
488  /* Convert # of vectors left -> number of vectors there. */
489  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
490  n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left;
491 
492  f->n_vectors = n_vectors_in_frame;
493 
494  /* If vectors were added to frame, add to pending vector. */
495  if (PREDICT_TRUE (n_vectors_in_frame > 0))
496  {
498  u32 v0, v1;
499 
500  r->cached_next_index = next_index;
501 
502  if (!(f->frame_flags & VLIB_FRAME_PENDING))
503  {
504  __attribute__ ((unused)) vlib_node_t *node;
505  vlib_node_t *next_node;
506  vlib_node_runtime_t *next_runtime;
507 
508  node = vlib_get_node (vm, r->node_index);
509  next_node = vlib_get_next_node (vm, r->node_index, next_index);
510  next_runtime = vlib_node_get_runtime (vm, next_node->index);
511 
512  vec_add2 (nm->pending_frames, p, 1);
513 
514  p->frame_index = nf->frame_index;
516  p->next_frame_index = nf - nm->next_frames;
517  nf->flags |= VLIB_FRAME_PENDING;
519 
520  /*
521  * If we're going to dispatch this frame on another thread,
522  * force allocation of a new frame. Otherwise, we create
523  * a dangling frame reference. Each thread has its own copy of
524  * the next_frames vector.
525  */
526  if (0 && r->thread_index != next_runtime->thread_index)
527  {
528  nf->frame_index = ~0;
530  }
531  }
532 
533  /* Copy trace flag from next_frame and from runtime. */
534  nf->flags |=
535  (nf->flags & VLIB_NODE_FLAG_TRACE) | (r->
537 
539  v1 = v0 + n_vectors_in_frame;
541  if (PREDICT_FALSE (v1 < v0))
542  {
543  vlib_node_t *node = vlib_get_node (vm, r->node_index);
544  vec_elt (node->n_vectors_by_next_node, next_index) += v0;
545  }
546  }
547 }
548 
549 /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */
550 never_inline void
553  uword n_calls, uword n_vectors, uword n_clocks,
554  uword n_ticks0, uword n_ticks1)
555 {
556  vlib_node_t *n = vlib_get_node (vm, r->node_index);
557 
558  n->stats_total.calls += n_calls + r->calls_since_last_overflow;
559  n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
560  n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
561  n->stats_total.perf_counter0_ticks += n_ticks0 +
563  n->stats_total.perf_counter1_ticks += n_ticks1 +
565  n->stats_total.perf_counter_vectors += n_vectors +
569 
576 }
577 
578 always_inline void __attribute__ ((unused))
580  vlib_process_t * p,
581  uword n_calls, uword n_vectors, uword n_clocks,
582  uword n_ticks0, uword n_ticks1)
583 {
584  vlib_node_runtime_t *rt = &p->node_runtime;
586  vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks,
587  n_ticks0, n_ticks1);
588  n->stats_total.suspends += p->n_suspends;
589  p->n_suspends = 0;
590 }
591 
592 void
594 {
596 
597  if (n->type == VLIB_NODE_TYPE_PROCESS)
598  {
599  /* Nothing to do for PROCESS nodes except in main thread */
600  if (vm != &vlib_global_main)
601  return;
602 
605  p->n_suspends = 0;
606  rt = &p->node_runtime;
607  }
608  else
609  rt =
611  n->runtime_index);
612 
613  vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0, 0, 0);
614 
615  /* Sync up runtime next frame vector counters with main node structure. */
616  {
617  vlib_next_frame_t *nf;
618  uword i;
619  for (i = 0; i < rt->n_next_nodes; i++)
620  {
621  nf = vlib_node_runtime_get_next_frame (vm, rt, i);
625  }
626  }
627 }
628 
631  vlib_node_runtime_t * node,
632  uword n_calls,
633  uword n_vectors, uword n_clocks,
634  uword n_ticks0, uword n_ticks1)
635 {
636  u32 ca0, ca1, v0, v1, cl0, cl1, r;
637  u32 ptick00, ptick01, ptick10, ptick11, pvec0, pvec1;
638 
639  cl0 = cl1 = node->clocks_since_last_overflow;
640  ca0 = ca1 = node->calls_since_last_overflow;
641  v0 = v1 = node->vectors_since_last_overflow;
642  ptick00 = ptick01 = node->perf_counter0_ticks_since_last_overflow;
643  ptick10 = ptick11 = node->perf_counter1_ticks_since_last_overflow;
644  pvec0 = pvec1 = node->perf_counter_vectors_since_last_overflow;
645 
646  ca1 = ca0 + n_calls;
647  v1 = v0 + n_vectors;
648  cl1 = cl0 + n_clocks;
649  ptick01 = ptick00 + n_ticks0;
650  ptick11 = ptick10 + n_ticks1;
651  pvec1 = pvec0 + n_vectors;
652 
653  node->calls_since_last_overflow = ca1;
654  node->clocks_since_last_overflow = cl1;
655  node->vectors_since_last_overflow = v1;
659 
660  node->max_clock_n = node->max_clock > n_clocks ?
661  node->max_clock_n : n_vectors;
662  node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks;
663 
664  r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
665 
666  if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0) || (ptick01 < ptick00)
667  || (ptick11 < ptick10) || (pvec1 < pvec0))
668  {
669  node->calls_since_last_overflow = ca0;
670  node->clocks_since_last_overflow = cl0;
671  node->vectors_since_last_overflow = v0;
675 
676  vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks,
677  n_ticks0, n_ticks1);
678  }
679 
680  return r;
681 }
682 
683 static inline void
685 {
686  *pmc0 = 0;
687  *pmc1 = 0;
689  (*vm->vlib_node_runtime_perf_counter_cb) (vm, pmc0, pmc1);
690 }
691 
692 always_inline void
694  vlib_process_t * p,
695  uword n_calls, uword n_vectors, uword n_clocks)
696 {
698  n_calls, n_vectors, n_clocks, 0ULL, 0ULL);
699 }
700 
701 static clib_error_t *
703  unformat_input_t * input, vlib_cli_command_t * cmd)
704 {
706  return 0;
707 }
708 
709 /* *INDENT-OFF* */
710 VLIB_CLI_COMMAND (elog_clear_cli, static) = {
711  .path = "event-logger clear",
712  .short_help = "Clear the event log",
713  .function = vlib_cli_elog_clear,
714 };
715 /* *INDENT-ON* */
716 
717 #ifdef CLIB_UNIX
718 static clib_error_t *
720  unformat_input_t * input, vlib_cli_command_t * cmd)
721 {
722  elog_main_t *em = &vm->elog_main;
723  char *file, *chroot_file;
724  clib_error_t *error = 0;
725 
726  if (!unformat (input, "%s", &file))
727  {
728  vlib_cli_output (vm, "expected file name, got `%U'",
729  format_unformat_error, input);
730  return 0;
731  }
732 
733  /* It's fairly hard to get "../oopsie" through unformat; just in case */
734  if (strstr (file, "..") || index (file, '/'))
735  {
736  vlib_cli_output (vm, "illegal characters in filename '%s'", file);
737  return 0;
738  }
739 
740  chroot_file = (char *) format (0, "/tmp/%s%c", file, 0);
741 
742  vec_free (file);
743 
744  vlib_cli_output (vm, "Saving %wd of %wd events to %s",
746  elog_buffer_capacity (em), chroot_file);
747 
749  error = elog_write_file (em, chroot_file, 1 /* flush ring */ );
751  vec_free (chroot_file);
752  return error;
753 }
754 
755 void
757 {
759  elog_main_t *em = &vm->elog_main;
760  u8 *filename;
761  clib_error_t *error;
762 
763  if (!vm->elog_post_mortem_dump)
764  return;
765 
766  filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0);
767  error = elog_write_file (em, (char *) filename, 1 /* flush ring */ );
768  if (error)
769  clib_error_report (error);
770  vec_free (filename);
771 }
772 
773 /* *INDENT-OFF* */
774 VLIB_CLI_COMMAND (elog_save_cli, static) = {
775  .path = "event-logger save",
776  .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
777  .function = elog_save_buffer,
778 };
779 /* *INDENT-ON* */
780 
781 static clib_error_t *
783  unformat_input_t * input, vlib_cli_command_t * cmd)
784 {
785  elog_main_t *em = &vm->elog_main;
786 
788 
789  vlib_cli_output (vm, "Stopped the event logger...");
790  return 0;
791 }
792 
793 /* *INDENT-OFF* */
794 VLIB_CLI_COMMAND (elog_stop_cli, static) = {
795  .path = "event-logger stop",
796  .short_help = "Stop the event-logger",
797  .function = elog_stop,
798 };
799 /* *INDENT-ON* */
800 
801 static clib_error_t *
803  unformat_input_t * input, vlib_cli_command_t * cmd)
804 {
805  elog_main_t *em = &vm->elog_main;
806 
808 
809  vlib_cli_output (vm, "Restarted the event logger...");
810  return 0;
811 }
812 
813 /* *INDENT-OFF* */
814 VLIB_CLI_COMMAND (elog_restart_cli, static) = {
815  .path = "event-logger restart",
816  .short_help = "Restart the event-logger",
817  .function = elog_restart,
818 };
819 /* *INDENT-ON* */
820 
821 static clib_error_t *
823  unformat_input_t * input, vlib_cli_command_t * cmd)
824 {
825  elog_main_t *em = &vm->elog_main;
826  u32 tmp;
827 
828  /* Stop the parade */
830 
831  if (unformat (input, "%d", &tmp))
832  {
833  elog_alloc (em, tmp);
835  }
836  else
837  return clib_error_return (0, "Must specify how many events in the ring");
838 
839  vlib_cli_output (vm, "Resized ring and restarted the event logger...");
840  return 0;
841 }
842 
843 /* *INDENT-OFF* */
844 VLIB_CLI_COMMAND (elog_resize_cli, static) = {
845  .path = "event-logger resize",
846  .short_help = "event-logger resize <nnn>",
847  .function = elog_resize,
848 };
849 /* *INDENT-ON* */
850 
851 #endif /* CLIB_UNIX */
852 
853 static void
855 {
856  elog_main_t *em = &vm->elog_main;
857  elog_event_t *e, *es;
858  f64 dt;
859 
860  /* Show events in VLIB time since log clock starts after VLIB clock. */
861  dt = (em->init_time.cpu - vm->clib_time.init_cpu_time)
863 
864  es = elog_peek_events (em);
865  vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es),
866  em->event_ring_size,
868  "running" : "stopped");
869  vec_foreach (e, es)
870  {
871  vlib_cli_output (vm, "%18.9f: %U",
872  e->time + dt, format_elog_event, em, e);
873  n_events_to_show--;
874  if (n_events_to_show == 0)
875  break;
876  }
877  vec_free (es);
878 
879 }
880 
881 static clib_error_t *
883  unformat_input_t * input, vlib_cli_command_t * cmd)
884 {
885  u32 n_events_to_show;
886  clib_error_t *error = 0;
887 
888  n_events_to_show = 250;
890  {
891  if (unformat (input, "%d", &n_events_to_show))
892  ;
893  else if (unformat (input, "all"))
894  n_events_to_show = ~0;
895  else
896  return unformat_parse_error (input);
897  }
898  elog_show_buffer_internal (vm, n_events_to_show);
899  return error;
900 }
901 
902 /* *INDENT-OFF* */
903 VLIB_CLI_COMMAND (elog_show_cli, static) = {
904  .path = "show event-logger",
905  .short_help = "Show event logger info",
906  .function = elog_show_buffer,
907 };
908 /* *INDENT-ON* */
909 
910 void
912 {
914 }
915 
916 static inline void
918  u32 node_index,
919  u64 time, u32 n_vectors, u32 is_return)
920 {
922  elog_main_t *em = &evm->elog_main;
923  int enabled = evm->elog_trace_graph_dispatch |
925 
926  if (PREDICT_FALSE (enabled && n_vectors))
927  {
928  if (PREDICT_FALSE (!elog_is_enabled (em)))
929  {
930  evm->elog_trace_graph_dispatch = 0;
931  evm->elog_trace_graph_circuit = 0;
932  return;
933  }
934  if (PREDICT_TRUE
936  (evm->elog_trace_graph_circuit &&
937  node_index == evm->elog_trace_graph_circuit_node_index)))
938  {
939  elog_track (em,
940  /* event type */
941  vec_elt_at_index (is_return
944  node_index),
945  /* track */
946  (vm->thread_index ?
948  : &em->default_track),
949  /* data to log */ n_vectors);
950  }
951  }
952 }
953 
954 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
955 void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 node_index);
956 void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
957 
958 void
959 vlib_buffer_trace_trajectory_init (vlib_buffer_t * b)
960 {
961  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_init_cb != 0))
962  {
963  (*vlib_buffer_trace_trajectory_init_cb) (b);
964  }
965 }
966 
967 #endif
968 
969 static inline void
971 {
972 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
973  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_cb != 0))
974  {
975  (*vlib_buffer_trace_trajectory_cb) (b, node_index);
976  }
977 #endif
978 }
979 
980 u8 *format_vnet_buffer_flags (u8 * s, va_list * args) __attribute__ ((weak));
981 u8 *
982 format_vnet_buffer_flags (u8 * s, va_list * args)
983 {
984  s = format (s, "BUG STUB %s", __FUNCTION__);
985  return s;
986 }
987 
988 u8 *format_vnet_buffer_opaque (u8 * s, va_list * args) __attribute__ ((weak));
989 u8 *
990 format_vnet_buffer_opaque (u8 * s, va_list * args)
991 {
992  s = format (s, "BUG STUB %s", __FUNCTION__);
993  return s;
994 }
995 
996 u8 *format_vnet_buffer_opaque2 (u8 * s, va_list * args)
997  __attribute__ ((weak));
998 u8 *
999 format_vnet_buffer_opaque2 (u8 * s, va_list * args)
1000 {
1001  s = format (s, "BUG STUB %s", __FUNCTION__);
1002  return s;
1003 }
1004 
1005 static u8 *
1006 format_buffer_metadata (u8 * s, va_list * args)
1007 {
1008  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
1009 
1010  s = format (s, "flags: %U\n", format_vnet_buffer_flags, b);
1011  s = format (s, "current_data: %d, current_length: %d\n",
1012  (i32) (b->current_data), (i32) (b->current_length));
1013  s = format (s, "current_config_index: %d, flow_id: %x, next_buffer: %x\n",
1015  s = format (s, "error: %d, ref_count: %d, buffer_pool_index: %d\n",
1016  (u32) (b->error), (u32) (b->ref_count),
1017  (u32) (b->buffer_pool_index));
1018  s = format (s,
1019  "trace_index: %d, len_not_first_buf: %d\n",
1021  return s;
1022 }
1023 
1024 #define A(x) vec_add1(vm->pcap_buffer, (x))
1025 
1026 static void
1028  vlib_node_runtime_t * node, vlib_frame_t * frame)
1029 {
1030  int i;
1031  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **bufp, *b;
1032  pcap_main_t *pm = &vm->dispatch_pcap_main;
1033  vlib_trace_main_t *tm = &vm->trace_main;
1034  u32 capture_size;
1035  vlib_node_t *n;
1036  i32 n_left;
1037  f64 time_now = vlib_time_now (vm);
1038  u32 *from;
1039  u8 *d;
1040  u8 string_count;
1041 
1042  /* Input nodes don't have frames yet */
1043  if (frame == 0 || frame->n_vectors == 0)
1044  return;
1045 
1046  from = vlib_frame_vector_args (frame);
1047  vlib_get_buffers (vm, from, bufs, frame->n_vectors);
1048  bufp = bufs;
1049 
1050  n = vlib_get_node (vm, node->node_index);
1051 
1052  for (i = 0; i < frame->n_vectors; i++)
1053  {
1055  {
1056  b = bufp[i];
1057 
1059  string_count = 0;
1060 
1061  /* Version, flags */
1064  A (0 /* string_count */ );
1065  A (n->protocol_hint);
1066 
1067  /* Buffer index (big endian) */
1068  A ((from[i] >> 24) & 0xff);
1069  A ((from[i] >> 16) & 0xff);
1070  A ((from[i] >> 8) & 0xff);
1071  A ((from[i] >> 0) & 0xff);
1072 
1073  /* Node name, NULL-terminated ASCII */
1074  vm->pcap_buffer = format (vm->pcap_buffer, "%v%c", n->name, 0);
1075  string_count++;
1076 
1077  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1078  format_buffer_metadata, b, 0);
1079  string_count++;
1080  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1082  string_count++;
1083  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1085  string_count++;
1086 
1087  /* Is this packet traced? */
1088  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
1089  {
1092 
1093  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1094  format_vlib_trace, vm, h[0], 0);
1095  string_count++;
1096  }
1097 
1098  /* Save the string count */
1099  vm->pcap_buffer[2] = string_count;
1100 
1101  /* Figure out how many bytes in the pcap trace */
1102  capture_size = vec_len (vm->pcap_buffer) +
1103  +vlib_buffer_length_in_chain (vm, b);
1104 
1106  n_left = clib_min (capture_size, 16384);
1107  d = pcap_add_packet (pm, time_now, n_left, capture_size);
1108 
1109  /* Copy the header */
1111  d += vec_len (vm->pcap_buffer);
1112 
1113  n_left = clib_min
1114  (vlib_buffer_length_in_chain (vm, b),
1115  (16384 - vec_len (vm->pcap_buffer)));
1116  /* Copy the packet data */
1117  while (1)
1118  {
1119  u32 copy_length = clib_min ((u32) n_left, b->current_length);
1120  clib_memcpy_fast (d, b->data + b->current_data, copy_length);
1121  n_left -= b->current_length;
1122  if (n_left <= 0)
1123  break;
1124  d += b->current_length;
1125  ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
1126  b = vlib_get_buffer (vm, b->next_buffer);
1127  }
1129  }
1130  }
1131 }
1132 
1135  vlib_node_runtime_t * node,
1136  vlib_node_type_t type,
1137  vlib_node_state_t dispatch_state,
1138  vlib_frame_t * frame, u64 last_time_stamp)
1139 {
1140  uword n, v;
1141  u64 t;
1142  vlib_node_main_t *nm = &vm->node_main;
1143  vlib_next_frame_t *nf;
1144  u64 pmc_before[2], pmc_after[2], pmc_delta[2];
1145 
1146  if (CLIB_DEBUG > 0)
1147  {
1148  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1149  ASSERT (n->type == type);
1150  }
1151 
1152  /* Only non-internal nodes may be disabled. */
1153  if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state)
1154  {
1155  ASSERT (type != VLIB_NODE_TYPE_INTERNAL);
1156  return last_time_stamp;
1157  }
1158 
1159  if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT)
1160  && dispatch_state != VLIB_NODE_STATE_INTERRUPT)
1161  {
1163  /* Only call node when count reaches zero. */
1164  if (c)
1165  {
1166  node->input_main_loops_per_call = c - 1;
1167  return last_time_stamp;
1168  }
1169  }
1170 
1171  /* Speculatively prefetch next frames. */
1172  if (node->n_next_nodes > 0)
1173  {
1174  nf = vec_elt_at_index (nm->next_frames, node->next_frame_index);
1175  CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE);
1176  }
1177 
1178  vm->cpu_time_last_node_dispatch = last_time_stamp;
1179 
1181  last_time_stamp, frame ? frame->n_vectors : 0,
1182  /* is_after */ 0);
1183 
1184  vlib_node_runtime_perf_counter (vm, &pmc_before[0], &pmc_before[1]);
1185 
1186  /*
1187  * Turn this on if you run into
1188  * "bad monkey" contexts, and you want to know exactly
1189  * which nodes they've visited... See ixge.c...
1190  */
1191  if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
1192  {
1193  int i;
1194  u32 *from;
1195  from = vlib_frame_vector_args (frame);
1196  for (i = 0; i < frame->n_vectors; i++)
1197  {
1198  vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
1199  add_trajectory_trace (b, node->node_index);
1200  }
1202  dispatch_pcap_trace (vm, node, frame);
1203  n = node->function (vm, node, frame);
1204  }
1205  else
1206  {
1208  dispatch_pcap_trace (vm, node, frame);
1209  n = node->function (vm, node, frame);
1210  }
1211 
1212  t = clib_cpu_time_now ();
1213 
1214  /*
1215  * To validate accounting: pmc_delta = t - pmc_before;
1216  * perf ticks should equal clocks/pkt...
1217  */
1218  vlib_node_runtime_perf_counter (vm, &pmc_after[0], &pmc_after[1]);
1219 
1220  pmc_delta[0] = pmc_after[0] - pmc_before[0];
1221  pmc_delta[1] = pmc_after[1] - pmc_before[1];
1222 
1223  vlib_elog_main_loop_event (vm, node->node_index, t, n, 1 /* is_after */ );
1224 
1225  vm->main_loop_vectors_processed += n;
1226  vm->main_loop_nodes_processed += n > 0;
1227 
1228  v = vlib_node_runtime_update_stats (vm, node,
1229  /* n_calls */ 1,
1230  /* n_vectors */ n,
1231  /* n_clocks */ t - last_time_stamp,
1232  pmc_delta[0] /* PMC0 */ ,
1233  pmc_delta[1] /* PMC1 */ );
1234 
1235  /* When in interrupt mode and vector rate crosses threshold switch to
1236  polling mode. */
1237  if (PREDICT_FALSE ((dispatch_state == VLIB_NODE_STATE_INTERRUPT)
1238  || (dispatch_state == VLIB_NODE_STATE_POLLING
1239  && (node->flags
1240  &
1242  {
1243  /* *INDENT-OFF* */
1244  ELOG_TYPE_DECLARE (e) =
1245  {
1246  .function = (char *) __FUNCTION__,
1247  .format = "%s vector length %d, switching to %s",
1248  .format_args = "T4i4t4",
1249  .n_enum_strings = 2,
1250  .enum_strings = {
1251  "interrupt", "polling",
1252  },
1253  };
1254  /* *INDENT-ON* */
1255  struct
1256  {
1257  u32 node_name, vector_length, is_polling;
1258  } *ed;
1259 
1260  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
1261  && v >= nm->polling_threshold_vector_length) &&
1262  !(node->flags &
1264  {
1265  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1266  n->state = VLIB_NODE_STATE_POLLING;
1267  node->state = VLIB_NODE_STATE_POLLING;
1268  node->flags &=
1271  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
1272  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
1273 
1274  if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
1275  {
1277  + vm->thread_index;
1278 
1279  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1280  w->elog_track);
1281  ed->node_name = n->name_elog_string;
1282  ed->vector_length = v;
1283  ed->is_polling = 1;
1284  }
1285  }
1286  else if (dispatch_state == VLIB_NODE_STATE_POLLING
1287  && v <= nm->interrupt_threshold_vector_length)
1288  {
1289  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1290  if (node->flags &
1292  {
1293  /* Switch to interrupt mode after dispatch in polling one more time.
1294  This allows driver to re-enable interrupts. */
1295  n->state = VLIB_NODE_STATE_INTERRUPT;
1296  node->state = VLIB_NODE_STATE_INTERRUPT;
1297  node->flags &=
1299  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -= 1;
1300  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] += 1;
1301 
1302  }
1303  else
1304  {
1306  + vm->thread_index;
1307  node->flags |=
1309  if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
1310  {
1311  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1312  w->elog_track);
1313  ed->node_name = n->name_elog_string;
1314  ed->vector_length = v;
1315  ed->is_polling = 0;
1316  }
1317  }
1318  }
1319  }
1320 
1321  return t;
1322 }
1323 
1324 static u64
1325 dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
1326  u64 last_time_stamp)
1327 {
1328  vlib_node_main_t *nm = &vm->node_main;
1329  vlib_frame_t *f;
1330  vlib_next_frame_t *nf, nf_dummy;
1332  u32 restore_frame_index;
1334 
1335  /* See comment below about dangling references to nm->pending_frames */
1336  p = nm->pending_frames + pending_frame_index;
1337 
1339  p->node_runtime_index);
1340 
1341  f = vlib_get_frame (vm, p->frame_index);
1343  {
1344  /* No next frame: so use dummy on stack. */
1345  nf = &nf_dummy;
1347  nf->frame_index = ~p->frame_index;
1348  }
1349  else
1351 
1353 
1354  /* Force allocation of new frame while current frame is being
1355  dispatched. */
1356  restore_frame_index = ~0;
1357  if (nf->frame_index == p->frame_index)
1358  {
1359  nf->frame_index = ~0;
1362  restore_frame_index = p->frame_index;
1363  }
1364 
1365  /* Frame must be pending. */
1367  ASSERT (f->n_vectors > 0);
1368 
1369  /* Copy trace flag from next frame to node.
1370  Trace flag indicates that at least one vector in the dispatched
1371  frame is traced. */
1372  n->flags &= ~VLIB_NODE_FLAG_TRACE;
1373  n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0;
1374  nf->flags &= ~VLIB_FRAME_TRACE;
1375 
1376  last_time_stamp = dispatch_node (vm, n,
1378  VLIB_NODE_STATE_POLLING,
1379  f, last_time_stamp);
1380 
1382 
1383  /* Frame is ready to be used again, so restore it. */
1384  if (restore_frame_index != ~0)
1385  {
1386  /*
1387  * We musn't restore a frame that is flagged to be freed. This
1388  * shouldn't happen since frames to be freed post dispatch are
1389  * those used when the to-node frame becomes full i.e. they form a
1390  * sort of queue of frames to a single node. If we get here then
1391  * the to-node frame and the pending frame *were* the same, and so
1392  * we removed the to-node frame. Therefore this frame is no
1393  * longer part of the queue for that node and hence it cannot be
1394  * it's overspill.
1395  */
1397 
1398  /*
1399  * NB: dispatching node n can result in the creation and scheduling
1400  * of new frames, and hence in the reallocation of nm->pending_frames.
1401  * Recompute p, or no supper. This was broken for more than 10 years.
1402  */
1403  p = nm->pending_frames + pending_frame_index;
1404 
1405  /*
1406  * p->next_frame_index can change during node dispatch if node
1407  * function decides to change graph hook up.
1408  */
1411 
1412  if (~0 == nf->frame_index)
1413  {
1414  /* no new frame has been assigned to this node, use the saved one */
1415  nf->frame_index = restore_frame_index;
1416  f->n_vectors = 0;
1417  }
1418  else
1419  {
1420  /* The node has gained a frame, implying packets from the current frame
1421  were re-queued to this same node. we don't need the saved one
1422  anymore */
1423  vlib_frame_free (vm, n, f);
1424  }
1425  }
1426  else
1427  {
1429  {
1431  vlib_frame_free (vm, n, f);
1432  }
1433  }
1434 
1435  return last_time_stamp;
1436 }
1437 
1440 {
1441  return p->stack[0] == VLIB_PROCESS_STACK_MAGIC;
1442 }
1443 
1444 typedef struct
1445 {
1450 
1451 /* Called in process stack. */
1452 static uword
1454 {
1456  vlib_main_t *vm;
1457  vlib_node_runtime_t *node;
1458  vlib_frame_t *f;
1459  vlib_process_t *p;
1460  uword n;
1461 
1463 
1464  vm = a->vm;
1465  p = a->process;
1466  f = a->frame;
1467  node = &p->node_runtime;
1468 
1469  n = node->function (vm, node, f);
1470 
1472 
1473  clib_longjmp (&p->return_longjmp, n);
1474 
1475  return n;
1476 }
1477 
1478 /* Called in main stack. */
1481 {
1483  uword r;
1484 
1485  a.vm = vm;
1486  a.process = p;
1487  a.frame = f;
1488 
1492  (void *) p->stack + (1 << p->log2_n_stack_bytes));
1493 
1494  return r;
1495 }
1496 
1499 {
1500  uword r;
1507  return r;
1508 }
1509 
1510 static u64
1512  vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
1513 {
1514  vlib_node_main_t *nm = &vm->node_main;
1515  vlib_node_runtime_t *node_runtime = &p->node_runtime;
1516  vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index);
1517  u32 old_process_index;
1518  u64 t;
1519  uword n_vectors, is_suspend;
1520 
1521  if (node->state != VLIB_NODE_STATE_POLLING
1524  return last_time_stamp;
1525 
1527 
1528  t = last_time_stamp;
1529  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1530  f ? f->n_vectors : 0, /* is_after */ 0);
1531 
1532  /* Save away current process for suspend. */
1533  old_process_index = nm->current_process_index;
1535 
1536  n_vectors = vlib_process_startup (vm, p, f);
1537 
1538  nm->current_process_index = old_process_index;
1539 
1541  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1542  if (is_suspend)
1543  {
1545 
1546  n_vectors = 0;
1548  pf->node_runtime_index = node->runtime_index;
1549  pf->frame_index = f ? vlib_frame_index (vm, f) : ~0;
1550  pf->next_frame_index = ~0;
1551 
1552  p->n_suspends += 1;
1554 
1556  {
1557  TWT (tw_timer_wheel) * tw =
1558  (TWT (tw_timer_wheel) *) nm->timing_wheel;
1559  p->stop_timer_handle =
1560  TW (tw_timer_start) (tw,
1562  (node->runtime_index) /* [sic] pool idex */ ,
1563  0 /* timer_id */ ,
1565  }
1566  }
1567  else
1569 
1570  t = clib_cpu_time_now ();
1571 
1572  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
1573  /* is_after */ 1);
1574 
1576  /* n_calls */ !is_suspend,
1577  /* n_vectors */ n_vectors,
1578  /* n_clocks */ t - last_time_stamp);
1579 
1580  return t;
1581 }
1582 
1583 void
1585 {
1586  vlib_node_main_t *nm = &vm->node_main;
1587  vlib_process_t *p = vec_elt (nm->processes, process_index);
1588  dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
1589 }
1590 
1591 static u64
1593  uword process_index, u64 last_time_stamp)
1594 {
1595  vlib_node_main_t *nm = &vm->node_main;
1596  vlib_node_runtime_t *node_runtime;
1597  vlib_node_t *node;
1598  vlib_frame_t *f;
1599  vlib_process_t *p;
1601  u64 t, n_vectors, is_suspend;
1602 
1603  t = last_time_stamp;
1604 
1605  p = vec_elt (nm->processes, process_index);
1607  return last_time_stamp;
1608 
1611 
1614 
1615  node_runtime = &p->node_runtime;
1616  node = vlib_get_node (vm, node_runtime->node_index);
1617  f = pf->frame_index != ~0 ? vlib_get_frame (vm, pf->frame_index) : 0;
1618 
1619  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1620  f ? f->n_vectors : 0, /* is_after */ 0);
1621 
1622  /* Save away current process for suspend. */
1624 
1625  n_vectors = vlib_process_resume (p);
1626  t = clib_cpu_time_now ();
1627 
1628  nm->current_process_index = ~0;
1629 
1630  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1631  if (is_suspend)
1632  {
1633  /* Suspend it again. */
1634  n_vectors = 0;
1635  p->n_suspends += 1;
1637  {
1638  p->stop_timer_handle =
1639  TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1641  (node->runtime_index) /* [sic] pool idex */ ,
1642  0 /* timer_id */ ,
1644  }
1645  }
1646  else
1647  {
1652  }
1653 
1654  t = clib_cpu_time_now ();
1655  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend,
1656  /* is_after */ 1);
1657 
1659  /* n_calls */ !is_suspend,
1660  /* n_vectors */ n_vectors,
1661  /* n_clocks */ t - last_time_stamp);
1662 
1663  return t;
1664 }
1665 
1666 void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak));
1667 void
1669 {
1670 }
1671 
1672 
1675 {
1676  vlib_node_main_t *nm = &vm->node_main;
1678  uword i;
1679  u64 cpu_time_now;
1681  u32 *last_node_runtime_indices = 0;
1682  u32 frame_queue_check_counter = 0;
1683 
1684  /* Initialize pending node vector. */
1685  if (is_main)
1686  {
1687  vec_resize (nm->pending_frames, 32);
1688  _vec_len (nm->pending_frames) = 0;
1689  }
1690 
1691  /* Mark time of main loop start. */
1692  if (is_main)
1693  {
1694  cpu_time_now = vm->clib_time.last_cpu_time;
1695  vm->cpu_time_main_loop_start = cpu_time_now;
1696  }
1697  else
1698  cpu_time_now = clib_cpu_time_now ();
1699 
1700  /* Pre-allocate interupt runtime indices and lock. */
1702  vec_alloc (last_node_runtime_indices, 32);
1703  if (!is_main)
1705 
1706  /* Pre-allocate expired nodes. */
1711 
1714 
1715  /* Start all processes. */
1716  if (is_main)
1717  {
1718  uword i;
1719  nm->current_process_index = ~0;
1720  for (i = 0; i < vec_len (nm->processes); i++)
1721  cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
1722  cpu_time_now);
1723  }
1724 
1725  while (1)
1726  {
1728 
1729  if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
1730  {
1731  if (!is_main)
1733  }
1734 
1735  if (!is_main)
1736  {
1739  frame_queue_check_counter))
1740  {
1741  u32 processed = 0;
1742 
1743  if (vm->check_frame_queues)
1744  {
1745  frame_queue_check_counter = 100;
1746  vm->check_frame_queues = 0;
1747  }
1748 
1749  vec_foreach (fqm, tm->frame_queue_mains)
1750  processed += vlib_frame_queue_dequeue (vm, fqm);
1751 
1752  /* No handoff queue work found? */
1753  if (processed)
1754  frame_queue_check_counter = 100;
1755  else
1756  frame_queue_check_counter--;
1757  }
1759  ((void (*)(vlib_main_t *)) vm->worker_thread_main_loop_callback)
1760  (vm);
1761  }
1762 
1763  /* Process pre-input nodes. */
1765  cpu_time_now = dispatch_node (vm, n,
1767  VLIB_NODE_STATE_POLLING,
1768  /* frame */ 0,
1769  cpu_time_now);
1770 
1771  /* Next process input nodes. */
1773  cpu_time_now = dispatch_node (vm, n,
1775  VLIB_NODE_STATE_POLLING,
1776  /* frame */ 0,
1777  cpu_time_now);
1778 
1779  if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
1780  vm->queue_signal_callback (vm);
1781 
1782  /* Next handle interrupts. */
1783  {
1784  /* unlocked read, for performance */
1785  uword l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1786  uword i;
1787  if (PREDICT_FALSE (l > 0))
1788  {
1789  u32 *tmp;
1790  if (!is_main)
1791  {
1793  /* Re-read w/ lock held, in case another thread added an item */
1794  l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1795  }
1796 
1799  last_node_runtime_indices;
1800  last_node_runtime_indices = tmp;
1801  _vec_len (last_node_runtime_indices) = 0;
1802  if (!is_main)
1804  for (i = 0; i < l; i++)
1805  {
1807  last_node_runtime_indices[i]);
1808  cpu_time_now =
1810  VLIB_NODE_STATE_INTERRUPT,
1811  /* frame */ 0,
1812  cpu_time_now);
1813  }
1814  }
1815  }
1816  /* Input nodes may have added work to the pending vector.
1817  Process pending vector until there is nothing left.
1818  All pending vectors will be processed from input -> output. */
1819  for (i = 0; i < _vec_len (nm->pending_frames); i++)
1820  cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
1821  /* Reset pending vector for next iteration. */
1822  _vec_len (nm->pending_frames) = 0;
1823 
1824  if (is_main)
1825  {
1826  /* *INDENT-OFF* */
1827  ELOG_TYPE_DECLARE (es) =
1828  {
1829  .format = "process tw start",
1830  .format_args = "",
1831  };
1832  ELOG_TYPE_DECLARE (ee) =
1833  {
1834  .format = "process tw end: %d",
1835  .format_args = "i4",
1836  };
1837  /* *INDENT-ON* */
1838 
1839  struct
1840  {
1841  int nready_procs;
1842  } *ed;
1843 
1844  /* Check if process nodes have expired from timing wheel. */
1846 
1848  ed = ELOG_DATA (&vlib_global_main.elog_main, es);
1849 
1852  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
1854 
1856 
1858  {
1859  ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
1860  ed->nready_procs =
1861  _vec_len (nm->data_from_advancing_timing_wheel);
1862  }
1863 
1864  if (PREDICT_FALSE
1865  (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
1866  {
1867  uword i;
1868 
1869  for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
1870  i++)
1871  {
1874 
1876  {
1879  di);
1880  vlib_node_t *n =
1882  vlib_process_t *p =
1883  vec_elt (nm->processes, n->runtime_index);
1884  void *data;
1885  data =
1887  te->event_type_index,
1888  te->n_data_elts,
1889  te->n_data_elt_bytes);
1890  if (te->n_data_bytes < sizeof (te->inline_event_data))
1892  te->n_data_bytes);
1893  else
1894  {
1896  te->n_data_bytes);
1898  }
1900  }
1901  else
1902  {
1903  cpu_time_now = clib_cpu_time_now ();
1904  cpu_time_now =
1905  dispatch_suspended_process (vm, di, cpu_time_now);
1906  }
1907  }
1908  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1909  }
1910  }
1912 
1913  /* Record time stamp in case there are no enabled nodes and above
1914  calls do not update time stamp. */
1915  cpu_time_now = clib_cpu_time_now ();
1916  }
1917 }
1918 
1919 static void
1921 {
1922  vlib_main_or_worker_loop (vm, /* is_main */ 1);
1923 }
1924 
1925 void
1927 {
1928  vlib_main_or_worker_loop (vm, /* is_main */ 0);
1929 }
1930 
1932 
1933 static clib_error_t *
1935 {
1936  int turn_on_mem_trace = 0;
1937 
1938  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1939  {
1940  if (unformat (input, "memory-trace"))
1941  turn_on_mem_trace = 1;
1942 
1943  else if (unformat (input, "elog-events %d",
1944  &vm->elog_main.event_ring_size))
1945  ;
1946  else if (unformat (input, "elog-post-mortem-dump"))
1947  vm->elog_post_mortem_dump = 1;
1948  else
1949  return unformat_parse_error (input);
1950  }
1951 
1952  unformat_free (input);
1953 
1954  /* Enable memory trace as early as possible. */
1955  if (turn_on_mem_trace)
1956  clib_mem_trace (1);
1957 
1958  return 0;
1959 }
1960 
1962 
1963 static void
1965 {
1966 }
1967 
1968 #define foreach_weak_reference_stub \
1969 _(vlib_map_stat_segment_init) \
1970 _(vpe_api_init) \
1971 _(vlibmemory_init) \
1972 _(map_api_segment_init)
1973 
1974 #define _(name) \
1975 clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \
1976 clib_error_t *name (vlib_main_t *vm) { return 0; }
1978 #undef _
1979 
1980 /* Main function. */
1981 int
1983 {
1984  clib_error_t *volatile error;
1985  vlib_node_main_t *nm = &vm->node_main;
1986 
1988 
1989  clib_time_init (&vm->clib_time);
1990 
1991  /* Turn on event log. */
1992  if (!vm->elog_main.event_ring_size)
1993  vm->elog_main.event_ring_size = 128 << 10;
1995  elog_enable_disable (&vm->elog_main, 1);
1996 
1997  /* Default name. */
1998  if (!vm->name)
1999  vm->name = "VLIB";
2000 
2001  if ((error = vlib_physmem_init (vm)))
2002  {
2003  clib_error_report (error);
2004  goto done;
2005  }
2006 
2007  if ((error = vlib_map_stat_segment_init (vm)))
2008  {
2009  clib_error_report (error);
2010  goto done;
2011  }
2012 
2013  if ((error = vlib_buffer_main_init (vm)))
2014  {
2015  clib_error_report (error);
2016  goto done;
2017  }
2018 
2019  if ((error = vlib_thread_init (vm)))
2020  {
2021  clib_error_report (error);
2022  goto done;
2023  }
2024 
2025  /* Register static nodes so that init functions may use them. */
2027 
2028  /* Set seed for random number generator.
2029  Allow user to specify seed to make random sequence deterministic. */
2030  if (!unformat (input, "seed %wd", &vm->random_seed))
2031  vm->random_seed = clib_cpu_time_now ();
2033 
2034  /* Initialize node graph. */
2035  if ((error = vlib_node_main_init (vm)))
2036  {
2037  /* Arrange for graph hook up error to not be fatal when debugging. */
2038  if (CLIB_DEBUG > 0)
2039  clib_error_report (error);
2040  else
2041  goto done;
2042  }
2043 
2044  /* Direct call / weak reference, for vlib standalone use-cases */
2045  if ((error = vpe_api_init (vm)))
2046  {
2047  clib_error_report (error);
2048  goto done;
2049  }
2050 
2051  if ((error = vlibmemory_init (vm)))
2052  {
2053  clib_error_report (error);
2054  goto done;
2055  }
2056 
2057  if ((error = map_api_segment_init (vm)))
2058  {
2059  clib_error_report (error);
2060  goto done;
2061  }
2062 
2063  /* See unix/main.c; most likely already set up */
2064  if (vm->init_functions_called == 0)
2065  vm->init_functions_called = hash_create (0, /* value bytes */ 0);
2066  if ((error = vlib_call_all_init_functions (vm)))
2067  goto done;
2068 
2069  nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
2071 
2073  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
2074 
2075  /* Create the process timing wheel */
2076  TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
2077  0 /* no callback */ ,
2078  10e-6 /* timer period 10us */ ,
2079  ~0 /* max expirations per call */ );
2080 
2082  _vec_len (vm->pending_rpc_requests) = 0;
2084  _vec_len (vm->processing_rpc_requests) = 0;
2085 
2086  if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
2087  goto done;
2088 
2089  /* Call all main loop enter functions. */
2090  {
2091  clib_error_t *sub_error;
2092  sub_error = vlib_call_all_main_loop_enter_functions (vm);
2093  if (sub_error)
2094  clib_error_report (sub_error);
2095  }
2096 
2098  {
2100  vm->main_loop_exit_set = 1;
2101  break;
2102 
2104  goto done;
2105 
2106  default:
2107  error = vm->main_loop_error;
2108  goto done;
2109  }
2110 
2111  vlib_main_loop (vm);
2112 
2113 done:
2114  /* Call all exit functions. */
2115  {
2116  clib_error_t *sub_error;
2117  sub_error = vlib_call_all_main_loop_exit_functions (vm);
2118  if (sub_error)
2119  clib_error_report (sub_error);
2120  }
2121 
2122  if (error)
2123  clib_error_report (error);
2124 
2125  return 0;
2126 }
2127 
2128 static inline clib_error_t *
2130  unformat_input_t * input,
2131  vlib_cli_command_t * cmd, int rx_tx)
2132 {
2133  unformat_input_t _line_input, *line_input = &_line_input;
2134  pcap_main_t *pm = &vm->dispatch_pcap_main;
2135  u8 *filename = 0;
2136  u32 max = 1000;
2137  int enabled = 0;
2138  int is_error = 0;
2139  clib_error_t *error = 0;
2140  u32 node_index, add;
2141  vlib_trace_main_t *tm;
2142  vlib_trace_node_t *tn;
2143 
2144  /* Get a line of input. */
2145  if (!unformat_user (input, unformat_line_input, line_input))
2146  return 0;
2147 
2148  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2149  {
2150  if (unformat (line_input, "on"))
2151  {
2152  if (vm->dispatch_pcap_enable == 0)
2153  {
2154  enabled = 1;
2155  }
2156  else
2157  {
2158  vlib_cli_output (vm, "pcap dispatch capture already on...");
2159  is_error = 1;
2160  break;
2161  }
2162  }
2163  else if (unformat (line_input, "off"))
2164  {
2165  if (vm->dispatch_pcap_enable)
2166  {
2168  (vm, "captured %d pkts...", pm->n_packets_captured);
2169  if (pm->n_packets_captured)
2170  {
2172  error = pcap_write (pm);
2173  if (error)
2174  clib_error_report (error);
2175  else
2176  vlib_cli_output (vm, "saved to %s...", pm->file_name);
2177  }
2178  vm->dispatch_pcap_enable = 0;
2179  }
2180  else
2181  {
2182  vlib_cli_output (vm, "pcap tx capture already off...");
2183  is_error = 1;
2184  break;
2185  }
2186  }
2187  else if (unformat (line_input, "max %d", &max))
2188  {
2189  if (vm->dispatch_pcap_enable)
2190  {
2192  (vm,
2193  "can't change max value while pcap tx capture active...");
2194  is_error = 1;
2195  break;
2196  }
2197  pm->n_packets_to_capture = max;
2198  }
2199  else
2200  if (unformat
2201  (line_input, "file %U", unformat_vlib_tmpfile, &filename))
2202  {
2203  if (vm->dispatch_pcap_enable)
2204  {
2206  (vm, "can't change file while pcap tx capture active...");
2207  is_error = 1;
2208  break;
2209  }
2210  }
2211  else if (unformat (line_input, "status"))
2212  {
2213  if (vm->dispatch_pcap_enable)
2214  {
2216  (vm, "pcap dispatch capture is on: %d of %d pkts...",
2218  vlib_cli_output (vm, "Capture to file %s", pm->file_name);
2219  }
2220  else
2221  {
2222  vlib_cli_output (vm, "pcap dispatch capture is off...");
2223  }
2224  break;
2225  }
2226  else if (unformat (line_input, "buffer-trace %U %d",
2227  unformat_vlib_node, vm, &node_index, &add))
2228  {
2229  if (vnet_trace_dummy == 0)
2232  vlib_cli_output (vm, "Buffer tracing of %d pkts from %U enabled...",
2233  add, format_vlib_node_name, vm, node_index);
2234 
2235  /* *INDENT-OFF* */
2237  {
2238  tm = &this_vlib_main->trace_main;
2239  tm->verbose = 0; /* not sure this ever did anything... */
2240  vec_validate (tm->nodes, node_index);
2241  tn = tm->nodes + node_index;
2242  tn->limit += add;
2243  tm->trace_enable = 1;
2244  }));
2245  /* *INDENT-ON* */
2246  }
2247 
2248  else
2249  {
2250  error = clib_error_return (0, "unknown input `%U'",
2251  format_unformat_error, line_input);
2252  is_error = 1;
2253  break;
2254  }
2255  }
2256  unformat_free (line_input);
2257 
2258  if (is_error == 0)
2259  {
2260  /* Clean up from previous run */
2261  vec_free (pm->file_name);
2262  vec_free (pm->pcap_data);
2263 
2264  memset (pm, 0, sizeof (*pm));
2265  pm->n_packets_to_capture = max;
2266 
2267  if (enabled)
2268  {
2269  if (filename == 0)
2270  filename = format (0, "/tmp/dispatch.pcap%c", 0);
2271 
2272  pm->file_name = (char *) filename;
2273  pm->n_packets_captured = 0;
2274  pm->packet_type = PCAP_PACKET_TYPE_vpp;
2275  if (pm->lock == 0)
2276  clib_spinlock_init (&(pm->lock));
2277  vm->dispatch_pcap_enable = 1;
2278  vlib_cli_output (vm, "pcap dispatch capture on...");
2279  }
2280  }
2281 
2282  return error;
2283 }
2284 
2285 static clib_error_t *
2287  unformat_input_t * input,
2288  vlib_cli_command_t * cmd)
2289 {
2290  return pcap_dispatch_trace_command_internal (vm, input, cmd, VLIB_RX);
2291 }
2292 
2293 /*?
2294  * This command is used to start or stop pcap dispatch trace capture, or show
2295  * the capture status.
2296  *
2297  * This command has the following optional parameters:
2298  *
2299  * - <b>on|off</b> - Used to start or stop capture.
2300  *
2301  * - <b>max <nn></b> - Depth of local buffer. Once '<em>nn</em>' number
2302  * of packets have been received, buffer is flushed to file. Once another
2303  * '<em>nn</em>' number of packets have been received, buffer is flushed
2304  * to file, overwriting previous write. If not entered, value defaults
2305  * to 100. Can only be updated if packet capture is off.
2306  *
2307  * - <b>file <name></b> - Used to specify the output filename. The file will
2308  * be placed in the '<em>/tmp</em>' directory, so only the filename is
2309  * supported. Directory should not be entered. If file already exists, file
2310  * will be overwritten. If no filename is provided, '<em>/tmp/vpe.pcap</em>'
2311  * will be used. Can only be updated if packet capture is off.
2312  *
2313  * - <b>status</b> - Displays the current status and configured attributes
2314  * associated with a packet capture. If packet capture is in progress,
2315  * '<em>status</em>' also will return the number of packets currently in
2316  * the local buffer. All additional attributes entered on command line
2317  * with '<em>status</em>' will be ignored and not applied.
2318  *
2319  * @cliexpar
2320  * Example of how to display the status of capture when off:
2321  * @cliexstart{pcap dispatch trace status}
2322  * max is 100, for any interface to file /tmp/vpe.pcap
2323  * pcap dispatch capture is off...
2324  * @cliexend
2325  * Example of how to start a dispatch trace capture:
2326  * @cliexstart{pcap dispatch trace on max 35 file dispatchTrace.pcap}
2327  * pcap dispatch capture on...
2328  * @cliexend
2329  * Example of how to start a dispatch trace capture with buffer tracing
2330  * @cliexstart{pcap dispatch trace on max 10000 file dispatchTrace.pcap buffer-trace dpdk-input 1000}
2331  * pcap dispatch capture on...
2332  * @cliexend
2333  * Example of how to display the status of a tx packet capture in progress:
2334  * @cliexstart{pcap tx trace status}
2335  * max is 35, dispatch trace to file /tmp/vppTest.pcap
2336  * pcap tx capture is on: 20 of 35 pkts...
2337  * @cliexend
2338  * Example of how to stop a tx packet capture:
2339  * @cliexstart{vppctl pcap dispatch trace off}
2340  * captured 21 pkts...
2341  * saved to /tmp/dispatchTrace.pcap...
2342  * @cliexend
2343 ?*/
2344 /* *INDENT-OFF* */
2345 VLIB_CLI_COMMAND (pcap_dispatch_trace_command, static) = {
2346  .path = "pcap dispatch trace",
2347  .short_help =
2348  "pcap dispatch trace [on|off] [max <nn>] [file <name>] [status]\n"
2349  " [buffer-trace <input-node-name> <nn>]",
2350  .function = pcap_dispatch_trace_command_fn,
2351 };
2352 /* *INDENT-ON* */
2353 
2354 /*
2355  * fd.io coding-style-patch-verification: ON
2356  *
2357  * Local Variables:
2358  * eval: (c-set-style "gnu")
2359  * End:
2360  */
u32 * next_nodes
Definition: node.h:333
static void elog_enable_disable(elog_main_t *em, int is_enabled)
Enable or disable event logging.
Definition: elog.h:217
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
u32 TW() tw_timer_start(TWT(tw_timer_wheel)*tw, u32 user_id, u32 timer_id, u64 interval)
Start a Tw Timer.
clib_error_t * vlib_call_all_main_loop_exit_functions(vlib_main_t *vm)
Definition: init.c:87
uword * pending_rpc_requests
Definition: main.h:243
vlib_main_t vlib_global_main
Definition: main.c:1931
f64 time
Absolute time as floating point number in seconds.
Definition: elog.h:66
void TW() tw_timer_wheel_init(TWT(tw_timer_wheel)*tw, void *expired_timer_callback, f64 timer_interval_in_seconds, u32 max_expirations)
Initialize a tw timer wheel template instance.
u16 vector_size
Definition: node.h:319
u32 max_clock
Maximum clock cycle for an invocation.
Definition: node.h:477
u32 next_frame_index
Start of next frames for this node.
Definition: node.h:492
static clib_error_t * show_frame_stats(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:250
#define hash_set(h, key, value)
Definition: hash.h:255
char * file_name
File name of pcap output.
Definition: pcap.h:162
u32 flags
Definition: vhost_user.h:115
vlib_node_type_t
Definition: node.h:69
#define clib_min(x, y)
Definition: clib.h:295
clib_error_t * vlib_call_all_main_loop_enter_functions(vlib_main_t *vm)
Definition: init.c:80
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:89
static void vlib_next_frame_change_ownership(vlib_main_t *vm, vlib_node_runtime_t *node_runtime, u32 next_index)
Definition: main.c:280
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:74
static vlib_next_frame_t * vlib_node_get_next_frame(vlib_main_t *vm, u32 node_index, u32 next_index)
Get pointer to frame by (node_index, next_index).
Definition: node_funcs.h:329
u32 interrupt_threshold_vector_length
Definition: node.h:718
vlib_process_t ** processes
Definition: node.h:739
static uword vlib_process_bootstrap(uword _a)
Definition: main.c:1453
format_function_t format_vlib_node_name
Definition: node_funcs.h:1171
vlib_node_runtime_t node_runtime
Definition: node.h:553
a
Definition: bitmap.h:538
u32 n_suspends
Definition: node.h:581
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:460
u32 n_packets_to_capture
Number of packets to capture.
Definition: pcap.h:165
vlib_frame_t * vlib_get_next_frame_internal(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 allocate_new_next_frame)
Definition: main.c:363
uword * processing_rpc_requests
Definition: main.h:244
static u32 clib_get_current_numa_node()
Definition: cpu.h:171
#define VLIB_PCAP_MAJOR_VERSION
Definition: main.h:401
vlib_trace_node_t * nodes
Definition: trace.h:86
static void vlib_elog_main_loop_event(vlib_main_t *vm, u32 node_index, u64 time, u32 n_vectors, u32 is_return)
Definition: main.c:917
static clib_error_t * elog_save_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:719
int elog_post_mortem_dump
Definition: main.h:222
uword dispatch_pcap_enable
Definition: main.h:145
#define PREDICT_TRUE(x)
Definition: clib.h:112
uword random_seed
Definition: main.h:188
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
static void vlib_increment_main_loop_counter(vlib_main_t *vm)
Definition: main.h:357
elog_time_stamp_t init_time
Timestamps.
Definition: elog.h:169
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static void vlib_process_sync_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:579
void clib_random_buffer_init(clib_random_buffer_t *b, uword seed)
Definition: random_buffer.c:62
u32 index
Definition: node.h:279
void(* vlib_node_runtime_perf_counter_cb)(struct vlib_main_t *, u64 *, u64 *)
Definition: main.h:97
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:255
static void dummy_queue_signal_callback(vlib_main_t *vm)
Definition: main.c:1964
u32 current_process_index
Definition: node.h:742
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
static u32 clib_get_current_cpu_id()
Definition: cpu.h:163
u32 *TW() tw_timer_expire_timers_vec(TWT(tw_timer_wheel)*tw, f64 now, u32 *vec)
u32 thread_index
Definition: main.h:197
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static_always_inline uword vlib_process_startup(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f)
Definition: main.c:1480
u32 main_loop_exit_set
Definition: main.h:104
static u64 dispatch_pending_node(vlib_main_t *vm, uword pending_frame_index, u64 last_time_stamp)
Definition: main.c:1325
u8 data[0]
Packet data.
Definition: buffer.h:181
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static u64 clib_cpu_time_now(void)
Definition: time.h:75
elog_track_t elog_track
Definition: threads.h:100
u32 clocks_since_last_overflow
Number of clock cycles.
Definition: node.h:475
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:560
int i
uword unformat_user(unformat_input_t *input, unformat_function_t *func,...)
Definition: unformat.c:983
static void vlib_process_update_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:693
static void elog_show_buffer_internal(vlib_main_t *vm, u32 n_events_to_show)
Definition: main.c:854
#define VLIB_MAIN_LOOP_EXIT_CLI
Definition: main.h:111
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
void clib_longjmp(clib_longjmp_t *save, uword return_value)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static vlib_frame_size_t * get_frame_size_info(vlib_node_main_t *nm, u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:95
static clib_error_t * elog_show_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:882
u8 * format_elog_event(u8 *s, va_list *va)
Definition: elog.c:296
u8 data[128]
Definition: ipsec.api:248
clib_time_t clib_time
Definition: main.h:72
u32 numa_node
Definition: main.h:199
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
clib_spinlock_t pending_interrupt_lock
Definition: node.h:712
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:366
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:236
uword clib_setjmp(clib_longjmp_t *save, uword return_value_not_taken)
#define vec_alloc(V, N)
Allocate space for N more elements (no header, unspecified alignment)
Definition: vec.h:280
unsigned char u8
Definition: types.h:56
static uword vlib_process_stack_is_valid(vlib_process_t *p)
Definition: main.c:1439
static vlib_frame_t * vlib_get_frame_no_check(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:217
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
u64 perf_counter0_ticks
Definition: node.h:235
double f64
Definition: types.h:142
u16 scalar_size
Definition: node.h:319
u8 state
Definition: node.h:307
u16 thread_index
thread this node runs on
Definition: node.h:520
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
volatile uword check_frame_queues
Definition: main.h:240
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:390
u32 * pending_interrupt_node_runtime_indices
Definition: node.h:711
u32 input_main_loops_per_call
For input nodes: decremented on each main loop interation until it reaches zero and function is calle...
Definition: node.h:497
static void vlib_main_loop(vlib_main_t *vm)
Definition: main.c:1920
#define foreach_weak_reference_stub
Definition: main.c:1968
vlib_trace_header_t ** trace_buffer_pool
Definition: trace.h:72
#define static_always_inline
Definition: clib.h:99
uword unformat_vlib_tmpfile(unformat_input_t *input, va_list *args)
Definition: format.c:192
PCAP main state data structure.
Definition: pcap.h:156
u32 trace_enable
Definition: trace.h:83
clib_spinlock_t lock
spinlock to protect e.g.
Definition: pcap.h:159
#define VLIB_FRAME_NO_APPEND
Definition: node.h:417
vlib_node_function_t * function
Node function to call.
Definition: node.h:467
void elog_init(elog_main_t *em, u32 n_events)
Definition: elog.c:493
#define VLIB_FRAME_MAGIC
#define always_inline
Definition: clib.h:98
u32 main_loop_vectors_processed
Definition: main.h:87
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:373
static u32 vlib_frame_alloc(vlib_main_t *vm, vlib_node_runtime_t *from_node_runtime, u32 to_next_index)
Definition: main.c:176
u16 log2_n_stack_bytes
Definition: node.h:576
vlib_node_t ** nodes
Definition: node.h:698
u32 vectors_since_last_overflow
Number of vector elements processed by this node.
Definition: node.h:485
vlib_frame_t * frame
Definition: main.c:1448
void di(unformat_input_t *i)
Definition: unformat.c:163
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:189
#define clib_error_return(e, args...)
Definition: error.h:99
static void dispatch_pcap_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: main.c:1027
#define VLIB_FRAME_ALIGN
Definition: node.h:377
u64 perf_counter_vectors
Definition: node.h:237
u32 cpu_id
Definition: main.h:198
static uword vlib_timing_wheel_data_is_timed_event(u32 d)
Definition: node.h:672
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:303
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
Definition: node.h:567
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:242
unsigned int u32
Definition: types.h:88
u64 cpu_time_main_loop_start
Definition: main.h:81
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:708
static u32 vlib_frame_bytes(u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:58
u32 * free_frame_indices
Definition: node.h:541
#define VLIB_MAIN_LOOP_EXIT_NONE
Definition: main.h:108
clib_error_t * vlib_node_main_init(vlib_main_t *vm)
Definition: node.c:631
#define VLIB_FRAME_SIZE
Definition: node.h:376
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:294
u32 perf_counter0_ticks_since_last_overflow
Perf counter 0 ticks.
Definition: node.h:488
u8 * pcap_buffer
Definition: main.h:146
#define VLIB_PROCESS_IS_RUNNING
Definition: node.h:573
#define VLIB_PROCESS_RETURN_LONGJMP_RETURN
Definition: node.h:558
u64 max_clock_n
Definition: node.h:234
unformat_function_t unformat_line_input
Definition: format.h:282
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
u32 calls_since_last_overflow
Number of calls.
Definition: node.h:483
char * name
Definition: main.h:117
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:36
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
static clib_error_t * elog_write_file(elog_main_t *em, char *clib_file, int flush_ring)
Definition: elog.h:529
u32 next_frame_index
Definition: node.h:457
static void * vlib_process_signal_event_helper(vlib_node_main_t *nm, vlib_node_t *n, vlib_process_t *p, uword t, uword n_data_elts, uword n_data_elt_bytes)
Definition: node_funcs.h:773
vlib_node_stats_t stats_total
Definition: node.h:269
static void vlib_put_next_frame_validate(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 next_index, u32 n_vectors_left)
Definition: main.c:429
u16 state
Input node state.
Definition: node.h:510
static u8 * format_buffer_metadata(u8 *s, va_list *args)
Definition: main.c:1006
u32 frame_index
Definition: node.h:404
format_function_t format_vlib_trace
Definition: trace.h:92
vlib_signal_timed_event_data_t * signal_timed_event_data_pool
Definition: node.h:729
static u32 vlib_frame_index(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:252
u16 frame_flags
Definition: node.h:383
#define VLIB_FRAME_IS_ALLOCATED
Definition: node.h:424
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:413
u8 * pcap_data
Vector of pcap data.
Definition: pcap.h:184
struct _unformat_input_t unformat_input_t
static_always_inline u64 dispatch_node(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_node_type_t type, vlib_node_state_t dispatch_state, vlib_frame_t *frame, u64 last_time_stamp)
Definition: main.c:1134
u32 n_alloc_frames
Definition: node.h:538
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:198
static clib_error_t * pcap_dispatch_trace_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:2286
static uword vlib_timing_wheel_data_get_index(u32 d)
Definition: node.h:690
u8 inline_event_data[64-3 *sizeof(u32)-2 *sizeof(u16)]
Definition: node.h:663
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
elog_event_type_t * node_return_elog_event_types
Definition: main.h:183
#define TWT(a)
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:218
u32 polling_threshold_vector_length
Definition: node.h:717
#define ELOG_DATA(em, f)
Definition: elog.h:481
u64 * n_vectors_by_next_node
Definition: node.h:342
#define PREDICT_FALSE(x)
Definition: clib.h:111
f64 seconds_per_clock
Definition: time.h:57
void elog_post_mortem_dump(void)
Definition: main.c:756
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:238
u8 protocol_hint
Definition: node.h:313
u32 node_index
Node index.
Definition: node.h:495
#define foreach_vlib_main(body)
Definition: threads.h:235
uword * init_functions_called
Definition: main.h:194
static void * pcap_add_packet(pcap_main_t *pm, f64 time_now, u32 n_bytes_in_trace, u32 n_bytes_in_packet)
Add packet.
Definition: pcap_funcs.h:37
#define VLIB_PROCESS_STACK_MAGIC
Definition: node.h:624
#define VLIB_PROCESS_RESUME_LONGJMP_RESUME
Definition: node.h:564
void clib_time_init(clib_time_t *c)
Definition: time.c:178
uword * frame_size_hash
Definition: node.h:754
void elog_alloc(elog_main_t *em, u32 n_events)
Definition: elog.c:479
clib_error_t * vlibmemory_init(vlib_main_t *vm)
Definition: memory_api.c:953
volatile u32 queue_signal_pending
Definition: main.h:210
clib_error_t * map_api_segment_init(vlib_main_t *vm)
Definition: memory_api.c:482
u8 * name
Definition: node.h:263
static u64 dispatch_process(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f, u64 last_time_stamp)
Definition: main.c:1511
u32 wraps
Definition: main.c:55
static u32 vlib_node_runtime_update_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:630
static u32 vlib_frame_index_no_check(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:225
u32 owner_node_index
Definition: node.h:353
static void vlib_node_runtime_perf_counter(vlib_main_t *vm, u64 *pmc0, u64 *pmc1)
Definition: main.c:684
void vlib_register_all_static_nodes(vlib_main_t *vm)
Definition: node.c:540
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:216
static u32 vlib_node_runtime_update_main_loop_vector_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_vectors)
Definition: node_funcs.h:1059
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
svmdb_client_t * c
u16 n_vectors
Definition: node.h:395
u32 runtime_index
Definition: node.h:282
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
vlib_main_t * vm
Definition: buffer.c:312
int elog_trace_graph_dispatch
Definition: main.h:177
static uword elog_n_events_in_buffer(elog_main_t *em)
Return number of events in the event-log buffer.
Definition: elog.h:188
static_always_inline void vlib_main_or_worker_loop(vlib_main_t *vm, int is_main)
Definition: main.c:1674
u32 node_runtime_index
Definition: node.h:451
vlib_pending_frame_t * pending_frames
Definition: node.h:724
u32 flow_id
Generic flow identifier.
Definition: buffer.h:127
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1598
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
clib_error_t * vpe_api_init(vlib_main_t *vm)
Definition: api.c:557
#define VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:292
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:163
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_PROCESS_RESUME_PENDING
Definition: node.h:570
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
u32 perf_counter_vectors_since_last_overflow
Perf counter vectors.
Definition: node.h:490
#define VLIB_BUFFER_TRACE_TRAJECTORY
Compile time buffer trajectory tracing option Turn this on if you run into "bad monkey" contexts...
Definition: buffer.h:442
elog_main_t elog_main
Definition: main.h:172
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:788
#define VLIB_FRAME_PENDING
Definition: node.h:427
static u32 vlib_frame_vector_byte_offset(u32 scalar_size)
Definition: node_funcs.h:263
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:147
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:465
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
clib_error_t * pcap_write(pcap_main_t *pm)
Write PCAP file.
Definition: pcap.c:89
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:241
u32 n_total_events_disable_limit
When count reaches limit logging is disabled.
Definition: elog.h:138
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:155
#define VLIB_PCAP_MINOR_VERSION
Definition: main.h:402
#define never_inline
Definition: clib.h:95
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define uword_to_pointer(u, type)
Definition: types.h:136
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:514
#define pool_put_index(p, i)
Free pool element with given index.
Definition: pool.h:311
#define ASSERT(truth)
u64 last_cpu_time
Definition: time.h:50
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:551
static void elog_reset_buffer(elog_main_t *em)
Reset the event buffer.
Definition: elog.h:207
u64 perf_counter1_ticks
Definition: node.h:236
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:326
#define ELOG_TRACK_DATA(em, f, track)
Definition: elog.h:475
uword event_ring_size
Power of 2 number of elements in ring.
Definition: elog.h:144
u16 flags
Definition: node.h:386
u32 main_loop_nodes_processed
Definition: main.h:88
u64 cpu
CPU cycle counter.
Definition: elog.h:125
#define clib_error_report(e)
Definition: error.h:113
u32 elog_trace_graph_circuit_node_index
Definition: main.h:179
vlib_trace_main_t trace_main
Definition: main.h:141
u8 * format_vnet_buffer_opaque(u8 *s, va_list *args)
Definition: main.c:990
u8 * vnet_trace_dummy
Definition: trace.c:43
u8 * format_vnet_buffer_opaque2(u8 *s, va_list *args)
Definition: main.c:999
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static u32 vlib_frame_alloc_to_node(vlib_main_t *vm, u32 to_node_index, u32 frame_flags)
Definition: main.c:121
#define A(x)
Definition: main.c:1024
void(* queue_signal_callback)(struct vlib_main_t *)
Definition: main.h:212
void vlib_gdb_show_event_log(void)
Definition: main.c:911
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static_always_inline uword vlib_process_resume(vlib_process_t *p)
Definition: main.c:1498
#define VLIB_FRAME_OWNER
Definition: node.h:421
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:593
#define vec_elt(v, i)
Get vector value at index i.
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
Definition: main.c:1982
#define unformat_parse_error(input)
Definition: format.h:268
pcap_packet_type_t packet_type
Packet type.
Definition: pcap.h:168
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1926
static vlib_node_t * vlib_get_next_node(vlib_main_t *vm, u32 node_index, u32 next_index)
Get vlib node by graph arc (next) index.
Definition: node_funcs.h:72
static clib_error_t * pcap_dispatch_trace_command_internal(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd, int rx_tx)
Definition: main.c:2129
void vl_api_send_pending_rpc_requests(vlib_main_t *)
Definition: main.c:1668
#define clib_mem_alloc_aligned_no_fail(size, align)
Definition: mem.h:166
CJ_GLOBAL_LOG_PROTOTYPE
Definition: main.c:49
u32 * data_from_advancing_timing_wheel
Definition: node.h:732
clib_error_t * vlib_physmem_init(vlib_main_t *vm)
Definition: physmem.c:94
u32 perf_counter1_ticks_since_last_overflow
Perf counter 1 ticks.
Definition: node.h:489
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 input_node_counts_by_state[VLIB_N_NODE_STATE]
Definition: node.h:751
vlib_pending_frame_t * suspended_process_frames
Definition: node.h:745
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
vlib_node_main_t node_main
Definition: main.h:135
static u32 * vlib_frame_find_magic(vlib_frame_t *f, vlib_node_t *node)
Definition: main.c:83
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
u32 owner_next_index
Definition: node.h:353
u8 vector_size
Definition: node.h:392
vlib_next_frame_t * next_frames
Definition: node.h:721
static clib_error_t * elog_stop(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:782
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
clib_error_t * vlib_call_all_config_functions(vlib_main_t *vm, unformat_input_t *input, int is_early)
Definition: init.c:94
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:274
typedef key
Definition: ipsec.api:244
static uword elog_is_enabled(elog_main_t *em)
event logging enabled predicate
Definition: elog.h:273
#define VLIB_FRAME_TRACE
Definition: node.h:433
u32 stop_timer_handle
Definition: node.h:606
clib_error_t * vlib_call_all_init_functions(vlib_main_t *vm)
Definition: init.c:67
#define VLIB_FRAME_SIZE_EXTRA
unformat_function_t unformat_vlib_node
Definition: node_funcs.h:1177
static void elog_track(elog_main_t *em, elog_event_type_t *type, elog_track_t *track, u32 data)
Log a single-datum event to a specific track, non-inline version.
Definition: elog.h:395
static u64 dispatch_suspended_process(vlib_main_t *vm, uword process_index, u64 last_time_stamp)
Definition: main.c:1592
vlib_frame_size_t * frame_sizes
Definition: node.h:757
u64 resume_clock_interval
Definition: node.h:603
static void add_trajectory_trace(vlib_buffer_t *b, u32 node_index)
Definition: main.c:970
u16 flags
Definition: node.h:566
elog_event_type_t * node_call_elog_event_types
Definition: main.h:182
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:140
vlib_node_type_t type
Definition: node.h:276
static vlib_process_t * vlib_get_process_from_node(vlib_main_t *vm, vlib_node_t *node)
Definition: node_funcs.h:208
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
u32 name_elog_string
Definition: node.h:266
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1487
vlib_node_state_t
Definition: node.h:249
pcap_main_t dispatch_pcap_main
Definition: main.h:144
u64 init_cpu_time
Definition: time.h:62
u32 suspended_process_frame_index
Definition: node.h:578
static void validate_frame_magic(vlib_main_t *vm, vlib_frame_t *f, vlib_node_t *n, uword next_index)
Definition: main.c:354
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static clib_error_t * vlib_main_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: main.c:1934
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
static clib_error_t * vlib_cli_elog_clear(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:702
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
Definition: node.h:568
#define vec_foreach(var, vec)
Vector iterator.
elog_track_t default_track
Default track.
Definition: elog.h:163
clib_longjmp_t return_longjmp
Definition: node.h:556
u16 flags
Copy of main node flags.
Definition: node.h:508
u32 node_runtime_index
Definition: node.h:407
u32 n_total_events
Total number of events in buffer.
Definition: elog.h:134
u8 scalar_size
Definition: node.h:389
clib_longjmp_t main_loop_exit
Definition: main.h:107
clib_longjmp_t resume_longjmp
Definition: node.h:562
void clib_mem_trace(int enable)
Definition: mem_dlmalloc.c:404
static clib_error_t * elog_resize(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:822
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:301
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
#define TW(a)
clib_error_t * vlib_map_stat_segment_init(void)
Definition: stat_segment.c:223
static u32 vlib_timing_wheel_data_set_suspended_process(u32 i)
Definition: node.h:678
volatile void(* worker_thread_main_loop_callback)(struct vlib_main_t *)
Definition: main.h:216
void vlib_start_process(vlib_main_t *vm, uword process_index)
Definition: main.c:1584
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:725
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
void * timing_wheel
Definition: node.h:727
u8 * format_vnet_buffer_flags(u8 *s, va_list *args)
Definition: main.c:982
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
int elog_trace_graph_circuit
Definition: main.h:178
u32 n_packets_captured
Number of packets currently captured.
Definition: pcap.h:171
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
clib_random_buffer_t random_buffer
Definition: main.h:191
static clib_error_t * elog_restart(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:802
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:304
u8 *(* validate_frame)(struct vlib_main_t *vm, struct vlib_node_runtime_t *, struct vlib_frame_t *f)
Definition: node.h:363
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
Definition: defs.h:46
clib_error_t * main_loop_error
Definition: main.h:114
vlib_process_t * process
Definition: main.c:1447
elog_event_t * elog_peek_events(elog_main_t *em)
convert event ring events to events, and return them as a vector.
Definition: elog.c:536
static uword elog_buffer_capacity(elog_main_t *em)
Return number of events which can fit in the event buffer.
Definition: elog.h:198
u64 cpu_time_last_node_dispatch
Definition: main.h:78
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:430
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:237
u32 vectors_since_last_overflow
Definition: node.h:436
u32 max_clock_n
Number of vectors in the recorded max_clock.
Definition: node.h:480
#define VLIB_PROCESS_RETURN_LONGJMP_SUSPEND
Definition: node.h:559