FD.io VPP  v17.10-9-gd594711
Vector Packet Processing
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * main.c: main vector processing loop
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <math.h>
41 #include <vppinfra/format.h>
42 #include <vlib/vlib.h>
43 #include <vlib/threads.h>
45 
46 #include <vlib/unix/unix.h>
47 #include <vlib/unix/cj.h>
48 
50 
51 /* Actually allocate a few extra slots of vector data to support
52  speculative vector enqueues which overflow vector data in next frame. */
53 #define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4)
54 
56 
58 vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes)
59 {
60  u32 n_bytes;
61 
62  /* Make room for vlib_frame_t plus scalar arguments. */
63  n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes);
64 
65  /* Make room for vector arguments.
66  Allocate a few extra slots of vector data to support
67  speculative vector enqueues which overflow vector data in next frame. */
68 #define VLIB_FRAME_SIZE_EXTRA 4
69  n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes;
70 
71  /* Magic number is first 32bit number after vector data.
72  Used to make sure that vector data is never overrun. */
73 #define VLIB_FRAME_MAGIC (0xabadc0ed)
74  n_bytes += sizeof (u32);
75 
76  /* Pad to cache line. */
77  n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES);
78 
79  return n_bytes;
80 }
81 
84 {
85  void *p = f;
86 
88 
90 
91  return p;
92 }
93 
94 static vlib_frame_size_t *
96  u32 n_scalar_bytes, u32 n_vector_bytes)
97 {
98  uword key = (n_scalar_bytes << 16) | n_vector_bytes;
99  uword *p, i;
100 
101  p = hash_get (nm->frame_size_hash, key);
102  if (p)
103  i = p[0];
104  else
105  {
106  i = vec_len (nm->frame_sizes);
107  vec_validate (nm->frame_sizes, i);
108  hash_set (nm->frame_size_hash, key, i);
109  }
110 
111  return vec_elt_at_index (nm->frame_sizes, i);
112 }
113 
114 static u32
116  u32 frame_flags)
117 {
118  vlib_node_main_t *nm = &vm->node_main;
119  vlib_frame_size_t *fs;
120  vlib_node_t *to_node;
121  vlib_frame_t *f;
122  u32 fi, l, n, scalar_size, vector_size;
123 
124  to_node = vlib_get_node (vm, to_node_index);
125 
126  scalar_size = to_node->scalar_size;
127  vector_size = to_node->vector_size;
128 
129  fs = get_frame_size_info (nm, scalar_size, vector_size);
130  n = vlib_frame_bytes (scalar_size, vector_size);
131  if ((l = vec_len (fs->free_frame_indices)) > 0)
132  {
133  /* Allocate from end of free list. */
134  fi = fs->free_frame_indices[l - 1];
135  f = vlib_get_frame_no_check (vm, fi);
136  _vec_len (fs->free_frame_indices) = l - 1;
137  }
138  else
139  {
141  fi = vlib_frame_index_no_check (vm, f);
142  }
143 
144  /* Poison frame when debugging. */
145  if (CLIB_DEBUG > 0)
146  memset (f, 0xfe, n);
147 
148  /* Insert magic number. */
149  {
150  u32 *magic;
151 
152  magic = vlib_frame_find_magic (f, to_node);
153  *magic = VLIB_FRAME_MAGIC;
154  }
155 
156  f->flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
157  f->n_vectors = 0;
158  f->scalar_size = scalar_size;
159  f->vector_size = vector_size;
160 
161  fs->n_alloc_frames += 1;
162 
163  return fi;
164 }
165 
166 /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX.
167  Returns frame index. */
168 static u32
170  u32 to_next_index)
171 {
172  vlib_node_t *from_node;
173 
174  from_node = vlib_get_node (vm, from_node_runtime->node_index);
175  ASSERT (to_next_index < vec_len (from_node->next_nodes));
176 
177  return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index],
178  /* frame_flags */ 0);
179 }
180 
181 vlib_frame_t *
183 {
184  u32 fi = vlib_frame_alloc_to_node (vm, to_node_index,
185  /* frame_flags */
187  return vlib_get_frame (vm, fi);
188 }
189 
190 void
192 {
194  vlib_node_t *to_node;
195 
196  if (f->n_vectors == 0)
197  return;
198 
199  to_node = vlib_get_node (vm, to_node_index);
200 
201  vec_add2 (vm->node_main.pending_frames, p, 1);
202 
204  p->frame_index = vlib_frame_index (vm, f);
205  p->node_runtime_index = to_node->runtime_index;
207 }
208 
209 /* Free given frame. */
210 void
212 {
213  vlib_node_main_t *nm = &vm->node_main;
214  vlib_node_t *node;
215  vlib_frame_size_t *fs;
216  u32 frame_index;
217 
219 
220  node = vlib_get_node (vm, r->node_index);
221  fs = get_frame_size_info (nm, node->scalar_size, node->vector_size);
222 
223  frame_index = vlib_frame_index (vm, f);
224 
226 
227  /* No next frames may point to freed frame. */
228  if (CLIB_DEBUG > 0)
229  {
230  vlib_next_frame_t *nf;
232  ASSERT (nf->frame_index != frame_index);
233  }
234 
236 
237  vec_add1 (fs->free_frame_indices, frame_index);
238  ASSERT (fs->n_alloc_frames > 0);
239  fs->n_alloc_frames -= 1;
240 }
241 
242 static clib_error_t *
244  unformat_input_t * input, vlib_cli_command_t * cmd)
245 {
246  vlib_node_main_t *nm = &vm->node_main;
247  vlib_frame_size_t *fs;
248 
249  vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free");
250  vec_foreach (fs, nm->frame_sizes)
251  {
252  u32 n_alloc = fs->n_alloc_frames;
253  u32 n_free = vec_len (fs->free_frame_indices);
254 
255  if (n_alloc + n_free > 0)
256  vlib_cli_output (vm, "%=6d%=12d%=12d",
257  fs - nm->frame_sizes, n_alloc, n_free);
258  }
259 
260  return 0;
261 }
262 
263 /* *INDENT-OFF* */
264 VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
265  .path = "show vlib frame-allocation",
266  .short_help = "Show node dispatch frame statistics",
267  .function = show_frame_stats,
268 };
269 /* *INDENT-ON* */
270 
271 /* Change ownership of enqueue rights to given next node. */
272 static void
274  vlib_node_runtime_t * node_runtime,
275  u32 next_index)
276 {
277  vlib_node_main_t *nm = &vm->node_main;
278  vlib_next_frame_t *next_frame;
279  vlib_node_t *node, *next_node;
280 
281  node = vec_elt (nm->nodes, node_runtime->node_index);
282 
283  /* Only internal & input nodes are allowed to call other nodes. */
285  || node->type == VLIB_NODE_TYPE_INPUT
286  || node->type == VLIB_NODE_TYPE_PROCESS);
287 
288  ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes);
289 
290  next_frame =
291  vlib_node_runtime_get_next_frame (vm, node_runtime, next_index);
292  next_node = vec_elt (nm->nodes, node->next_nodes[next_index]);
293 
294  if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX)
295  {
296  /* Get frame from previous owner. */
297  vlib_next_frame_t *owner_next_frame;
298  vlib_next_frame_t tmp;
299 
300  owner_next_frame =
302  next_node->owner_node_index,
303  next_node->owner_next_index);
304 
305  /* Swap target next frame with owner's. */
306  tmp = owner_next_frame[0];
307  owner_next_frame[0] = next_frame[0];
308  next_frame[0] = tmp;
309 
310  /*
311  * If next_frame is already pending, we have to track down
312  * all pending frames and fix their next_frame_index fields.
313  */
314  if (next_frame->flags & VLIB_FRAME_PENDING)
315  {
317  if (next_frame->frame_index != ~0)
318  {
319  vec_foreach (p, nm->pending_frames)
320  {
321  if (p->frame_index == next_frame->frame_index)
322  {
323  p->next_frame_index =
324  next_frame - vm->node_main.next_frames;
325  }
326  }
327  }
328  }
329  }
330  else
331  {
332  /* No previous owner. Take ownership. */
333  next_frame->flags |= VLIB_FRAME_OWNER;
334  }
335 
336  /* Record new owner. */
337  next_node->owner_node_index = node->index;
338  next_node->owner_next_index = next_index;
339 
340  /* Now we should be owner. */
341  ASSERT (next_frame->flags & VLIB_FRAME_OWNER);
342 }
343 
344 /* Make sure that magic number is still there.
345  Otherwise, it is likely that caller has overrun frame arguments. */
346 always_inline void
348  vlib_frame_t * f, vlib_node_t * n, uword next_index)
349 {
350  vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]);
351  u32 *magic = vlib_frame_find_magic (f, next_node);
352  ASSERT (VLIB_FRAME_MAGIC == magic[0]);
353 }
354 
355 vlib_frame_t *
357  vlib_node_runtime_t * node,
358  u32 next_index, u32 allocate_new_next_frame)
359 {
360  vlib_frame_t *f;
361  vlib_next_frame_t *nf;
362  u32 n_used;
363 
364  nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
365 
366  /* Make sure this next frame owns right to enqueue to destination frame. */
367  if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER)))
368  vlib_next_frame_change_ownership (vm, node, next_index);
369 
370  /* ??? Don't need valid flag: can use frame_index == ~0 */
372  {
373  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
375  }
376 
377  f = vlib_get_frame (vm, nf->frame_index);
378 
379  /* Has frame been removed from pending vector (e.g. finished dispatching)?
380  If so we can reuse frame. */
381  if ((nf->flags & VLIB_FRAME_PENDING) && !(f->flags & VLIB_FRAME_PENDING))
382  {
383  nf->flags &= ~VLIB_FRAME_PENDING;
384  f->n_vectors = 0;
385  }
386 
387  /* Allocate new frame if current one is already full. */
388  n_used = f->n_vectors;
389  if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0))
390  {
391  /* Old frame may need to be freed after dispatch, since we'll have
392  two redundant frames from node -> next node. */
394  {
395  vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame_index);
397  }
398 
399  /* Allocate new frame to replace full one. */
400  nf->frame_index = vlib_frame_alloc (vm, node, next_index);
401  f = vlib_get_frame (vm, nf->frame_index);
402  n_used = f->n_vectors;
403  }
404 
405  /* Should have free vectors in frame now. */
406  ASSERT (n_used < VLIB_FRAME_SIZE);
407 
408  if (CLIB_DEBUG > 0)
409  {
410  validate_frame_magic (vm, f,
411  vlib_get_node (vm, node->node_index), next_index);
412  }
413 
414  return f;
415 }
416 
417 static void
419  vlib_node_runtime_t * rt,
420  u32 next_index, u32 n_vectors_left)
421 {
422  vlib_node_main_t *nm = &vm->node_main;
423  vlib_next_frame_t *nf;
424  vlib_frame_t *f;
425  vlib_node_runtime_t *next_rt;
426  vlib_node_t *next_node;
427  u32 n_before, n_after;
428 
429  nf = vlib_node_runtime_get_next_frame (vm, rt, next_index);
430  f = vlib_get_frame (vm, nf->frame_index);
431 
432  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
433  n_after = VLIB_FRAME_SIZE - n_vectors_left;
434  n_before = f->n_vectors;
435 
436  ASSERT (n_after >= n_before);
437 
439  nf->node_runtime_index);
440  next_node = vlib_get_node (vm, next_rt->node_index);
441  if (n_after > 0 && next_node->validate_frame)
442  {
443  u8 *msg = next_node->validate_frame (vm, rt, f);
444  if (msg)
445  {
446  clib_warning ("%v", msg);
447  ASSERT (0);
448  }
449  vec_free (msg);
450  }
451 }
452 
453 void
456  u32 next_index, u32 n_vectors_left)
457 {
458  vlib_node_main_t *nm = &vm->node_main;
459  vlib_next_frame_t *nf;
460  vlib_frame_t *f;
461  u32 n_vectors_in_frame;
462 
463  if (vm->buffer_main->callbacks_registered == 0 && CLIB_DEBUG > 0)
464  vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
465 
466  nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
467  f = vlib_get_frame (vm, nf->frame_index);
468 
469  /* Make sure that magic number is still there. Otherwise, caller
470  has overrun frame meta data. */
471  if (CLIB_DEBUG > 0)
472  {
473  vlib_node_t *node = vlib_get_node (vm, r->node_index);
474  validate_frame_magic (vm, f, node, next_index);
475  }
476 
477  /* Convert # of vectors left -> number of vectors there. */
478  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
479  n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left;
480 
481  f->n_vectors = n_vectors_in_frame;
482 
483  /* If vectors were added to frame, add to pending vector. */
484  if (PREDICT_TRUE (n_vectors_in_frame > 0))
485  {
487  u32 v0, v1;
488 
489  r->cached_next_index = next_index;
490 
491  if (!(f->flags & VLIB_FRAME_PENDING))
492  {
493  __attribute__ ((unused)) vlib_node_t *node;
494  vlib_node_t *next_node;
495  vlib_node_runtime_t *next_runtime;
496 
497  node = vlib_get_node (vm, r->node_index);
498  next_node = vlib_get_next_node (vm, r->node_index, next_index);
499  next_runtime = vlib_node_get_runtime (vm, next_node->index);
500 
501  vec_add2 (nm->pending_frames, p, 1);
502 
503  p->frame_index = nf->frame_index;
505  p->next_frame_index = nf - nm->next_frames;
506  nf->flags |= VLIB_FRAME_PENDING;
508 
509  /*
510  * If we're going to dispatch this frame on another thread,
511  * force allocation of a new frame. Otherwise, we create
512  * a dangling frame reference. Each thread has its own copy of
513  * the next_frames vector.
514  */
515  if (0 && r->thread_index != next_runtime->thread_index)
516  {
517  nf->frame_index = ~0;
519  }
520  }
521 
522  /* Copy trace flag from next_frame and from runtime. */
523  nf->flags |=
524  (nf->flags & VLIB_NODE_FLAG_TRACE) | (r->
526 
528  v1 = v0 + n_vectors_in_frame;
530  if (PREDICT_FALSE (v1 < v0))
531  {
532  vlib_node_t *node = vlib_get_node (vm, r->node_index);
533  vec_elt (node->n_vectors_by_next_node, next_index) += v0;
534  }
535  }
536 }
537 
538 /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */
539 never_inline void
542  uword n_calls, uword n_vectors, uword n_clocks)
543 {
544  vlib_node_t *n = vlib_get_node (vm, r->node_index);
545 
546  n->stats_total.calls += n_calls + r->calls_since_last_overflow;
547  n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
548  n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
551 
555 }
556 
557 always_inline void __attribute__ ((unused))
559  vlib_process_t * p,
560  uword n_calls, uword n_vectors, uword n_clocks)
561 {
562  vlib_node_runtime_t *rt = &p->node_runtime;
564  vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks);
565  n->stats_total.suspends += p->n_suspends;
566  p->n_suspends = 0;
567 }
568 
569 void
571 {
573 
574  if (n->type == VLIB_NODE_TYPE_PROCESS)
575  {
576  /* Nothing to do for PROCESS nodes except in main thread */
577  if (vm != &vlib_global_main)
578  return;
579 
582  p->n_suspends = 0;
583  rt = &p->node_runtime;
584  }
585  else
586  rt =
588  n->runtime_index);
589 
590  vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0);
591 
592  /* Sync up runtime next frame vector counters with main node structure. */
593  {
594  vlib_next_frame_t *nf;
595  uword i;
596  for (i = 0; i < rt->n_next_nodes; i++)
597  {
598  nf = vlib_node_runtime_get_next_frame (vm, rt, i);
602  }
603  }
604 }
605 
608  vlib_node_runtime_t * node,
609  uword n_calls,
610  uword n_vectors, uword n_clocks)
611 {
612  u32 ca0, ca1, v0, v1, cl0, cl1, r;
613 
614  cl0 = cl1 = node->clocks_since_last_overflow;
615  ca0 = ca1 = node->calls_since_last_overflow;
616  v0 = v1 = node->vectors_since_last_overflow;
617 
618  ca1 = ca0 + n_calls;
619  v1 = v0 + n_vectors;
620  cl1 = cl0 + n_clocks;
621 
622  node->calls_since_last_overflow = ca1;
623  node->clocks_since_last_overflow = cl1;
624  node->vectors_since_last_overflow = v1;
625  node->max_clock_n = node->max_clock > n_clocks ?
626  node->max_clock_n : n_vectors;
627  node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks;
628 
629  r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
630 
631  if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0))
632  {
633  node->calls_since_last_overflow = ca0;
634  node->clocks_since_last_overflow = cl0;
635  node->vectors_since_last_overflow = v0;
636  vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks);
637  }
638 
639  return r;
640 }
641 
642 always_inline void
644  vlib_process_t * p,
645  uword n_calls, uword n_vectors, uword n_clocks)
646 {
648  n_calls, n_vectors, n_clocks);
649 }
650 
651 static clib_error_t *
653  unformat_input_t * input, vlib_cli_command_t * cmd)
654 {
656  return 0;
657 }
658 
659 /* *INDENT-OFF* */
660 VLIB_CLI_COMMAND (elog_clear_cli, static) = {
661  .path = "event-logger clear",
662  .short_help = "Clear the event log",
663  .function = vlib_cli_elog_clear,
664 };
665 /* *INDENT-ON* */
666 
667 #ifdef CLIB_UNIX
668 static clib_error_t *
670  unformat_input_t * input, vlib_cli_command_t * cmd)
671 {
672  elog_main_t *em = &vm->elog_main;
673  char *file, *chroot_file;
674  clib_error_t *error = 0;
675 
676  if (!unformat (input, "%s", &file))
677  {
678  vlib_cli_output (vm, "expected file name, got `%U'",
679  format_unformat_error, input);
680  return 0;
681  }
682 
683  /* It's fairly hard to get "../oopsie" through unformat; just in case */
684  if (strstr (file, "..") || index (file, '/'))
685  {
686  vlib_cli_output (vm, "illegal characters in filename '%s'", file);
687  return 0;
688  }
689 
690  chroot_file = (char *) format (0, "/tmp/%s%c", file, 0);
691 
692  vec_free (file);
693 
694  vlib_cli_output (vm, "Saving %wd of %wd events to %s",
696  elog_buffer_capacity (em), chroot_file);
697 
699  error = elog_write_file (em, chroot_file, 1 /* flush ring */ );
701  vec_free (chroot_file);
702  return error;
703 }
704 
705 void
707 {
709  elog_main_t *em = &vm->elog_main;
710  u8 *filename;
711  clib_error_t *error;
712 
713  if (!vm->elog_post_mortem_dump)
714  return;
715 
716  filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0);
717  error = elog_write_file (em, (char *) filename, 1 /* flush ring */ );
718  if (error)
719  clib_error_report (error);
720  vec_free (filename);
721 }
722 
723 /* *INDENT-OFF* */
724 VLIB_CLI_COMMAND (elog_save_cli, static) = {
725  .path = "event-logger save",
726  .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
727  .function = elog_save_buffer,
728 };
729 /* *INDENT-ON* */
730 
731 static clib_error_t *
733  unformat_input_t * input, vlib_cli_command_t * cmd)
734 {
735  elog_main_t *em = &vm->elog_main;
736 
738 
739  vlib_cli_output (vm, "Stopped the event logger...");
740  return 0;
741 }
742 
743 /* *INDENT-OFF* */
744 VLIB_CLI_COMMAND (elog_stop_cli, static) = {
745  .path = "event-logger stop",
746  .short_help = "Stop the event-logger",
747  .function = elog_stop,
748 };
749 /* *INDENT-ON* */
750 
751 static clib_error_t *
753  unformat_input_t * input, vlib_cli_command_t * cmd)
754 {
755  elog_main_t *em = &vm->elog_main;
756 
758 
759  vlib_cli_output (vm, "Restarted the event logger...");
760  return 0;
761 }
762 
763 /* *INDENT-OFF* */
764 VLIB_CLI_COMMAND (elog_restart_cli, static) = {
765  .path = "event-logger restart",
766  .short_help = "Restart the event-logger",
767  .function = elog_restart,
768 };
769 /* *INDENT-ON* */
770 
771 static clib_error_t *
773  unformat_input_t * input, vlib_cli_command_t * cmd)
774 {
775  elog_main_t *em = &vm->elog_main;
776  u32 tmp;
777 
778  /* Stop the parade */
780 
781  if (unformat (input, "%d", &tmp))
782  {
783  elog_alloc (em, tmp);
785  }
786  else
787  return clib_error_return (0, "Must specify how many events in the ring");
788 
789  vlib_cli_output (vm, "Resized ring and restarted the event logger...");
790  return 0;
791 }
792 
793 /* *INDENT-OFF* */
794 VLIB_CLI_COMMAND (elog_resize_cli, static) = {
795  .path = "event-logger resize",
796  .short_help = "event-logger resize <nnn>",
797  .function = elog_resize,
798 };
799 /* *INDENT-ON* */
800 
801 #endif /* CLIB_UNIX */
802 
803 static void
805 {
806  elog_main_t *em = &vm->elog_main;
807  elog_event_t *e, *es;
808  f64 dt;
809 
810  /* Show events in VLIB time since log clock starts after VLIB clock. */
811  dt = (em->init_time.cpu - vm->clib_time.init_cpu_time)
813 
814  es = elog_peek_events (em);
815  vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es),
816  em->event_ring_size,
818  "running" : "stopped");
819  vec_foreach (e, es)
820  {
821  vlib_cli_output (vm, "%18.9f: %U",
822  e->time + dt, format_elog_event, em, e);
823  n_events_to_show--;
824  if (n_events_to_show == 0)
825  break;
826  }
827  vec_free (es);
828 
829 }
830 
831 static clib_error_t *
833  unformat_input_t * input, vlib_cli_command_t * cmd)
834 {
835  u32 n_events_to_show;
836  clib_error_t *error = 0;
837 
838  n_events_to_show = 250;
840  {
841  if (unformat (input, "%d", &n_events_to_show))
842  ;
843  else if (unformat (input, "all"))
844  n_events_to_show = ~0;
845  else
846  return unformat_parse_error (input);
847  }
848  elog_show_buffer_internal (vm, n_events_to_show);
849  return error;
850 }
851 
852 /* *INDENT-OFF* */
853 VLIB_CLI_COMMAND (elog_show_cli, static) = {
854  .path = "show event-logger",
855  .short_help = "Show event logger info",
856  .function = elog_show_buffer,
857 };
858 /* *INDENT-ON* */
859 
860 void
862 {
864 }
865 
866 static inline void
868  u32 node_index,
869  u64 time, u32 n_vectors, u32 is_return)
870 {
872  elog_main_t *em = &evm->elog_main;
873 
874  if (VLIB_ELOG_MAIN_LOOP && n_vectors)
875  elog_track (em,
876  /* event type */
877  vec_elt_at_index (is_return
880  node_index),
881  /* track */
883  elog_track : &em->default_track),
884  /* data to log */ n_vectors);
885 }
886 
887 void
889 {
890  vlib_node_main_t *vnm = &vm->node_main;
891  vlib_buffer_t *b;
892  u8 i, n;
893 
895  {
896  b = vlib_get_buffer (vm, bi);
897  n = b->pre_data[0];
898 
899  fformat (stderr, "Context trace for bi %d b 0x%llx, visited %d\n",
900  bi, b, n);
901 
902  if (n == 0 || n > 20)
903  {
904  fformat (stderr, "n is unreasonable\n");
905  return;
906  }
907 
908 
909  for (i = 0; i < n; i++)
910  {
911  u32 node_index;
912 
913  node_index = b->pre_data[i + 1];
914 
915  if (node_index > vec_len (vnm->nodes))
916  {
917  fformat (stderr, "Skip bogus node index %d\n", node_index);
918  continue;
919  }
920 
921  fformat (stderr, "%v (%d)\n", vnm->nodes[node_index]->name,
922  node_index);
923  }
924  }
925  else
926  {
927  fformat (stderr,
928  "in vlib/buffers.h, #define VLIB_BUFFER_TRACE_TRAJECTORY 1\n");
929  }
930 }
931 
932 
935  vlib_node_runtime_t * node,
936  vlib_node_type_t type,
937  vlib_node_state_t dispatch_state,
938  vlib_frame_t * frame, u64 last_time_stamp)
939 {
940  uword n, v;
941  u64 t;
942  vlib_node_main_t *nm = &vm->node_main;
943  vlib_next_frame_t *nf;
944 
945  if (CLIB_DEBUG > 0)
946  {
947  vlib_node_t *n = vlib_get_node (vm, node->node_index);
948  ASSERT (n->type == type);
949  }
950 
951  /* Only non-internal nodes may be disabled. */
952  if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state)
953  {
955  return last_time_stamp;
956  }
957 
958  if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT)
959  && dispatch_state != VLIB_NODE_STATE_INTERRUPT)
960  {
962  /* Only call node when count reaches zero. */
963  if (c)
964  {
965  node->input_main_loops_per_call = c - 1;
966  return last_time_stamp;
967  }
968  }
969 
970  /* Speculatively prefetch next frames. */
971  if (node->n_next_nodes > 0)
972  {
974  CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE);
975  }
976 
977  vm->cpu_time_last_node_dispatch = last_time_stamp;
978 
979  if (1 /* || vm->thread_index == node->thread_index */ )
980  {
981  vlib_main_t *stat_vm;
982 
983  stat_vm = /* vlib_mains ? vlib_mains[0] : */ vm;
984 
986  last_time_stamp,
987  frame ? frame->n_vectors : 0,
988  /* is_after */ 0);
989 
990  /*
991  * Turn this on if you run into
992  * "bad monkey" contexts, and you want to know exactly
993  * which nodes they've visited... See ixge.c...
994  */
995  if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
996  {
997  int i;
998  int log_index;
999  u32 *from;
1000  from = vlib_frame_vector_args (frame);
1001  for (i = 0; i < frame->n_vectors; i++)
1002  {
1003  vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
1004  ASSERT (b->pre_data[0] < 32);
1005  log_index = b->pre_data[0]++ + 1;
1006  b->pre_data[log_index] = node->node_index;
1007  }
1008  n = node->function (vm, node, frame);
1009  }
1010  else
1011  n = node->function (vm, node, frame);
1012 
1013  t = clib_cpu_time_now ();
1014 
1015  vlib_elog_main_loop_event (vm, node->node_index, t, n, /* is_after */
1016  1);
1017 
1018  vm->main_loop_vectors_processed += n;
1019  vm->main_loop_nodes_processed += n > 0;
1020 
1021  v = vlib_node_runtime_update_stats (stat_vm, node,
1022  /* n_calls */ 1,
1023  /* n_vectors */ n,
1024  /* n_clocks */ t - last_time_stamp);
1025 
1026  /* When in interrupt mode and vector rate crosses threshold switch to
1027  polling mode. */
1028  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT)
1029  || (dispatch_state == VLIB_NODE_STATE_POLLING
1030  && (node->flags
1032  {
1033 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1034  ELOG_TYPE_DECLARE (e) =
1035  {
1036  .function = (char *) __FUNCTION__,.format =
1037  "%s vector length %d, switching to %s",.format_args =
1038  "T4i4t4",.n_enum_strings = 2,.enum_strings =
1039  {
1040  "interrupt", "polling",},};
1041  struct
1042  {
1043  u32 node_name, vector_length, is_polling;
1044  } *ed;
1046 #endif
1047 
1048  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
1049  && v >= nm->polling_threshold_vector_length) &&
1050  !(node->flags &
1052  {
1053  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1054  n->state = VLIB_NODE_STATE_POLLING;
1055  node->state = VLIB_NODE_STATE_POLLING;
1056  node->flags &=
1058  node->flags |=
1060  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
1061  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
1062 
1063 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1064  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1065  w->elog_track);
1066  ed->node_name = n->name_elog_string;
1067  ed->vector_length = v;
1068  ed->is_polling = 1;
1069 #endif
1070  }
1071  else if (dispatch_state == VLIB_NODE_STATE_POLLING
1072  && v <= nm->interrupt_threshold_vector_length)
1073  {
1074  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1075  if (node->flags &
1077  {
1078  /* Switch to interrupt mode after dispatch in polling one more time.
1079  This allows driver to re-enable interrupts. */
1080  n->state = VLIB_NODE_STATE_INTERRUPT;
1081  node->state = VLIB_NODE_STATE_INTERRUPT;
1082  node->flags &=
1084  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -=
1085  1;
1086  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] +=
1087  1;
1088 
1089  }
1090  else
1091  {
1092  node->flags |=
1094 #ifdef DISPATCH_NODE_ELOG_REQUIRED
1095  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1096  w->elog_track);
1097  ed->node_name = n->name_elog_string;
1098  ed->vector_length = v;
1099  ed->is_polling = 0;
1100 #endif
1101  }
1102  }
1103  }
1104  }
1105 
1106  return t;
1107 }
1108 
1109 static u64
1110 dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
1111  u64 last_time_stamp)
1112 {
1113  vlib_node_main_t *nm = &vm->node_main;
1114  vlib_frame_t *f;
1115  vlib_next_frame_t *nf, nf_dummy;
1117  u32 restore_frame_index;
1119 
1120  /* See comment below about dangling references to nm->pending_frames */
1121  p = nm->pending_frames + pending_frame_index;
1122 
1124  p->node_runtime_index);
1125 
1126  f = vlib_get_frame (vm, p->frame_index);
1128  {
1129  /* No next frame: so use dummy on stack. */
1130  nf = &nf_dummy;
1131  nf->flags = f->flags & VLIB_NODE_FLAG_TRACE;
1132  nf->frame_index = ~p->frame_index;
1133  }
1134  else
1136 
1138 
1139  /* Force allocation of new frame while current frame is being
1140  dispatched. */
1141  restore_frame_index = ~0;
1142  if (nf->frame_index == p->frame_index)
1143  {
1144  nf->frame_index = ~0;
1147  restore_frame_index = p->frame_index;
1148  }
1149 
1150  /* Frame must be pending. */
1152  ASSERT (f->n_vectors > 0);
1153 
1154  /* Copy trace flag from next frame to node.
1155  Trace flag indicates that at least one vector in the dispatched
1156  frame is traced. */
1157  n->flags &= ~VLIB_NODE_FLAG_TRACE;
1158  n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0;
1159  nf->flags &= ~VLIB_FRAME_TRACE;
1160 
1161  last_time_stamp = dispatch_node (vm, n,
1163  VLIB_NODE_STATE_POLLING,
1164  f, last_time_stamp);
1165 
1166  f->flags &= ~VLIB_FRAME_PENDING;
1167 
1168  /* Frame is ready to be used again, so restore it. */
1169  if (restore_frame_index != ~0)
1170  {
1171  /*
1172  * We musn't restore a frame that is flagged to be freed. This
1173  * shouldn't happen since frames to be freed post dispatch are
1174  * those used when the to-node frame becomes full i.e. they form a
1175  * sort of queue of frames to a single node. If we get here then
1176  * the to-node frame and the pending frame *were* the same, and so
1177  * we removed the to-node frame. Therefore this frame is no
1178  * longer part of the queue for that node and hence it cannot be
1179  * it's overspill.
1180  */
1182 
1183  /*
1184  * NB: dispatching node n can result in the creation and scheduling
1185  * of new frames, and hence in the reallocation of nm->pending_frames.
1186  * Recompute p, or no supper. This was broken for more than 10 years.
1187  */
1188  p = nm->pending_frames + pending_frame_index;
1189 
1190  /*
1191  * p->next_frame_index can change during node dispatch if node
1192  * function decides to change graph hook up.
1193  */
1196 
1197  if (~0 == nf->frame_index)
1198  {
1199  /* no new frame has been assigned to this node, use the saved one */
1200  nf->frame_index = restore_frame_index;
1201  f->n_vectors = 0;
1202  }
1203  else
1204  {
1205  /* The node has gained a frame, implying packets from the current frame
1206  were re-queued to this same node. we don't need the saved one
1207  anymore */
1208  vlib_frame_free (vm, n, f);
1209  }
1210  }
1211  else
1212  {
1214  {
1216  vlib_frame_free (vm, n, f);
1217  }
1218  }
1219 
1220  return last_time_stamp;
1221 }
1222 
1225 {
1226  return p->stack[0] == VLIB_PROCESS_STACK_MAGIC;
1227 }
1228 
1229 typedef struct
1230 {
1235 
1236 /* Called in process stack. */
1237 static uword
1239 {
1241  vlib_main_t *vm;
1242  vlib_node_runtime_t *node;
1243  vlib_frame_t *f;
1244  vlib_process_t *p;
1245  uword n;
1246 
1248 
1249  vm = a->vm;
1250  p = a->process;
1251  f = a->frame;
1252  node = &p->node_runtime;
1253 
1254  n = node->function (vm, node, f);
1255 
1257 
1258  clib_longjmp (&p->return_longjmp, n);
1259 
1260  return n;
1261 }
1262 
1263 /* Called in main stack. */
1266 {
1268  uword r;
1269 
1270  a.vm = vm;
1271  a.process = p;
1272  a.frame = f;
1273 
1277  (void *) p->stack + (1 << p->log2_n_stack_bytes));
1278 
1279  return r;
1280 }
1281 
1284 {
1285  uword r;
1292  return r;
1293 }
1294 
1295 static u64
1297  vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
1298 {
1299  vlib_node_main_t *nm = &vm->node_main;
1300  vlib_node_runtime_t *node_runtime = &p->node_runtime;
1301  vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index);
1302  u64 t;
1303  uword n_vectors, is_suspend;
1304 
1305  if (node->state != VLIB_NODE_STATE_POLLING
1308  return last_time_stamp;
1309 
1311 
1312  t = last_time_stamp;
1313  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1314  f ? f->n_vectors : 0, /* is_after */ 0);
1315 
1316  /* Save away current process for suspend. */
1318 
1319  n_vectors = vlib_process_startup (vm, p, f);
1320 
1321  nm->current_process_index = ~0;
1322 
1324  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1325  if (is_suspend)
1326  {
1328 
1329  n_vectors = 0;
1331  pf->node_runtime_index = node->runtime_index;
1332  pf->frame_index = f ? vlib_frame_index (vm, f) : ~0;
1333  pf->next_frame_index = ~0;
1334 
1335  p->n_suspends += 1;
1337 
1339  {
1340  TWT (tw_timer_wheel) * tw =
1341  (TWT (tw_timer_wheel) *) nm->timing_wheel;
1342  p->stop_timer_handle =
1343  TW (tw_timer_start) (tw,
1345  (node->runtime_index) /* [sic] pool idex */ ,
1346  0 /* timer_id */ ,
1348  }
1349  }
1350  else
1352 
1353  t = clib_cpu_time_now ();
1354 
1355  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
1356  /* is_after */ 1);
1357 
1359  /* n_calls */ !is_suspend,
1360  /* n_vectors */ n_vectors,
1361  /* n_clocks */ t - last_time_stamp);
1362 
1363  return t;
1364 }
1365 
1366 void
1368 {
1369  vlib_node_main_t *nm = &vm->node_main;
1370  vlib_process_t *p = vec_elt (nm->processes, process_index);
1371  dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
1372 }
1373 
1374 static u64
1376  uword process_index, u64 last_time_stamp)
1377 {
1378  vlib_node_main_t *nm = &vm->node_main;
1379  vlib_node_runtime_t *node_runtime;
1380  vlib_node_t *node;
1381  vlib_frame_t *f;
1382  vlib_process_t *p;
1384  u64 t, n_vectors, is_suspend;
1385 
1386  t = last_time_stamp;
1387 
1388  p = vec_elt (nm->processes, process_index);
1390  return last_time_stamp;
1391 
1394 
1395  pf =
1398 
1399  node_runtime = &p->node_runtime;
1400  node = vlib_get_node (vm, node_runtime->node_index);
1401  f = pf->frame_index != ~0 ? vlib_get_frame (vm, pf->frame_index) : 0;
1402 
1403  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1404  f ? f->n_vectors : 0, /* is_after */ 0);
1405 
1406  /* Save away current process for suspend. */
1408 
1409  n_vectors = vlib_process_resume (p);
1410  t = clib_cpu_time_now ();
1411 
1412  nm->current_process_index = ~0;
1413 
1414  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1415  if (is_suspend)
1416  {
1417  /* Suspend it again. */
1418  n_vectors = 0;
1419  p->n_suspends += 1;
1421  {
1422  p->stop_timer_handle =
1423  TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1425  (node->runtime_index) /* [sic] pool idex */ ,
1426  0 /* timer_id */ ,
1428  }
1429  }
1430  else
1431  {
1435  }
1436 
1437  t = clib_cpu_time_now ();
1438  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend,
1439  /* is_after */ 1);
1440 
1442  /* n_calls */ !is_suspend,
1443  /* n_vectors */ n_vectors,
1444  /* n_clocks */ t - last_time_stamp);
1445 
1446  return t;
1447 }
1448 
1451 {
1452  vlib_node_main_t *nm = &vm->node_main;
1454  uword i;
1455  u64 cpu_time_now;
1457  u32 *last_node_runtime_indices = 0;
1458 
1459  /* Initialize pending node vector. */
1460  if (is_main)
1461  {
1462  vec_resize (nm->pending_frames, 32);
1463  _vec_len (nm->pending_frames) = 0;
1464  }
1465 
1466  /* Mark time of main loop start. */
1467  if (is_main)
1468  {
1469  cpu_time_now = vm->clib_time.last_cpu_time;
1470  vm->cpu_time_main_loop_start = cpu_time_now;
1471  }
1472  else
1473  cpu_time_now = clib_cpu_time_now ();
1474 
1475  /* Pre-allocate interupt runtime indices and lock. */
1477  vec_alloc (last_node_runtime_indices, 32);
1478  if (!is_main)
1480 
1481  /* Pre-allocate expired nodes. */
1486 
1487  /* Start all processes. */
1488  if (is_main)
1489  {
1490  uword i;
1491  nm->current_process_index = ~0;
1492  for (i = 0; i < vec_len (nm->processes); i++)
1493  cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
1494  cpu_time_now);
1495  }
1496 
1497  while (1)
1498  {
1500 
1501  if (!is_main)
1502  {
1504  vec_foreach (fqm, tm->frame_queue_mains)
1505  vlib_frame_queue_dequeue (vm, fqm);
1506  }
1507 
1508  /* Process pre-input nodes. */
1509  if (is_main)
1511  cpu_time_now = dispatch_node (vm, n,
1513  VLIB_NODE_STATE_POLLING,
1514  /* frame */ 0,
1515  cpu_time_now);
1516 
1517  /* Next process input nodes. */
1519  cpu_time_now = dispatch_node (vm, n,
1521  VLIB_NODE_STATE_POLLING,
1522  /* frame */ 0,
1523  cpu_time_now);
1524 
1525  if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
1526  vm->queue_signal_callback (vm);
1527 
1528  /* Next handle interrupts. */
1529  {
1530  uword l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1531  uword i;
1532  if (l > 0)
1533  {
1534  u32 *tmp;
1535  if (!is_main)
1539  last_node_runtime_indices;
1540  last_node_runtime_indices = tmp;
1541  _vec_len (last_node_runtime_indices) = 0;
1542  if (!is_main)
1544  for (i = 0; i < l; i++)
1545  {
1547  last_node_runtime_indices[i]);
1548  cpu_time_now =
1550  VLIB_NODE_STATE_INTERRUPT,
1551  /* frame */ 0,
1552  cpu_time_now);
1553  }
1554  }
1555  }
1556 
1557  if (is_main)
1558  {
1559  /* Check if process nodes have expired from timing wheel. */
1561 
1564  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
1566 
1568 
1569  if (PREDICT_FALSE
1570  (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
1571  {
1572  uword i;
1573 
1574  processes_timing_wheel_data:
1575  for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
1576  i++)
1577  {
1580 
1582  {
1585  di);
1586  vlib_node_t *n =
1588  vlib_process_t *p =
1589  vec_elt (nm->processes, n->runtime_index);
1590  void *data;
1591  data =
1593  te->event_type_index,
1594  te->n_data_elts,
1595  te->n_data_elt_bytes);
1596  if (te->n_data_bytes < sizeof (te->inline_event_data))
1597  clib_memcpy (data, te->inline_event_data,
1598  te->n_data_bytes);
1599  else
1600  {
1601  clib_memcpy (data, te->event_data_as_vector,
1602  te->n_data_bytes);
1604  }
1606  }
1607  else
1608  {
1609  cpu_time_now = clib_cpu_time_now ();
1610  cpu_time_now =
1611  dispatch_suspended_process (vm, di, cpu_time_now);
1612  }
1613  }
1614  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1615  }
1616  }
1617 
1618  /* Input nodes may have added work to the pending vector.
1619  Process pending vector until there is nothing left.
1620  All pending vectors will be processed from input -> output. */
1621  for (i = 0; i < _vec_len (nm->pending_frames); i++)
1622  cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
1623  /* Reset pending vector for next iteration. */
1624  _vec_len (nm->pending_frames) = 0;
1625 
1626  /* Pending internal nodes may resume processes. */
1627  if (is_main && _vec_len (nm->data_from_advancing_timing_wheel) > 0)
1628  goto processes_timing_wheel_data;
1629 
1631 
1632  /* Record time stamp in case there are no enabled nodes and above
1633  calls do not update time stamp. */
1634  cpu_time_now = clib_cpu_time_now ();
1635  }
1636 }
1637 
1638 static void
1640 {
1641  vlib_main_or_worker_loop (vm, /* is_main */ 1);
1642 }
1643 
1644 void
1646 {
1647  vlib_main_or_worker_loop (vm, /* is_main */ 0);
1648 }
1649 
1651 
1652 static clib_error_t *
1654 {
1655  int turn_on_mem_trace = 0;
1656 
1657  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1658  {
1659  if (unformat (input, "memory-trace"))
1660  turn_on_mem_trace = 1;
1661 
1662  else if (unformat (input, "elog-events %d",
1663  &vm->elog_main.event_ring_size))
1664  ;
1665  else if (unformat (input, "elog-post-mortem-dump"))
1666  vm->elog_post_mortem_dump = 1;
1667  else
1668  return unformat_parse_error (input);
1669  }
1670 
1671  unformat_free (input);
1672 
1673  /* Enable memory trace as early as possible. */
1674  if (turn_on_mem_trace)
1675  clib_mem_trace (1);
1676 
1677  return 0;
1678 }
1679 
1681 
1682 static void
1684 {
1685 }
1686 
1687 /* Main function. */
1688 int
1690 {
1691  clib_error_t *volatile error;
1692  vlib_node_main_t *nm = &vm->node_main;
1693 
1695 
1696  clib_time_init (&vm->clib_time);
1697 
1698  /* Turn on event log. */
1699  if (!vm->elog_main.event_ring_size)
1700  vm->elog_main.event_ring_size = 128 << 10;
1702  elog_enable_disable (&vm->elog_main, 1);
1703 
1704  /* Default name. */
1705  if (!vm->name)
1706  vm->name = "VLIB";
1707 
1708  if ((error = unix_physmem_init (vm)))
1709  {
1710  clib_error_report (error);
1711  goto done;
1712  }
1713 
1714  if ((error = vlib_buffer_main_init (vm)))
1715  {
1716  clib_error_report (error);
1717  goto done;
1718  }
1719 
1720  if ((error = vlib_thread_init (vm)))
1721  {
1722  clib_error_report (error);
1723  goto done;
1724  }
1725 
1726  /* Register static nodes so that init functions may use them. */
1728 
1729  /* Set seed for random number generator.
1730  Allow user to specify seed to make random sequence deterministic. */
1731  if (!unformat (input, "seed %wd", &vm->random_seed))
1732  vm->random_seed = clib_cpu_time_now ();
1734 
1735  /* Initialize node graph. */
1736  if ((error = vlib_node_main_init (vm)))
1737  {
1738  /* Arrange for graph hook up error to not be fatal when debugging. */
1739  if (CLIB_DEBUG > 0)
1740  clib_error_report (error);
1741  else
1742  goto done;
1743  }
1744 
1745  /* See unix/main.c; most likely already set up */
1746  if (vm->init_functions_called == 0)
1747  vm->init_functions_called = hash_create (0, /* value bytes */ 0);
1748  if ((error = vlib_call_all_init_functions (vm)))
1749  goto done;
1750 
1751  /* Create default buffer free list. */
1754  "default");
1755 
1756  nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
1758 
1760  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1761 
1762  /* Create the process timing wheel */
1763  TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1764  0 /* no callback */ ,
1765  10e-6 /* timer period 10us */ ,
1766  ~0 /* max expirations per call */ );
1767 
1769  {
1771  vm->main_loop_exit_set = 1;
1772  break;
1773 
1775  goto done;
1776 
1777  default:
1778  error = vm->main_loop_error;
1779  goto done;
1780  }
1781 
1782  if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
1783  goto done;
1784 
1785  /* Call all main loop enter functions. */
1786  {
1787  clib_error_t *sub_error;
1788  sub_error = vlib_call_all_main_loop_enter_functions (vm);
1789  if (sub_error)
1790  clib_error_report (sub_error);
1791  }
1792 
1793  vlib_main_loop (vm);
1794 
1795 done:
1796  /* Call all exit functions. */
1797  {
1798  clib_error_t *sub_error;
1799  sub_error = vlib_call_all_main_loop_exit_functions (vm);
1800  if (sub_error)
1801  clib_error_report (sub_error);
1802  }
1803 
1804  if (error)
1805  clib_error_report (error);
1806 
1807  return 0;
1808 }
1809 
1810 /*
1811  * fd.io coding-style-patch-verification: ON
1812  *
1813  * Local Variables:
1814  * eval: (c-set-style "gnu")
1815  * End:
1816  */
u32 * next_nodes
Definition: node.h:288
static void elog_enable_disable(elog_main_t *em, int is_enabled)
Enable or disable event logging.
Definition: elog.h:217
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:432
clib_error_t * vlib_call_all_main_loop_exit_functions(vlib_main_t *vm)
Definition: init.c:87
vlib_main_t vlib_global_main
Definition: main.c:1650
f64 time
Absolute time as floating point number in seconds.
Definition: elog.h:66
void TW() tw_timer_wheel_init(TWT(tw_timer_wheel)*tw, void *expired_timer_callback, f64 timer_interval_in_seconds, u32 max_expirations)
Initialize a tw timer wheel template instance.
u16 vector_size
Definition: node.h:274
u32 max_clock
Maximum clock cycle for an invocation.
Definition: node.h:423
u32 next_frame_index
Start of next frames for this node.
Definition: node.h:434
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:540
static clib_error_t * show_frame_stats(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:243
#define hash_set(h, key, value)
Definition: hash.h:254
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
clib_error_t * vlib_call_all_main_loop_enter_functions(vlib_main_t *vm)
Definition: init.c:80
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:72
void vlib_dump_context_trace(vlib_main_t *vm, u32 bi)
Definition: main.c:888
static void vlib_next_frame_change_ownership(vlib_main_t *vm, vlib_node_runtime_t *node_runtime, u32 next_index)
Definition: main.c:273
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:50
static vlib_next_frame_t * vlib_node_get_next_frame(vlib_main_t *vm, u32 node_index, u32 next_index)
Get pointer to frame by (node_index, next_index).
Definition: node_funcs.h:325
u32 interrupt_threshold_vector_length
Definition: node.h:660
vlib_process_t ** processes
Definition: node.h:681
static uword vlib_process_bootstrap(uword _a)
Definition: main.c:1238
vlib_node_runtime_t node_runtime
Definition: node.h:495
a
Definition: bitmap.h:516
u32 n_suspends
Definition: node.h:523
vlib_frame_t * vlib_get_next_frame_internal(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 allocate_new_next_frame)
Definition: main.c:356
static void vlib_elog_main_loop_event(vlib_main_t *vm, u32 node_index, u64 time, u32 n_vectors, u32 is_return)
Definition: main.c:867
static clib_error_t * elog_save_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:669
int elog_post_mortem_dump
Definition: main.h:196
#define VLIB_FRAME_OWNER
Definition: node.h:367
#define PREDICT_TRUE(x)
Definition: clib.h:98
uword random_seed
Definition: main.h:164
#define VLIB_FRAME_IS_ALLOCATED
Definition: node.h:370
static void vlib_increment_main_loop_counter(vlib_main_t *vm)
Definition: main.h:323
elog_time_stamp_t init_time
Timestamps.
Definition: elog.h:169
void clib_random_buffer_init(clib_random_buffer_t *b, uword seed)
Definition: random_buffer.c:62
u32 index
Definition: node.h:237
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:221
static void dummy_queue_signal_callback(vlib_main_t *vm)
Definition: main.c:1683
u32 current_process_index
Definition: node.h:684
u32 *TW() tw_timer_expire_timers_vec(TWT(tw_timer_wheel)*tw, f64 now, u32 *vec)
u32 thread_index
Definition: main.h:173
static_always_inline uword vlib_process_startup(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f)
Definition: main.c:1265
u32 main_loop_exit_set
Definition: main.h:87
static u64 dispatch_pending_node(vlib_main_t *vm, uword pending_frame_index, u64 last_time_stamp)
Definition: main.c:1110
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:518
static u64 clib_cpu_time_now(void)
Definition: time.h:73
elog_track_t elog_track
Definition: threads.h:116
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:362
u32 clocks_since_last_overflow
Number of clock cycles.
Definition: node.h:421
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:557
vlib_buffer_main_t * buffer_main
Definition: main.h:106
static void vlib_process_update_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:643
static void elog_show_buffer_internal(vlib_main_t *vm, u32 n_events_to_show)
Definition: main.c:804
#define VLIB_MAIN_LOOP_EXIT_CLI
Definition: main.h:94
void clib_longjmp(clib_longjmp_t *save, uword return_value)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
static vlib_frame_size_t * get_frame_size_info(vlib_node_main_t *nm, u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:95
static clib_error_t * elog_show_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:832
u8 * format_elog_event(u8 *s, va_list *va)
Definition: elog.c:296
clib_time_t clib_time
Definition: main.h:62
clib_spinlock_t pending_interrupt_lock
Definition: node.h:654
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:225
uword clib_setjmp(clib_longjmp_t *save, uword return_value_not_taken)
#define vec_alloc(V, N)
Allocate space for N more elements (no header, unspecified alignment)
Definition: vec.h:275
vlib_node_state_t
Definition: node.h:207
#define VLIB_PROCESS_RESUME_PENDING
Definition: node.h:512
static uword vlib_process_stack_is_valid(vlib_process_t *p)
Definition: main.c:1224
static vlib_frame_t * vlib_get_frame_no_check(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:216
u16 scalar_size
Definition: node.h:274
u8 state
Definition: node.h:265
u16 thread_index
thread this node runs on
Definition: node.h:462
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:212
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:392
u32 * pending_interrupt_node_runtime_indices
Definition: node.h:653
u32 input_main_loops_per_call
For input nodes: decremented on each main loop interation until it reaches zero and function is calle...
Definition: node.h:439
static void vlib_main_loop(vlib_main_t *vm)
Definition: main.c:1639
#define static_always_inline
Definition: clib.h:85
vlib_node_function_t * function
Node function to call.
Definition: node.h:413
void elog_init(elog_main_t *em, u32 n_events)
Definition: elog.c:466
#define VLIB_FRAME_MAGIC
#define always_inline
Definition: clib.h:84
u32 main_loop_vectors_processed
Definition: main.h:74
static u32 vlib_frame_alloc(vlib_main_t *vm, vlib_node_runtime_t *from_node_runtime, u32 to_next_index)
Definition: main.c:169
u16 log2_n_stack_bytes
Definition: node.h:518
vlib_node_t ** nodes
Definition: node.h:640
u32 vectors_since_last_overflow
Number of vector elements processed by this node.
Definition: node.h:431
vlib_frame_t * frame
Definition: main.c:1233
void di(unformat_input_t *i)
Definition: unformat.c:163
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:182
#define clib_error_return(e, args...)
Definition: error.h:99
#define VLIB_FRAME_ALIGN
Definition: node.h:329
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:149
static void vlib_process_sync_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:558
#define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES
Definition: buffer.h:423
unsigned long u64
Definition: types.h:89
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:237
u32 TW() tw_timer_start(TWT(tw_timer_wheel)*tw, u32 pool_index, u32 timer_id, u64 interval)
Start a Tw Timer.
u64 cpu_time_main_loop_start
Definition: main.h:68
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:650
static u32 vlib_frame_bytes(u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:58
u32 * free_frame_indices
Definition: node.h:483
#define VLIB_MAIN_LOOP_EXIT_NONE
Definition: main.h:91
#define VLIB_FRAME_TRACE
Definition: node.h:379
clib_error_t * vlib_node_main_init(vlib_main_t *vm)
Definition: node.c:547
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:290
static uword pointer_to_uword(const void *p)
Definition: types.h:131
u64 max_clock_n
Definition: node.h:195
int callbacks_registered
Definition: buffer.h:441
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:33
u32 calls_since_last_overflow
Number of calls.
Definition: node.h:429
char * name
Definition: main.h:100
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:35
#define hash_get(h, key)
Definition: hash.h:248
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:458
u32 next_frame_index
Definition: node.h:403
static void * vlib_process_signal_event_helper(vlib_node_main_t *nm, vlib_node_t *n, vlib_process_t *p, uword t, uword n_data_elts, uword n_data_elt_bytes)
Definition: node_funcs.h:769
vlib_node_stats_t stats_total
Definition: node.h:227
static void vlib_put_next_frame_validate(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 next_index, u32 n_vectors_left)
Definition: main.c:418
u16 state
Input node state.
Definition: node.h:452
u32 frame_index
Definition: node.h:353
vlib_signal_timed_event_data_t * signal_timed_event_data_pool
Definition: node.h:671
vlib_node_type_t
Definition: node.h:58
static u32 vlib_frame_index(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:245
#define v
Definition: acl.c:323
struct _unformat_input_t unformat_input_t
static_always_inline u64 dispatch_node(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_node_type_t type, vlib_node_state_t dispatch_state, vlib_frame_t *frame, u64 last_time_stamp)
Definition: main.c:934
u32 n_alloc_frames
Definition: node.h:480
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:191
static clib_error_t * elog_write_file(elog_main_t *em, char *unix_file, int flush_ring)
Definition: elog.h:528
u8 inline_event_data[64-3 *sizeof(u32)-2 *sizeof(u16)]
Definition: node.h:605
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:270
elog_event_type_t * node_return_elog_event_types
Definition: main.h:159
#define TWT(a)
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:211
u32 polling_threshold_vector_length
Definition: node.h:659
u64 * n_vectors_by_next_node
Definition: node.h:297
#define PREDICT_FALSE(x)
Definition: clib.h:97
#define VLIB_FRAME_PENDING
Definition: node.h:373
static u32 vlib_timing_wheel_data_set_suspended_process(u32 i)
Definition: node.h:620
#define VLIB_FRAME_SIZE
Definition: node.h:328
f64 seconds_per_clock
Definition: time.h:57
void elog_post_mortem_dump(void)
Definition: main.c:706
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, uword frame_index)
Definition: node_funcs.h:237
u32 node_index
Node index.
Definition: node.h:437
uword * init_functions_called
Definition: main.h:170
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:376
#define VLIB_ELOG_MAIN_LOOP
Definition: main.h:56
void clib_time_init(clib_time_t *c)
Definition: time.c:175
uword * frame_size_hash
Definition: node.h:696
void elog_alloc(elog_main_t *em, u32 n_events)
Definition: elog.c:452
volatile u32 queue_signal_pending
Definition: main.h:187
#define VLIB_PROCESS_RETURN_LONGJMP_SUSPEND
Definition: node.h:501
word fformat(FILE *f, char *fmt,...)
Definition: format.c:453
#define uword_to_pointer(u, type)
Definition: types.h:136
u8 * name
Definition: node.h:221
static u64 dispatch_process(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f, u64 last_time_stamp)
Definition: main.c:1296
u32 wraps
Definition: main.c:55
static u32 vlib_frame_index_no_check(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:224
u32 owner_node_index
Definition: node.h:308
void vlib_register_all_static_nodes(vlib_main_t *vm)
Definition: node.c:518
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:140
static u32 vlib_node_runtime_update_main_loop_vector_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_vectors)
Definition: node_funcs.h:1045
#define UNFORMAT_END_OF_INPUT
Definition: format.h:143
svmdb_client_t * c
u16 n_vectors
Definition: node.h:344
u32 runtime_index
Definition: node.h:240
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:82
vlib_main_t * vm
Definition: buffer.c:283
static uword elog_n_events_in_buffer(elog_main_t *em)
Return number of events in the event-log buffer.
Definition: elog.h:188
static_always_inline void vlib_main_or_worker_loop(vlib_main_t *vm, int is_main)
Definition: main.c:1450
u32 node_runtime_index
Definition: node.h:397
vlib_pending_frame_t * pending_frames
Definition: node.h:666
u32 vlib_buffer_get_or_create_free_list(vlib_main_t *vm, u32 n_data_bytes, char *fmt,...)
Definition: buffer.c:436
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1583
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
static u32 vlib_node_runtime_update_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:607
#define clib_warning(format, args...)
Definition: error.h:59
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
#define clib_memcpy(a, b, c)
Definition: string.h:69
#define VLIB_BUFFER_TRACE_TRAJECTORY
Compile time buffer trajectory tracing option Turn this on if you run into "bad monkey" contexts...
Definition: buffer.h:512
elog_main_t elog_main
Definition: main.h:155
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:1057
#define VLIB_PROCESS_RETURN_LONGJMP_RETURN
Definition: node.h:500
static u32 vlib_frame_vector_byte_offset(u32 scalar_size)
Definition: node_funcs.h:256
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:454
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:278
u32 n_total_events_disable_limit
When count reaches limit logging is disabled.
Definition: elog.h:138
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
#define never_inline
Definition: clib.h:81
#define hash_create(elts, value_bytes)
Definition: hash.h:658
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
Definition: node.h:510
static uword vlib_timing_wheel_data_is_timed_event(u32 d)
Definition: node.h:614
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:456
#define ASSERT(truth)
u64 last_cpu_time
Definition: time.h:50
unsigned int u32
Definition: types.h:88
static void elog_reset_buffer(elog_main_t *em)
Reset the event buffer.
Definition: elog.h:207
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:335
#define ELOG_TRACK_DATA(em, f, track)
Definition: elog.h:475
uword event_ring_size
Power of 2 number of elements in ring.
Definition: elog.h:144
#define VLIB_PROCESS_STACK_MAGIC
Definition: node.h:566
u16 flags
Definition: node.h:335
u32 main_loop_nodes_processed
Definition: main.h:75
u64 cpu
CPU cycle counter.
Definition: elog.h:125
#define VLIB_PROCESS_RESUME_LONGJMP_RESUME
Definition: node.h:506
#define clib_error_report(e)
Definition: error.h:113
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:259
static u32 vlib_frame_alloc_to_node(vlib_main_t *vm, u32 to_node_index, u32 frame_flags)
Definition: main.c:115
void(* queue_signal_callback)(struct vlib_main_t *)
Definition: main.h:189
void vlib_gdb_show_event_log(void)
Definition: main.c:861
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static_always_inline uword vlib_process_resume(vlib_process_t *p)
Definition: main.c:1283
u64 uword
Definition: types.h:112
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:570
#define vec_elt(v, i)
Get vector value at index i.
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
Definition: main.c:1689
#define unformat_parse_error(input)
Definition: format.h:267
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1645
static vlib_node_t * vlib_get_next_node(vlib_main_t *vm, u32 node_index, u32 next_index)
Get vlib node by graph arc (next) index.
Definition: node_funcs.h:72
#define clib_mem_alloc_aligned_no_fail(size, align)
Definition: mem.h:146
CJ_GLOBAL_LOG_PROTOTYPE
Definition: main.c:49
u32 * data_from_advancing_timing_wheel
Definition: node.h:674
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
double f64
Definition: types.h:142
u32 input_node_counts_by_state[VLIB_N_NODE_STATE]
Definition: node.h:693
unsigned char u8
Definition: types.h:56
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:406
vlib_pending_frame_t * suspended_process_frames
Definition: node.h:687
vlib_node_main_t node_main
Definition: main.h:129
static u32 * vlib_frame_find_magic(vlib_frame_t *f, vlib_node_t *node)
Definition: main.c:83
u32 owner_next_index
Definition: node.h:308
u8 vector_size
Definition: node.h:341
vlib_next_frame_t * next_frames
Definition: node.h:663
static clib_error_t * elog_stop(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:732
static void unformat_free(unformat_input_t *i)
Definition: format.h:161
clib_error_t * vlib_call_all_config_functions(vlib_main_t *vm, unformat_input_t *input, int is_early)
Definition: init.c:94
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:262
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:267
u32 stop_timer_handle
Definition: node.h:548
clib_error_t * vlib_call_all_init_functions(vlib_main_t *vm)
Definition: init.c:67
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
Definition: node.h:509
#define VLIB_FRAME_SIZE_EXTRA
static void elog_track(elog_main_t *em, elog_event_type_t *type, elog_track_t *track, u32 data)
Log a single-datum event to a specific track, non-inline version.
Definition: elog.h:395
clib_error_t * unix_physmem_init(vlib_main_t *vm)
Definition: physmem.c:224
static u64 dispatch_suspended_process(vlib_main_t *vm, uword process_index, u64 last_time_stamp)
Definition: main.c:1375
vlib_frame_size_t * frame_sizes
Definition: node.h:699
u64 resume_clock_interval
Definition: node.h:545
u16 flags
Definition: node.h:508
elog_event_type_t * node_call_elog_event_types
Definition: main.h:158
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:120
vlib_node_type_t type
Definition: node.h:234
static vlib_process_t * vlib_get_process_from_node(vlib_main_t *vm, vlib_node_t *node)
Definition: node_funcs.h:207
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
u32 name_elog_string
Definition: node.h:224
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1488
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:325
u64 init_cpu_time
Definition: time.h:60
u32 suspended_process_frame_index
Definition: node.h:520
static void validate_frame_magic(vlib_main_t *vm, vlib_frame_t *f, vlib_node_t *n, uword next_index)
Definition: main.c:347
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:261
static clib_error_t * vlib_main_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: main.c:1653
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
static clib_error_t * vlib_cli_elog_clear(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:652
static uword vlib_timing_wheel_data_get_index(u32 d)
Definition: node.h:632
#define vec_foreach(var, vec)
Vector iterator.
elog_track_t default_track
Default track.
Definition: elog.h:163
clib_longjmp_t return_longjmp
Definition: node.h:498
u16 flags
Copy of main node flags.
Definition: node.h:450
u32 node_runtime_index
Definition: node.h:356
u32 n_total_events
Total number of events in buffer.
Definition: elog.h:134
u8 scalar_size
Definition: node.h:338
clib_longjmp_t main_loop_exit
Definition: main.h:90
clib_longjmp_t resume_longjmp
Definition: node.h:504
void clib_mem_trace(int enable)
Definition: mem_mheap.c:154
static clib_error_t * elog_resize(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:772
u32 flags
Definition: vhost-user.h:77
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
#define TW(a)
void vlib_start_process(vlib_main_t *vm, uword process_index)
Definition: main.c:1367
#define VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:250
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:680
void * timing_wheel
Definition: node.h:669
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:57
clib_random_buffer_t random_buffer
Definition: main.h:167
static clib_error_t * elog_restart(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:752
u8 *(* validate_frame)(struct vlib_main_t *vm, struct vlib_node_runtime_t *, struct vlib_frame_t *f)
Definition: node.h:318
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
clib_error_t * main_loop_error
Definition: main.h:97
vlib_process_t * process
Definition: main.c:1232
elog_event_t * elog_peek_events(elog_main_t *em)
convert event ring events to events, and return them as a vector.
Definition: elog.c:509
static uword elog_buffer_capacity(elog_main_t *em)
Return number of events which can fit in the event buffer.
Definition: elog.h:198
u64 cpu_time_last_node_dispatch
Definition: main.h:65
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:169
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:322
u32 vectors_since_last_overflow
Definition: node.h:382
#define VLIB_PROCESS_IS_RUNNING
Definition: node.h:515
u32 max_clock_n
Number of vectors in the recorded max_clock.
Definition: node.h:426