FD.io VPP  v20.01-46-g7006026de
Vector Packet Processing
main.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * main.c: main vector processing loop
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <math.h>
41 #include <vppinfra/format.h>
42 #include <vlib/vlib.h>
43 #include <vlib/threads.h>
45 
46 #include <vlib/unix/unix.h>
47 #include <vlib/unix/cj.h>
48 
50 
51 /* Actually allocate a few extra slots of vector data to support
52  speculative vector enqueues which overflow vector data in next frame. */
53 #define VLIB_FRAME_SIZE_ALLOC (VLIB_FRAME_SIZE + 4)
54 
56 vlib_frame_bytes (u32 n_scalar_bytes, u32 n_vector_bytes)
57 {
58  u32 n_bytes;
59 
60  /* Make room for vlib_frame_t plus scalar arguments. */
61  n_bytes = vlib_frame_vector_byte_offset (n_scalar_bytes);
62 
63  /* Make room for vector arguments.
64  Allocate a few extra slots of vector data to support
65  speculative vector enqueues which overflow vector data in next frame. */
66 #define VLIB_FRAME_SIZE_EXTRA 4
67  n_bytes += (VLIB_FRAME_SIZE + VLIB_FRAME_SIZE_EXTRA) * n_vector_bytes;
68 
69  /* Magic number is first 32bit number after vector data.
70  Used to make sure that vector data is never overrun. */
71 #define VLIB_FRAME_MAGIC (0xabadc0ed)
72  n_bytes += sizeof (u32);
73 
74  /* Pad to cache line. */
75  n_bytes = round_pow2 (n_bytes, CLIB_CACHE_LINE_BYTES);
76 
77  return n_bytes;
78 }
79 
82 {
83  void *p = f;
84 
86 
88 
89  return p;
90 }
91 
92 static inline vlib_frame_size_t *
94  u32 n_scalar_bytes, u32 n_vector_bytes)
95 {
96 #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
97  uword key = (n_scalar_bytes << 16) | n_vector_bytes;
98  uword *p, i;
99 
100  p = hash_get (nm->frame_size_hash, key);
101  if (p)
102  i = p[0];
103  else
104  {
105  i = vec_len (nm->frame_sizes);
106  vec_validate (nm->frame_sizes, i);
107  hash_set (nm->frame_size_hash, key, i);
108  }
109 
110  return vec_elt_at_index (nm->frame_sizes, i);
111 #else
112  ASSERT (vlib_frame_bytes (n_scalar_bytes, n_vector_bytes)
113  == (vlib_frame_bytes (0, 4)));
114  return vec_elt_at_index (nm->frame_sizes, 0);
115 #endif
116 }
117 
118 static vlib_frame_t *
120  u32 frame_flags)
121 {
122  vlib_node_main_t *nm = &vm->node_main;
123  vlib_frame_size_t *fs;
124  vlib_node_t *to_node;
125  vlib_frame_t *f;
126  u32 l, n, scalar_size, vector_size;
127 
128  to_node = vlib_get_node (vm, to_node_index);
129 
130  scalar_size = to_node->scalar_size;
131  vector_size = to_node->vector_size;
132 
133  fs = get_frame_size_info (nm, scalar_size, vector_size);
134  n = vlib_frame_bytes (scalar_size, vector_size);
135  if ((l = vec_len (fs->free_frames)) > 0)
136  {
137  /* Allocate from end of free list. */
138  f = fs->free_frames[l - 1];
139  _vec_len (fs->free_frames) = l - 1;
140  }
141  else
142  {
144  }
145 
146  /* Poison frame when debugging. */
147  if (CLIB_DEBUG > 0)
148  clib_memset (f, 0xfe, n);
149 
150  /* Insert magic number. */
151  {
152  u32 *magic;
153 
154  magic = vlib_frame_find_magic (f, to_node);
155  *magic = VLIB_FRAME_MAGIC;
156  }
157 
158  f->frame_flags = VLIB_FRAME_IS_ALLOCATED | frame_flags;
159  f->n_vectors = 0;
160  f->scalar_size = scalar_size;
161  f->vector_size = vector_size;
162  f->flags = 0;
163 
164  fs->n_alloc_frames += 1;
165 
166  return f;
167 }
168 
169 /* Allocate a frame for from FROM_NODE to TO_NODE via TO_NEXT_INDEX.
170  Returns frame index. */
171 static vlib_frame_t *
173  u32 to_next_index)
174 {
175  vlib_node_t *from_node;
176 
177  from_node = vlib_get_node (vm, from_node_runtime->node_index);
178  ASSERT (to_next_index < vec_len (from_node->next_nodes));
179 
180  return vlib_frame_alloc_to_node (vm, from_node->next_nodes[to_next_index],
181  /* frame_flags */ 0);
182 }
183 
184 vlib_frame_t *
186 {
187  vlib_frame_t *f = vlib_frame_alloc_to_node (vm, to_node_index,
188  /* frame_flags */
190  return vlib_get_frame (vm, f);
191 }
192 
193 void
195 {
197  vlib_node_t *to_node;
198 
199  if (f->n_vectors == 0)
200  return;
201 
202  to_node = vlib_get_node (vm, to_node_index);
203 
204  vec_add2 (vm->node_main.pending_frames, p, 1);
205 
207  p->frame = vlib_get_frame (vm, f);
208  p->node_runtime_index = to_node->runtime_index;
210 }
211 
212 /* Free given frame. */
213 void
215 {
216  vlib_node_main_t *nm = &vm->node_main;
217  vlib_node_t *node;
218  vlib_frame_size_t *fs;
219 
221 
222  node = vlib_get_node (vm, r->node_index);
223  fs = get_frame_size_info (nm, node->scalar_size, node->vector_size);
224 
226 
227  /* No next frames may point to freed frame. */
228  if (CLIB_DEBUG > 0)
229  {
230  vlib_next_frame_t *nf;
231  vec_foreach (nf, vm->node_main.next_frames) ASSERT (nf->frame != f);
232  }
233 
235 
236  vec_add1 (fs->free_frames, f);
237  ASSERT (fs->n_alloc_frames > 0);
238  fs->n_alloc_frames -= 1;
239 }
240 
241 static clib_error_t *
243  unformat_input_t * input, vlib_cli_command_t * cmd)
244 {
245  vlib_node_main_t *nm = &vm->node_main;
246  vlib_frame_size_t *fs;
247 
248  vlib_cli_output (vm, "%=6s%=12s%=12s", "Size", "# Alloc", "# Free");
249  vec_foreach (fs, nm->frame_sizes)
250  {
251  u32 n_alloc = fs->n_alloc_frames;
252  u32 n_free = vec_len (fs->free_frames);
253 
254  if (n_alloc + n_free > 0)
255  vlib_cli_output (vm, "%=6d%=12d%=12d",
256  fs - nm->frame_sizes, n_alloc, n_free);
257  }
258 
259  return 0;
260 }
261 
262 /* *INDENT-OFF* */
263 VLIB_CLI_COMMAND (show_frame_stats_cli, static) = {
264  .path = "show vlib frame-allocation",
265  .short_help = "Show node dispatch frame statistics",
266  .function = show_frame_stats,
267 };
268 /* *INDENT-ON* */
269 
270 /* Change ownership of enqueue rights to given next node. */
271 static void
273  vlib_node_runtime_t * node_runtime,
274  u32 next_index)
275 {
276  vlib_node_main_t *nm = &vm->node_main;
277  vlib_next_frame_t *next_frame;
278  vlib_node_t *node, *next_node;
279 
280  node = vec_elt (nm->nodes, node_runtime->node_index);
281 
282  /* Only internal & input nodes are allowed to call other nodes. */
284  || node->type == VLIB_NODE_TYPE_INPUT
285  || node->type == VLIB_NODE_TYPE_PROCESS);
286 
287  ASSERT (vec_len (node->next_nodes) == node_runtime->n_next_nodes);
288 
289  next_frame =
290  vlib_node_runtime_get_next_frame (vm, node_runtime, next_index);
291  next_node = vec_elt (nm->nodes, node->next_nodes[next_index]);
292 
293  if (next_node->owner_node_index != VLIB_INVALID_NODE_INDEX)
294  {
295  /* Get frame from previous owner. */
296  vlib_next_frame_t *owner_next_frame;
297  vlib_next_frame_t tmp;
298 
299  owner_next_frame =
301  next_node->owner_node_index,
302  next_node->owner_next_index);
303 
304  /* Swap target next frame with owner's. */
305  tmp = owner_next_frame[0];
306  owner_next_frame[0] = next_frame[0];
307  next_frame[0] = tmp;
308 
309  /*
310  * If next_frame is already pending, we have to track down
311  * all pending frames and fix their next_frame_index fields.
312  */
313  if (next_frame->flags & VLIB_FRAME_PENDING)
314  {
316  if (next_frame->frame != NULL)
317  {
318  vec_foreach (p, nm->pending_frames)
319  {
320  if (p->frame == next_frame->frame)
321  {
322  p->next_frame_index =
323  next_frame - vm->node_main.next_frames;
324  }
325  }
326  }
327  }
328  }
329  else
330  {
331  /* No previous owner. Take ownership. */
332  next_frame->flags |= VLIB_FRAME_OWNER;
333  }
334 
335  /* Record new owner. */
336  next_node->owner_node_index = node->index;
337  next_node->owner_next_index = next_index;
338 
339  /* Now we should be owner. */
340  ASSERT (next_frame->flags & VLIB_FRAME_OWNER);
341 }
342 
343 /* Make sure that magic number is still there.
344  Otherwise, it is likely that caller has overrun frame arguments. */
345 always_inline void
347  vlib_frame_t * f, vlib_node_t * n, uword next_index)
348 {
349  vlib_node_t *next_node = vlib_get_node (vm, n->next_nodes[next_index]);
350  u32 *magic = vlib_frame_find_magic (f, next_node);
351  ASSERT (VLIB_FRAME_MAGIC == magic[0]);
352 }
353 
354 vlib_frame_t *
357  u32 next_index, u32 allocate_new_next_frame)
358 {
359  vlib_frame_t *f;
360  vlib_next_frame_t *nf;
361  u32 n_used;
362 
363  nf = vlib_node_runtime_get_next_frame (vm, node, next_index);
364 
365  /* Make sure this next frame owns right to enqueue to destination frame. */
366  if (PREDICT_FALSE (!(nf->flags & VLIB_FRAME_OWNER)))
367  vlib_next_frame_change_ownership (vm, node, next_index);
368 
369  /* ??? Don't need valid flag: can use frame_index == ~0 */
371  {
372  nf->frame = vlib_frame_alloc (vm, node, next_index);
374  }
375 
376  f = nf->frame;
377 
378  /* Has frame been removed from pending vector (e.g. finished dispatching)?
379  If so we can reuse frame. */
380  if ((nf->flags & VLIB_FRAME_PENDING)
381  && !(f->frame_flags & VLIB_FRAME_PENDING))
382  {
383  nf->flags &= ~VLIB_FRAME_PENDING;
384  f->n_vectors = 0;
385  f->flags = 0;
386  }
387 
388  /* Allocate new frame if current one is marked as no-append or
389  it is already full. */
390  n_used = f->n_vectors;
391  if (n_used >= VLIB_FRAME_SIZE || (allocate_new_next_frame && n_used > 0) ||
393  {
394  /* Old frame may need to be freed after dispatch, since we'll have
395  two redundant frames from node -> next node. */
397  {
398  vlib_frame_t *f_old = vlib_get_frame (vm, nf->frame);
400  }
401 
402  /* Allocate new frame to replace full one. */
403  f = nf->frame = vlib_frame_alloc (vm, node, next_index);
404  n_used = f->n_vectors;
405  }
406 
407  /* Should have free vectors in frame now. */
408  ASSERT (n_used < VLIB_FRAME_SIZE);
409 
410  if (CLIB_DEBUG > 0)
411  {
412  validate_frame_magic (vm, f,
413  vlib_get_node (vm, node->node_index), next_index);
414  }
415 
416  return f;
417 }
418 
419 static void
421  vlib_node_runtime_t * rt,
422  u32 next_index, u32 n_vectors_left)
423 {
424  vlib_node_main_t *nm = &vm->node_main;
425  vlib_next_frame_t *nf;
426  vlib_frame_t *f;
427  vlib_node_runtime_t *next_rt;
428  vlib_node_t *next_node;
429  u32 n_before, n_after;
430 
431  nf = vlib_node_runtime_get_next_frame (vm, rt, next_index);
432  f = vlib_get_frame (vm, nf->frame);
433 
434  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
435  n_after = VLIB_FRAME_SIZE - n_vectors_left;
436  n_before = f->n_vectors;
437 
438  ASSERT (n_after >= n_before);
439 
441  nf->node_runtime_index);
442  next_node = vlib_get_node (vm, next_rt->node_index);
443  if (n_after > 0 && next_node->validate_frame)
444  {
445  u8 *msg = next_node->validate_frame (vm, rt, f);
446  if (msg)
447  {
448  clib_warning ("%v", msg);
449  ASSERT (0);
450  }
451  vec_free (msg);
452  }
453 }
454 
455 void
458  u32 next_index, u32 n_vectors_left)
459 {
460  vlib_node_main_t *nm = &vm->node_main;
461  vlib_next_frame_t *nf;
462  vlib_frame_t *f;
463  u32 n_vectors_in_frame;
464 
465  if (CLIB_DEBUG > 0)
466  vlib_put_next_frame_validate (vm, r, next_index, n_vectors_left);
467 
468  nf = vlib_node_runtime_get_next_frame (vm, r, next_index);
469  f = vlib_get_frame (vm, nf->frame);
470 
471  /* Make sure that magic number is still there. Otherwise, caller
472  has overrun frame meta data. */
473  if (CLIB_DEBUG > 0)
474  {
476  validate_frame_magic (vm, f, node, next_index);
477  }
478 
479  /* Convert # of vectors left -> number of vectors there. */
480  ASSERT (n_vectors_left <= VLIB_FRAME_SIZE);
481  n_vectors_in_frame = VLIB_FRAME_SIZE - n_vectors_left;
482 
483  f->n_vectors = n_vectors_in_frame;
484 
485  /* If vectors were added to frame, add to pending vector. */
486  if (PREDICT_TRUE (n_vectors_in_frame > 0))
487  {
489  u32 v0, v1;
490 
491  r->cached_next_index = next_index;
492 
493  if (!(f->frame_flags & VLIB_FRAME_PENDING))
494  {
495  __attribute__ ((unused)) vlib_node_t *node;
496  vlib_node_t *next_node;
497  vlib_node_runtime_t *next_runtime;
498 
499  node = vlib_get_node (vm, r->node_index);
500  next_node = vlib_get_next_node (vm, r->node_index, next_index);
501  next_runtime = vlib_node_get_runtime (vm, next_node->index);
502 
503  vec_add2 (nm->pending_frames, p, 1);
504 
505  p->frame = nf->frame;
507  p->next_frame_index = nf - nm->next_frames;
508  nf->flags |= VLIB_FRAME_PENDING;
510 
511  /*
512  * If we're going to dispatch this frame on another thread,
513  * force allocation of a new frame. Otherwise, we create
514  * a dangling frame reference. Each thread has its own copy of
515  * the next_frames vector.
516  */
517  if (0 && r->thread_index != next_runtime->thread_index)
518  {
519  nf->frame = NULL;
521  }
522  }
523 
524  /* Copy trace flag from next_frame and from runtime. */
525  nf->flags |=
526  (nf->flags & VLIB_NODE_FLAG_TRACE) | (r->
528 
530  v1 = v0 + n_vectors_in_frame;
532  if (PREDICT_FALSE (v1 < v0))
533  {
535  vec_elt (node->n_vectors_by_next_node, next_index) += v0;
536  }
537  }
538 }
539 
540 /* Sync up runtime (32 bit counters) and main node stats (64 bit counters). */
541 never_inline void
544  uword n_calls, uword n_vectors, uword n_clocks,
545  uword n_ticks0, uword n_ticks1)
546 {
547  vlib_node_t *n = vlib_get_node (vm, r->node_index);
548 
549  n->stats_total.calls += n_calls + r->calls_since_last_overflow;
550  n->stats_total.vectors += n_vectors + r->vectors_since_last_overflow;
551  n->stats_total.clocks += n_clocks + r->clocks_since_last_overflow;
552  n->stats_total.perf_counter0_ticks += n_ticks0 +
554  n->stats_total.perf_counter1_ticks += n_ticks1 +
556  n->stats_total.perf_counter_vectors += n_vectors +
560 
567 }
568 
569 always_inline void __attribute__ ((unused))
571  vlib_process_t * p,
572  uword n_calls, uword n_vectors, uword n_clocks,
573  uword n_ticks0, uword n_ticks1)
574 {
575  vlib_node_runtime_t *rt = &p->node_runtime;
576  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
577  vlib_node_runtime_sync_stats (vm, rt, n_calls, n_vectors, n_clocks,
578  n_ticks0, n_ticks1);
579  n->stats_total.suspends += p->n_suspends;
580  p->n_suspends = 0;
581 }
582 
583 void
585 {
587 
588  if (n->type == VLIB_NODE_TYPE_PROCESS)
589  {
590  /* Nothing to do for PROCESS nodes except in main thread */
591  if (vm != &vlib_global_main)
592  return;
593 
596  p->n_suspends = 0;
597  rt = &p->node_runtime;
598  }
599  else
600  rt =
602  n->runtime_index);
603 
604  vlib_node_runtime_sync_stats (vm, rt, 0, 0, 0, 0, 0);
605 
606  /* Sync up runtime next frame vector counters with main node structure. */
607  {
608  vlib_next_frame_t *nf;
609  uword i;
610  for (i = 0; i < rt->n_next_nodes; i++)
611  {
612  nf = vlib_node_runtime_get_next_frame (vm, rt, i);
616  }
617  }
618 }
619 
623  uword n_calls,
624  uword n_vectors, uword n_clocks,
625  uword n_ticks0, uword n_ticks1)
626 {
627  u32 ca0, ca1, v0, v1, cl0, cl1, r;
628  u32 ptick00, ptick01, ptick10, ptick11, pvec0, pvec1;
629 
630  cl0 = cl1 = node->clocks_since_last_overflow;
631  ca0 = ca1 = node->calls_since_last_overflow;
632  v0 = v1 = node->vectors_since_last_overflow;
633  ptick00 = ptick01 = node->perf_counter0_ticks_since_last_overflow;
634  ptick10 = ptick11 = node->perf_counter1_ticks_since_last_overflow;
635  pvec0 = pvec1 = node->perf_counter_vectors_since_last_overflow;
636 
637  ca1 = ca0 + n_calls;
638  v1 = v0 + n_vectors;
639  cl1 = cl0 + n_clocks;
640  ptick01 = ptick00 + n_ticks0;
641  ptick11 = ptick10 + n_ticks1;
642  pvec1 = pvec0 + n_vectors;
643 
644  node->calls_since_last_overflow = ca1;
645  node->clocks_since_last_overflow = cl1;
646  node->vectors_since_last_overflow = v1;
650 
651  node->max_clock_n = node->max_clock > n_clocks ?
652  node->max_clock_n : n_vectors;
653  node->max_clock = node->max_clock > n_clocks ? node->max_clock : n_clocks;
654 
655  r = vlib_node_runtime_update_main_loop_vector_stats (vm, node, n_vectors);
656 
657  if (PREDICT_FALSE (ca1 < ca0 || v1 < v0 || cl1 < cl0) || (ptick01 < ptick00)
658  || (ptick11 < ptick10) || (pvec1 < pvec0))
659  {
660  node->calls_since_last_overflow = ca0;
661  node->clocks_since_last_overflow = cl0;
662  node->vectors_since_last_overflow = v0;
666 
667  vlib_node_runtime_sync_stats (vm, node, n_calls, n_vectors, n_clocks,
668  n_ticks0, n_ticks1);
669  }
670 
671  return r;
672 }
673 
674 always_inline void
677  vlib_frame_t * frame, int before_or_after)
678 {
679  *pmc0 = 0;
680  *pmc1 = 0;
683  pmc1, node, frame, before_or_after);
684 }
685 
686 always_inline void
688  vlib_process_t * p,
689  uword n_calls, uword n_vectors, uword n_clocks)
690 {
692  n_calls, n_vectors, n_clocks, 0ULL, 0ULL);
693 }
694 
695 static clib_error_t *
697  unformat_input_t * input, vlib_cli_command_t * cmd)
698 {
700  return 0;
701 }
702 
703 /* *INDENT-OFF* */
704 VLIB_CLI_COMMAND (elog_clear_cli, static) = {
705  .path = "event-logger clear",
706  .short_help = "Clear the event log",
707  .function = vlib_cli_elog_clear,
708 };
709 /* *INDENT-ON* */
710 
711 #ifdef CLIB_UNIX
712 static clib_error_t *
714  unformat_input_t * input, vlib_cli_command_t * cmd)
715 {
716  elog_main_t *em = &vm->elog_main;
717  char *file, *chroot_file;
718  clib_error_t *error = 0;
719 
720  if (!unformat (input, "%s", &file))
721  {
722  vlib_cli_output (vm, "expected file name, got `%U'",
723  format_unformat_error, input);
724  return 0;
725  }
726 
727  /* It's fairly hard to get "../oopsie" through unformat; just in case */
728  if (strstr (file, "..") || index (file, '/'))
729  {
730  vlib_cli_output (vm, "illegal characters in filename '%s'", file);
731  return 0;
732  }
733 
734  chroot_file = (char *) format (0, "/tmp/%s%c", file, 0);
735 
736  vec_free (file);
737 
738  vlib_cli_output (vm, "Saving %wd of %wd events to %s",
740  elog_buffer_capacity (em), chroot_file);
741 
743  error = elog_write_file (em, chroot_file, 1 /* flush ring */ );
745  vec_free (chroot_file);
746  return error;
747 }
748 
749 void
751 {
753  elog_main_t *em = &vm->elog_main;
754  u8 *filename;
755  clib_error_t *error;
756 
757  if (!vm->elog_post_mortem_dump)
758  return;
759 
760  filename = format (0, "/tmp/elog_post_mortem.%d%c", getpid (), 0);
761  error = elog_write_file (em, (char *) filename, 1 /* flush ring */ );
762  if (error)
763  clib_error_report (error);
764  vec_free (filename);
765 }
766 
767 /* *INDENT-OFF* */
768 VLIB_CLI_COMMAND (elog_save_cli, static) = {
769  .path = "event-logger save",
770  .short_help = "event-logger save <filename> (saves log in /tmp/<filename>)",
771  .function = elog_save_buffer,
772 };
773 /* *INDENT-ON* */
774 
775 static clib_error_t *
777  unformat_input_t * input, vlib_cli_command_t * cmd)
778 {
779  elog_main_t *em = &vm->elog_main;
780 
782 
783  vlib_cli_output (vm, "Stopped the event logger...");
784  return 0;
785 }
786 
787 /* *INDENT-OFF* */
788 VLIB_CLI_COMMAND (elog_stop_cli, static) = {
789  .path = "event-logger stop",
790  .short_help = "Stop the event-logger",
791  .function = elog_stop,
792 };
793 /* *INDENT-ON* */
794 
795 static clib_error_t *
797  unformat_input_t * input, vlib_cli_command_t * cmd)
798 {
799  elog_main_t *em = &vm->elog_main;
800 
802 
803  vlib_cli_output (vm, "Restarted the event logger...");
804  return 0;
805 }
806 
807 /* *INDENT-OFF* */
808 VLIB_CLI_COMMAND (elog_restart_cli, static) = {
809  .path = "event-logger restart",
810  .short_help = "Restart the event-logger",
811  .function = elog_restart,
812 };
813 /* *INDENT-ON* */
814 
815 static clib_error_t *
817  unformat_input_t * input, vlib_cli_command_t * cmd)
818 {
819  elog_main_t *em = &vm->elog_main;
820  u32 tmp;
821 
822  /* Stop the parade */
824 
825  if (unformat (input, "%d", &tmp))
826  {
827  elog_alloc (em, tmp);
829  }
830  else
831  return clib_error_return (0, "Must specify how many events in the ring");
832 
833  vlib_cli_output (vm, "Resized ring and restarted the event logger...");
834  return 0;
835 }
836 
837 /* *INDENT-OFF* */
838 VLIB_CLI_COMMAND (elog_resize_cli, static) = {
839  .path = "event-logger resize",
840  .short_help = "event-logger resize <nnn>",
841  .function = elog_resize,
842 };
843 /* *INDENT-ON* */
844 
845 #endif /* CLIB_UNIX */
846 
847 static void
849 {
850  elog_main_t *em = &vm->elog_main;
851  elog_event_t *e, *es;
852  f64 dt;
853 
854  /* Show events in VLIB time since log clock starts after VLIB clock. */
855  dt = (em->init_time.cpu - vm->clib_time.init_cpu_time)
857 
858  es = elog_peek_events (em);
859  vlib_cli_output (vm, "%d of %d events in buffer, logger %s", vec_len (es),
860  em->event_ring_size,
862  "running" : "stopped");
863  vec_foreach (e, es)
864  {
865  vlib_cli_output (vm, "%18.9f: %U",
866  e->time + dt, format_elog_event, em, e);
867  n_events_to_show--;
868  if (n_events_to_show == 0)
869  break;
870  }
871  vec_free (es);
872 
873 }
874 
875 static clib_error_t *
877  unformat_input_t * input, vlib_cli_command_t * cmd)
878 {
879  u32 n_events_to_show;
880  clib_error_t *error = 0;
881 
882  n_events_to_show = 250;
884  {
885  if (unformat (input, "%d", &n_events_to_show))
886  ;
887  else if (unformat (input, "all"))
888  n_events_to_show = ~0;
889  else
890  return unformat_parse_error (input);
891  }
892  elog_show_buffer_internal (vm, n_events_to_show);
893  return error;
894 }
895 
896 /* *INDENT-OFF* */
897 VLIB_CLI_COMMAND (elog_show_cli, static) = {
898  .path = "show event-logger",
899  .short_help = "Show event logger info",
900  .function = elog_show_buffer,
901 };
902 /* *INDENT-ON* */
903 
904 void
906 {
908 }
909 
910 static inline void
912  u32 node_index,
913  u64 time, u32 n_vectors, u32 is_return)
914 {
916  elog_main_t *em = &evm->elog_main;
917  int enabled = evm->elog_trace_graph_dispatch |
919 
920  if (PREDICT_FALSE (enabled && n_vectors))
921  {
922  if (PREDICT_FALSE (!elog_is_enabled (em)))
923  {
924  evm->elog_trace_graph_dispatch = 0;
925  evm->elog_trace_graph_circuit = 0;
926  return;
927  }
928  if (PREDICT_TRUE
930  (evm->elog_trace_graph_circuit &&
931  node_index == evm->elog_trace_graph_circuit_node_index)))
932  {
933  elog_track (em,
934  /* event type */
935  vec_elt_at_index (is_return
938  node_index),
939  /* track */
940  (vm->thread_index ?
942  : &em->default_track),
943  /* data to log */ n_vectors);
944  }
945  }
946 }
947 
948 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
949 void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 node_index);
950 void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
951 
952 void
953 vlib_buffer_trace_trajectory_init (vlib_buffer_t * b)
954 {
955  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_init_cb != 0))
956  {
957  (*vlib_buffer_trace_trajectory_init_cb) (b);
958  }
959 }
960 
961 #endif
962 
963 static inline void
965 {
966 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
967  if (PREDICT_TRUE (vlib_buffer_trace_trajectory_cb != 0))
968  {
969  (*vlib_buffer_trace_trajectory_cb) (b, node_index);
970  }
971 #endif
972 }
973 
974 u8 *format_vnet_buffer_flags (u8 * s, va_list * args) __attribute__ ((weak));
975 u8 *
976 format_vnet_buffer_flags (u8 * s, va_list * args)
977 {
978  s = format (s, "BUG STUB %s", __FUNCTION__);
979  return s;
980 }
981 
982 u8 *format_vnet_buffer_opaque (u8 * s, va_list * args) __attribute__ ((weak));
983 u8 *
984 format_vnet_buffer_opaque (u8 * s, va_list * args)
985 {
986  s = format (s, "BUG STUB %s", __FUNCTION__);
987  return s;
988 }
989 
990 u8 *format_vnet_buffer_opaque2 (u8 * s, va_list * args)
991  __attribute__ ((weak));
992 u8 *
993 format_vnet_buffer_opaque2 (u8 * s, va_list * args)
994 {
995  s = format (s, "BUG STUB %s", __FUNCTION__);
996  return s;
997 }
998 
999 static u8 *
1000 format_buffer_metadata (u8 * s, va_list * args)
1001 {
1002  vlib_buffer_t *b = va_arg (*args, vlib_buffer_t *);
1003 
1004  s = format (s, "flags: %U\n", format_vnet_buffer_flags, b);
1005  s = format (s, "current_data: %d, current_length: %d\n",
1006  (i32) (b->current_data), (i32) (b->current_length));
1007  s = format
1008  (s,
1009  "current_config_index/punt_reason: %d, flow_id: %x, next_buffer: %x\n",
1011  s =
1012  format (s, "error: %d, ref_count: %d, buffer_pool_index: %d\n",
1013  (u32) (b->error), (u32) (b->ref_count),
1014  (u32) (b->buffer_pool_index));
1015  s =
1016  format (s, "trace_handle: 0x%x, len_not_first_buf: %d\n", b->trace_handle,
1018  return s;
1019 }
1020 
1021 #define A(x) vec_add1(vm->pcap_buffer, (x))
1022 
1023 static void
1026 {
1027  int i;
1028  vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **bufp, *b;
1029  pcap_main_t *pm = &vlib_global_main.dispatch_pcap_main;
1030  vlib_trace_main_t *tm = &vm->trace_main;
1031  u32 capture_size;
1032  vlib_node_t *n;
1033  i32 n_left;
1034  f64 time_now = vlib_time_now (vm);
1035  u32 *from;
1036  u8 *d;
1037  u8 string_count;
1038 
1039  /* Input nodes don't have frames yet */
1040  if (frame == 0 || frame->n_vectors == 0)
1041  return;
1042 
1043  from = vlib_frame_vector_args (frame);
1044  vlib_get_buffers (vm, from, bufs, frame->n_vectors);
1045  bufp = bufs;
1046 
1047  n = vlib_get_node (vm, node->node_index);
1048 
1049  for (i = 0; i < frame->n_vectors; i++)
1050  {
1052  {
1053  b = bufp[i];
1054 
1056  string_count = 0;
1057 
1058  /* Version, flags */
1061  A (0 /* string_count */ );
1062  A (n->protocol_hint);
1063 
1064  /* Buffer index (big endian) */
1065  A ((from[i] >> 24) & 0xff);
1066  A ((from[i] >> 16) & 0xff);
1067  A ((from[i] >> 8) & 0xff);
1068  A ((from[i] >> 0) & 0xff);
1069 
1070  /* Node name, NULL-terminated ASCII */
1071  vm->pcap_buffer = format (vm->pcap_buffer, "%v%c", n->name, 0);
1072  string_count++;
1073 
1074  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1075  format_buffer_metadata, b, 0);
1076  string_count++;
1077  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1079  string_count++;
1080  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1082  string_count++;
1083 
1084  /* Is this packet traced? */
1085  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
1086  {
1090 
1091  vm->pcap_buffer = format (vm->pcap_buffer, "%U%c",
1092  format_vlib_trace, vm, h[0], 0);
1093  string_count++;
1094  }
1095 
1096  /* Save the string count */
1097  vm->pcap_buffer[2] = string_count;
1098 
1099  /* Figure out how many bytes in the pcap trace */
1100  capture_size = vec_len (vm->pcap_buffer) +
1101  +vlib_buffer_length_in_chain (vm, b);
1102 
1104  n_left = clib_min (capture_size, 16384);
1105  d = pcap_add_packet (pm, time_now, n_left, capture_size);
1106 
1107  /* Copy the header */
1109  d += vec_len (vm->pcap_buffer);
1110 
1111  n_left = clib_min
1112  (vlib_buffer_length_in_chain (vm, b),
1113  (16384 - vec_len (vm->pcap_buffer)));
1114  /* Copy the packet data */
1115  while (1)
1116  {
1117  u32 copy_length = clib_min ((u32) n_left, b->current_length);
1118  clib_memcpy_fast (d, b->data + b->current_data, copy_length);
1119  n_left -= b->current_length;
1120  if (n_left <= 0)
1121  break;
1122  d += b->current_length;
1123  ASSERT (b->flags & VLIB_BUFFER_NEXT_PRESENT);
1124  b = vlib_get_buffer (vm, b->next_buffer);
1125  }
1127  }
1128  }
1129 }
1130 
1135  vlib_node_state_t dispatch_state,
1136  vlib_frame_t * frame, u64 last_time_stamp)
1137 {
1138  uword n, v;
1139  u64 t;
1140  vlib_node_main_t *nm = &vm->node_main;
1141  vlib_next_frame_t *nf;
1142  u64 pmc_before[2], pmc_after[2], pmc_delta[2];
1143 
1144  if (CLIB_DEBUG > 0)
1145  {
1146  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1147  ASSERT (n->type == type);
1148  }
1149 
1150  /* Only non-internal nodes may be disabled. */
1151  if (type != VLIB_NODE_TYPE_INTERNAL && node->state != dispatch_state)
1152  {
1153  ASSERT (type != VLIB_NODE_TYPE_INTERNAL);
1154  return last_time_stamp;
1155  }
1156 
1157  if ((type == VLIB_NODE_TYPE_PRE_INPUT || type == VLIB_NODE_TYPE_INPUT)
1158  && dispatch_state != VLIB_NODE_STATE_INTERRUPT)
1159  {
1161  /* Only call node when count reaches zero. */
1162  if (c)
1163  {
1164  node->input_main_loops_per_call = c - 1;
1165  return last_time_stamp;
1166  }
1167  }
1168 
1169  /* Speculatively prefetch next frames. */
1170  if (node->n_next_nodes > 0)
1171  {
1172  nf = vec_elt_at_index (nm->next_frames, node->next_frame_index);
1173  CLIB_PREFETCH (nf, 4 * sizeof (nf[0]), WRITE);
1174  }
1175 
1176  vm->cpu_time_last_node_dispatch = last_time_stamp;
1177 
1179  last_time_stamp, frame ? frame->n_vectors : 0,
1180  /* is_after */ 0);
1181 
1182  vlib_node_runtime_perf_counter (vm, &pmc_before[0], &pmc_before[1],
1183  node, frame, 0 /* before */ );
1184 
1185  /*
1186  * Turn this on if you run into
1187  * "bad monkey" contexts, and you want to know exactly
1188  * which nodes they've visited... See ixge.c...
1189  */
1190  if (VLIB_BUFFER_TRACE_TRAJECTORY && frame)
1191  {
1192  int i;
1193  u32 *from;
1194  from = vlib_frame_vector_args (frame);
1195  for (i = 0; i < frame->n_vectors; i++)
1196  {
1197  vlib_buffer_t *b = vlib_get_buffer (vm, from[i]);
1198  add_trajectory_trace (b, node->node_index);
1199  }
1201  dispatch_pcap_trace (vm, node, frame);
1202  n = node->function (vm, node, frame);
1203  }
1204  else
1205  {
1207  dispatch_pcap_trace (vm, node, frame);
1208  n = node->function (vm, node, frame);
1209  }
1210 
1211  t = clib_cpu_time_now ();
1212 
1213  /*
1214  * To validate accounting: pmc_delta = t - pmc_before;
1215  * perf ticks should equal clocks/pkt...
1216  */
1217  vlib_node_runtime_perf_counter (vm, &pmc_after[0], &pmc_after[1], node,
1218  frame, 1 /* after */ );
1219 
1220  pmc_delta[0] = pmc_after[0] - pmc_before[0];
1221  pmc_delta[1] = pmc_after[1] - pmc_before[1];
1222 
1223  vlib_elog_main_loop_event (vm, node->node_index, t, n, 1 /* is_after */ );
1224 
1225  vm->main_loop_vectors_processed += n;
1226  vm->main_loop_nodes_processed += n > 0;
1227 
1228  v = vlib_node_runtime_update_stats (vm, node,
1229  /* n_calls */ 1,
1230  /* n_vectors */ n,
1231  /* n_clocks */ t - last_time_stamp,
1232  pmc_delta[0] /* PMC0 */ ,
1233  pmc_delta[1] /* PMC1 */ );
1234 
1235  /* When in interrupt mode and vector rate crosses threshold switch to
1236  polling mode. */
1237  if (PREDICT_FALSE ((dispatch_state == VLIB_NODE_STATE_INTERRUPT)
1238  || (dispatch_state == VLIB_NODE_STATE_POLLING
1239  && (node->flags
1240  &
1242  {
1243  /* *INDENT-OFF* */
1244  ELOG_TYPE_DECLARE (e) =
1245  {
1246  .function = (char *) __FUNCTION__,
1247  .format = "%s vector length %d, switching to %s",
1248  .format_args = "T4i4t4",
1249  .n_enum_strings = 2,
1250  .enum_strings = {
1251  "interrupt", "polling",
1252  },
1253  };
1254  /* *INDENT-ON* */
1255  struct
1256  {
1257  u32 node_name, vector_length, is_polling;
1258  } *ed;
1259 
1260  if ((dispatch_state == VLIB_NODE_STATE_INTERRUPT
1261  && v >= nm->polling_threshold_vector_length) &&
1262  !(node->flags &
1264  {
1265  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1266  n->state = VLIB_NODE_STATE_POLLING;
1267  node->state = VLIB_NODE_STATE_POLLING;
1268  node->flags &=
1271  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] -= 1;
1272  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] += 1;
1273 
1274  if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
1275  {
1277  + vm->thread_index;
1278 
1279  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1280  w->elog_track);
1281  ed->node_name = n->name_elog_string;
1282  ed->vector_length = v;
1283  ed->is_polling = 1;
1284  }
1285  }
1286  else if (dispatch_state == VLIB_NODE_STATE_POLLING
1287  && v <= nm->interrupt_threshold_vector_length)
1288  {
1289  vlib_node_t *n = vlib_get_node (vm, node->node_index);
1290  if (node->flags &
1292  {
1293  /* Switch to interrupt mode after dispatch in polling one more time.
1294  This allows driver to re-enable interrupts. */
1295  n->state = VLIB_NODE_STATE_INTERRUPT;
1296  node->state = VLIB_NODE_STATE_INTERRUPT;
1297  node->flags &=
1299  nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] -= 1;
1300  nm->input_node_counts_by_state[VLIB_NODE_STATE_INTERRUPT] += 1;
1301 
1302  }
1303  else
1304  {
1306  + vm->thread_index;
1307  node->flags |=
1309  if (PREDICT_FALSE (vlib_global_main.elog_trace_graph_dispatch))
1310  {
1311  ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
1312  w->elog_track);
1313  ed->node_name = n->name_elog_string;
1314  ed->vector_length = v;
1315  ed->is_polling = 0;
1316  }
1317  }
1318  }
1319  }
1320 
1321  return t;
1322 }
1323 
1324 static u64
1325 dispatch_pending_node (vlib_main_t * vm, uword pending_frame_index,
1326  u64 last_time_stamp)
1327 {
1328  vlib_node_main_t *nm = &vm->node_main;
1329  vlib_frame_t *f;
1330  vlib_next_frame_t *nf, nf_dummy;
1332  vlib_frame_t *restore_frame;
1334 
1335  /* See comment below about dangling references to nm->pending_frames */
1336  p = nm->pending_frames + pending_frame_index;
1337 
1339  p->node_runtime_index);
1340 
1341  f = vlib_get_frame (vm, p->frame);
1343  {
1344  /* No next frame: so use dummy on stack. */
1345  nf = &nf_dummy;
1347  nf->frame = NULL;
1348  }
1349  else
1351 
1353 
1354  /* Force allocation of new frame while current frame is being
1355  dispatched. */
1356  restore_frame = NULL;
1357  if (nf->frame == p->frame)
1358  {
1359  nf->frame = NULL;
1362  restore_frame = p->frame;
1363  }
1364 
1365  /* Frame must be pending. */
1367  ASSERT (f->n_vectors > 0);
1368 
1369  /* Copy trace flag from next frame to node.
1370  Trace flag indicates that at least one vector in the dispatched
1371  frame is traced. */
1372  n->flags &= ~VLIB_NODE_FLAG_TRACE;
1373  n->flags |= (nf->flags & VLIB_FRAME_TRACE) ? VLIB_NODE_FLAG_TRACE : 0;
1374  nf->flags &= ~VLIB_FRAME_TRACE;
1375 
1376  last_time_stamp = dispatch_node (vm, n,
1378  VLIB_NODE_STATE_POLLING,
1379  f, last_time_stamp);
1380  /* Internal node vector-rate accounting, for summary stats */
1381  vm->internal_node_vectors += f->n_vectors;
1382  vm->internal_node_calls++;
1386 
1388 
1389  /* Frame is ready to be used again, so restore it. */
1390  if (restore_frame != NULL)
1391  {
1392  /*
1393  * We musn't restore a frame that is flagged to be freed. This
1394  * shouldn't happen since frames to be freed post dispatch are
1395  * those used when the to-node frame becomes full i.e. they form a
1396  * sort of queue of frames to a single node. If we get here then
1397  * the to-node frame and the pending frame *were* the same, and so
1398  * we removed the to-node frame. Therefore this frame is no
1399  * longer part of the queue for that node and hence it cannot be
1400  * it's overspill.
1401  */
1403 
1404  /*
1405  * NB: dispatching node n can result in the creation and scheduling
1406  * of new frames, and hence in the reallocation of nm->pending_frames.
1407  * Recompute p, or no supper. This was broken for more than 10 years.
1408  */
1409  p = nm->pending_frames + pending_frame_index;
1410 
1411  /*
1412  * p->next_frame_index can change during node dispatch if node
1413  * function decides to change graph hook up.
1414  */
1417 
1418  if (NULL == nf->frame)
1419  {
1420  /* no new frame has been assigned to this node, use the saved one */
1421  nf->frame = restore_frame;
1422  f->n_vectors = 0;
1423  }
1424  else
1425  {
1426  /* The node has gained a frame, implying packets from the current frame
1427  were re-queued to this same node. we don't need the saved one
1428  anymore */
1429  vlib_frame_free (vm, n, f);
1430  }
1431  }
1432  else
1433  {
1435  {
1437  vlib_frame_free (vm, n, f);
1438  }
1439  }
1440 
1441  return last_time_stamp;
1442 }
1443 
1446 {
1447  return p->stack[0] == VLIB_PROCESS_STACK_MAGIC;
1448 }
1449 
1450 typedef struct
1451 {
1456 
1457 /* Called in process stack. */
1458 static uword
1460 {
1462  vlib_main_t *vm;
1464  vlib_frame_t *f;
1465  vlib_process_t *p;
1466  uword n;
1467 
1469 
1470  vm = a->vm;
1471  p = a->process;
1472  f = a->frame;
1473  node = &p->node_runtime;
1474 
1475  n = node->function (vm, node, f);
1476 
1478 
1479  clib_longjmp (&p->return_longjmp, n);
1480 
1481  return n;
1482 }
1483 
1484 /* Called in main stack. */
1487 {
1489  uword r;
1490 
1491  a.vm = vm;
1492  a.process = p;
1493  a.frame = f;
1494 
1498  (void *) p->stack + (1 << p->log2_n_stack_bytes));
1499 
1500  return r;
1501 }
1502 
1505 {
1506  uword r;
1513  return r;
1514 }
1515 
1516 static u64
1518  vlib_process_t * p, vlib_frame_t * f, u64 last_time_stamp)
1519 {
1520  vlib_node_main_t *nm = &vm->node_main;
1521  vlib_node_runtime_t *node_runtime = &p->node_runtime;
1522  vlib_node_t *node = vlib_get_node (vm, node_runtime->node_index);
1523  u32 old_process_index;
1524  u64 t;
1525  uword n_vectors, is_suspend;
1526 
1527  if (node->state != VLIB_NODE_STATE_POLLING
1530  return last_time_stamp;
1531 
1533 
1534  t = last_time_stamp;
1535  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1536  f ? f->n_vectors : 0, /* is_after */ 0);
1537 
1538  /* Save away current process for suspend. */
1539  old_process_index = nm->current_process_index;
1541 
1542  n_vectors = vlib_process_startup (vm, p, f);
1543 
1544  nm->current_process_index = old_process_index;
1545 
1547  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1548  if (is_suspend)
1549  {
1551 
1552  n_vectors = 0;
1554  pf->node_runtime_index = node->runtime_index;
1555  pf->frame = f;
1556  pf->next_frame_index = ~0;
1557 
1558  p->n_suspends += 1;
1560 
1562  {
1563  TWT (tw_timer_wheel) * tw =
1564  (TWT (tw_timer_wheel) *) nm->timing_wheel;
1565  p->stop_timer_handle =
1566  TW (tw_timer_start) (tw,
1568  (node->runtime_index) /* [sic] pool idex */ ,
1569  0 /* timer_id */ ,
1571  }
1572  }
1573  else
1575 
1576  t = clib_cpu_time_now ();
1577 
1578  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, is_suspend,
1579  /* is_after */ 1);
1580 
1582  /* n_calls */ !is_suspend,
1583  /* n_vectors */ n_vectors,
1584  /* n_clocks */ t - last_time_stamp);
1585 
1586  return t;
1587 }
1588 
1589 void
1591 {
1592  vlib_node_main_t *nm = &vm->node_main;
1593  vlib_process_t *p = vec_elt (nm->processes, process_index);
1594  dispatch_process (vm, p, /* frame */ 0, /* cpu_time_now */ 0);
1595 }
1596 
1597 static u64
1599  uword process_index, u64 last_time_stamp)
1600 {
1601  vlib_node_main_t *nm = &vm->node_main;
1602  vlib_node_runtime_t *node_runtime;
1603  vlib_node_t *node;
1604  vlib_frame_t *f;
1605  vlib_process_t *p;
1607  u64 t, n_vectors, is_suspend;
1608 
1609  t = last_time_stamp;
1610 
1611  p = vec_elt (nm->processes, process_index);
1613  return last_time_stamp;
1614 
1617 
1620 
1621  node_runtime = &p->node_runtime;
1622  node = vlib_get_node (vm, node_runtime->node_index);
1623  f = pf->frame;
1624 
1625  vlib_elog_main_loop_event (vm, node_runtime->node_index, t,
1626  f ? f->n_vectors : 0, /* is_after */ 0);
1627 
1628  /* Save away current process for suspend. */
1630 
1631  n_vectors = vlib_process_resume (p);
1632  t = clib_cpu_time_now ();
1633 
1634  nm->current_process_index = ~0;
1635 
1636  is_suspend = n_vectors == VLIB_PROCESS_RETURN_LONGJMP_SUSPEND;
1637  if (is_suspend)
1638  {
1639  /* Suspend it again. */
1640  n_vectors = 0;
1641  p->n_suspends += 1;
1643  {
1644  p->stop_timer_handle =
1645  TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
1647  (node->runtime_index) /* [sic] pool idex */ ,
1648  0 /* timer_id */ ,
1650  }
1651  }
1652  else
1653  {
1658  }
1659 
1660  t = clib_cpu_time_now ();
1661  vlib_elog_main_loop_event (vm, node_runtime->node_index, t, !is_suspend,
1662  /* is_after */ 1);
1663 
1665  /* n_calls */ !is_suspend,
1666  /* n_vectors */ n_vectors,
1667  /* n_clocks */ t - last_time_stamp);
1668 
1669  return t;
1670 }
1671 
1672 void vl_api_send_pending_rpc_requests (vlib_main_t *) __attribute__ ((weak));
1673 void
1675 {
1676 }
1677 
1678 
1681 {
1682  vlib_node_main_t *nm = &vm->node_main;
1684  uword i;
1685  u64 cpu_time_now;
1687  u32 *last_node_runtime_indices = 0;
1688  u32 frame_queue_check_counter = 0;
1689 
1690  /* Initialize pending node vector. */
1691  if (is_main)
1692  {
1693  vec_resize (nm->pending_frames, 32);
1694  _vec_len (nm->pending_frames) = 0;
1695  }
1696 
1697  /* Mark time of main loop start. */
1698  if (is_main)
1699  {
1700  cpu_time_now = vm->clib_time.last_cpu_time;
1701  vm->cpu_time_main_loop_start = cpu_time_now;
1702  }
1703  else
1704  cpu_time_now = clib_cpu_time_now ();
1705 
1706  /* Pre-allocate interupt runtime indices and lock. */
1708  vec_alloc (last_node_runtime_indices, 32);
1709  if (!is_main)
1711 
1712  /* Pre-allocate expired nodes. */
1717 
1720 
1721  /* Start all processes. */
1722  if (is_main)
1723  {
1724  uword i;
1725 
1726  /*
1727  * Perform an initial barrier sync. Pays no attention to
1728  * the barrier sync hold-down timer scheme, which won't work
1729  * at this point in time.
1730  */
1732 
1733  nm->current_process_index = ~0;
1734  for (i = 0; i < vec_len (nm->processes); i++)
1735  cpu_time_now = dispatch_process (vm, nm->processes[i], /* frame */ 0,
1736  cpu_time_now);
1737  }
1738 
1739  while (1)
1740  {
1742 
1743  if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
1744  {
1745  if (!is_main)
1747  }
1748 
1749  if (!is_main)
1750  {
1753  frame_queue_check_counter))
1754  {
1755  u32 processed = 0;
1756 
1757  if (vm->check_frame_queues)
1758  {
1759  frame_queue_check_counter = 100;
1760  vm->check_frame_queues = 0;
1761  }
1762 
1763  vec_foreach (fqm, tm->frame_queue_mains)
1764  processed += vlib_frame_queue_dequeue (vm, fqm);
1765 
1766  /* No handoff queue work found? */
1767  if (processed)
1768  frame_queue_check_counter = 100;
1769  else
1770  frame_queue_check_counter--;
1771  }
1774  }
1775 
1776  /* Process pre-input nodes. */
1778  cpu_time_now = dispatch_node (vm, n,
1780  VLIB_NODE_STATE_POLLING,
1781  /* frame */ 0,
1782  cpu_time_now);
1783 
1784  /* Next process input nodes. */
1786  cpu_time_now = dispatch_node (vm, n,
1788  VLIB_NODE_STATE_POLLING,
1789  /* frame */ 0,
1790  cpu_time_now);
1791 
1792  if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
1793  vm->queue_signal_callback (vm);
1794 
1795  /* Next handle interrupts. */
1796  {
1797  /* unlocked read, for performance */
1798  uword l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1799  uword i;
1800  if (PREDICT_FALSE (l > 0))
1801  {
1802  u32 *tmp;
1803  if (!is_main)
1804  {
1806  /* Re-read w/ lock held, in case another thread added an item */
1807  l = _vec_len (nm->pending_interrupt_node_runtime_indices);
1808  }
1809 
1812  last_node_runtime_indices;
1813  last_node_runtime_indices = tmp;
1814  _vec_len (last_node_runtime_indices) = 0;
1815  if (!is_main)
1817  for (i = 0; i < l; i++)
1818  {
1820  last_node_runtime_indices[i]);
1821  cpu_time_now =
1823  VLIB_NODE_STATE_INTERRUPT,
1824  /* frame */ 0,
1825  cpu_time_now);
1826  }
1827  }
1828  }
1829  /* Input nodes may have added work to the pending vector.
1830  Process pending vector until there is nothing left.
1831  All pending vectors will be processed from input -> output. */
1832  for (i = 0; i < _vec_len (nm->pending_frames); i++)
1833  cpu_time_now = dispatch_pending_node (vm, i, cpu_time_now);
1834  /* Reset pending vector for next iteration. */
1835  _vec_len (nm->pending_frames) = 0;
1836 
1837  if (is_main)
1838  {
1839  /* *INDENT-OFF* */
1840  ELOG_TYPE_DECLARE (es) =
1841  {
1842  .format = "process tw start",
1843  .format_args = "",
1844  };
1845  ELOG_TYPE_DECLARE (ee) =
1846  {
1847  .format = "process tw end: %d",
1848  .format_args = "i4",
1849  };
1850  /* *INDENT-ON* */
1851 
1852  struct
1853  {
1854  int nready_procs;
1855  } *ed;
1856 
1857  /* Check if process nodes have expired from timing wheel. */
1859 
1861  ed = ELOG_DATA (&vlib_global_main.elog_main, es);
1862 
1865  ((TWT (tw_timer_wheel) *) nm->timing_wheel, vlib_time_now (vm),
1867 
1869 
1871  {
1872  ed = ELOG_DATA (&vlib_global_main.elog_main, ee);
1873  ed->nready_procs =
1874  _vec_len (nm->data_from_advancing_timing_wheel);
1875  }
1876 
1877  if (PREDICT_FALSE
1878  (_vec_len (nm->data_from_advancing_timing_wheel) > 0))
1879  {
1880  uword i;
1881 
1882  for (i = 0; i < _vec_len (nm->data_from_advancing_timing_wheel);
1883  i++)
1884  {
1887 
1889  {
1892  di);
1893  vlib_node_t *n =
1895  vlib_process_t *p =
1896  vec_elt (nm->processes, n->runtime_index);
1897  void *data;
1898  data =
1900  te->event_type_index,
1901  te->n_data_elts,
1902  te->n_data_elt_bytes);
1903  if (te->n_data_bytes < sizeof (te->inline_event_data))
1905  te->n_data_bytes);
1906  else
1907  {
1909  te->n_data_bytes);
1911  }
1913  }
1914  else
1915  {
1916  cpu_time_now = clib_cpu_time_now ();
1917  cpu_time_now =
1918  dispatch_suspended_process (vm, di, cpu_time_now);
1919  }
1920  }
1921  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
1922  }
1923  }
1925  /* Record time stamp in case there are no enabled nodes and above
1926  calls do not update time stamp. */
1927  cpu_time_now = clib_cpu_time_now ();
1928  }
1929 }
1930 
1931 static void
1933 {
1934  vlib_main_or_worker_loop (vm, /* is_main */ 1);
1935 }
1936 
1937 void
1939 {
1940  vlib_main_or_worker_loop (vm, /* is_main */ 0);
1941 }
1942 
1944 
1945 static clib_error_t *
1947 {
1948  int turn_on_mem_trace = 0;
1949 
1950  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1951  {
1952  if (unformat (input, "memory-trace"))
1953  turn_on_mem_trace = 1;
1954 
1955  else if (unformat (input, "elog-events %d",
1956  &vm->elog_main.event_ring_size))
1957  ;
1958  else if (unformat (input, "elog-post-mortem-dump"))
1959  vm->elog_post_mortem_dump = 1;
1960  else
1961  return unformat_parse_error (input);
1962  }
1963 
1964  unformat_free (input);
1965 
1966  /* Enable memory trace as early as possible. */
1967  if (turn_on_mem_trace)
1968  clib_mem_trace (1);
1969 
1970  return 0;
1971 }
1972 
1974 
1975 static void
1977 {
1978 }
1979 
1980 #define foreach_weak_reference_stub \
1981 _(vlib_map_stat_segment_init) \
1982 _(vpe_api_init) \
1983 _(vlibmemory_init) \
1984 _(map_api_segment_init)
1985 
1986 #define _(name) \
1987 clib_error_t *name (vlib_main_t *vm) __attribute__((weak)); \
1988 clib_error_t *name (vlib_main_t *vm) { return 0; }
1990 #undef _
1991 
1992 void vl_api_set_elog_main (elog_main_t * m) __attribute__ ((weak));
1993 void
1995 {
1996  clib_warning ("STUB");
1997 }
1998 
1999 int vl_api_set_elog_trace_api_messages (int enable) __attribute__ ((weak));
2000 int
2002 {
2003  clib_warning ("STUB");
2004  return 0;
2005 }
2006 
2007 int vl_api_get_elog_trace_api_messages (void) __attribute__ ((weak));
2008 int
2010 {
2011  clib_warning ("STUB");
2012  return 0;
2013 }
2014 
2015 /* Main function. */
2016 int
2018 {
2019  clib_error_t *volatile error;
2020  vlib_node_main_t *nm = &vm->node_main;
2021 
2023 
2024  clib_time_init (&vm->clib_time);
2025 
2026  /* Turn on event log. */
2027  if (!vm->elog_main.event_ring_size)
2028  vm->elog_main.event_ring_size = 128 << 10;
2030  elog_enable_disable (&vm->elog_main, 1);
2033 
2034  /* Default name. */
2035  if (!vm->name)
2036  vm->name = "VLIB";
2037 
2038  if ((error = vlib_physmem_init (vm)))
2039  {
2040  clib_error_report (error);
2041  goto done;
2042  }
2043 
2044  if ((error = vlib_map_stat_segment_init (vm)))
2045  {
2046  clib_error_report (error);
2047  goto done;
2048  }
2049 
2050  if ((error = vlib_buffer_main_init (vm)))
2051  {
2052  clib_error_report (error);
2053  goto done;
2054  }
2055 
2056  if ((error = vlib_thread_init (vm)))
2057  {
2058  clib_error_report (error);
2059  goto done;
2060  }
2061 
2062  /* Register static nodes so that init functions may use them. */
2064 
2065  /* Set seed for random number generator.
2066  Allow user to specify seed to make random sequence deterministic. */
2067  if (!unformat (input, "seed %wd", &vm->random_seed))
2068  vm->random_seed = clib_cpu_time_now ();
2070 
2071  /* Initialize node graph. */
2072  if ((error = vlib_node_main_init (vm)))
2073  {
2074  /* Arrange for graph hook up error to not be fatal when debugging. */
2075  if (CLIB_DEBUG > 0)
2076  clib_error_report (error);
2077  else
2078  goto done;
2079  }
2080 
2081  /* Direct call / weak reference, for vlib standalone use-cases */
2082  if ((error = vpe_api_init (vm)))
2083  {
2084  clib_error_report (error);
2085  goto done;
2086  }
2087 
2088  if ((error = vlibmemory_init (vm)))
2089  {
2090  clib_error_report (error);
2091  goto done;
2092  }
2093 
2094  if ((error = map_api_segment_init (vm)))
2095  {
2096  clib_error_report (error);
2097  goto done;
2098  }
2099 
2100  /* See unix/main.c; most likely already set up */
2101  if (vm->init_functions_called == 0)
2102  vm->init_functions_called = hash_create (0, /* value bytes */ 0);
2103  if ((error = vlib_call_all_init_functions (vm)))
2104  goto done;
2105 
2106  nm->timing_wheel = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
2108 
2110  _vec_len (nm->data_from_advancing_timing_wheel) = 0;
2111 
2112  /* Create the process timing wheel */
2113  TW (tw_timer_wheel_init) ((TWT (tw_timer_wheel) *) nm->timing_wheel,
2114  0 /* no callback */ ,
2115  10e-6 /* timer period 10us */ ,
2116  ~0 /* max expirations per call */ );
2117 
2119  _vec_len (vm->pending_rpc_requests) = 0;
2121  _vec_len (vm->processing_rpc_requests) = 0;
2122 
2123  if ((error = vlib_call_all_config_functions (vm, input, 0 /* is_early */ )))
2124  goto done;
2125 
2126  /* Sort per-thread init functions before we start threads */
2128 
2129  /* Call all main loop enter functions. */
2130  {
2131  clib_error_t *sub_error;
2132  sub_error = vlib_call_all_main_loop_enter_functions (vm);
2133  if (sub_error)
2134  clib_error_report (sub_error);
2135  }
2136 
2138  {
2140  vm->main_loop_exit_set = 1;
2141  break;
2142 
2144  goto done;
2145 
2146  default:
2147  error = vm->main_loop_error;
2148  goto done;
2149  }
2150 
2151  vlib_main_loop (vm);
2152 
2153 done:
2154  /* Call all exit functions. */
2155  {
2156  clib_error_t *sub_error;
2157  sub_error = vlib_call_all_main_loop_exit_functions (vm);
2158  if (sub_error)
2159  clib_error_report (sub_error);
2160  }
2161 
2162  if (error)
2163  clib_error_report (error);
2164 
2165  return 0;
2166 }
2167 
2168 int
2170 {
2172  pcap_main_t *pm = &vm->dispatch_pcap_main;
2173  vlib_trace_main_t *tm;
2174  vlib_trace_node_t *tn;
2175 
2176  if (a->status)
2177  {
2178  if (vm->dispatch_pcap_enable)
2179  {
2180  int i;
2182  (vm, "pcap dispatch capture enabled: %d of %d pkts...",
2184  vlib_cli_output (vm, "capture to file %s", pm->file_name);
2185 
2186  for (i = 0; i < vec_len (vm->dispatch_buffer_trace_nodes); i++)
2187  {
2188  vlib_cli_output (vm,
2189  "Buffer trace of %d pkts from %U enabled...",
2193  }
2194  }
2195  else
2196  vlib_cli_output (vm, "pcap dispatch capture disabled");
2197  return 0;
2198  }
2199 
2200  /* Consistency checks */
2201 
2202  /* Enable w/ capture already enabled not allowed */
2203  if (vm->dispatch_pcap_enable && a->enable)
2204  return -7; /* VNET_API_ERROR_INVALID_VALUE */
2205 
2206  /* Disable capture with capture already disabled, not interesting */
2207  if (vm->dispatch_pcap_enable == 0 && a->enable == 0)
2208  return -81; /* VNET_API_ERROR_VALUE_EXIST */
2209 
2210  /* Change number of packets to capture while capturing */
2211  if (vm->dispatch_pcap_enable && a->enable
2213  return -8; /* VNET_API_ERROR_INVALID_VALUE_2 */
2214 
2215  /* Independent of enable/disable, to allow buffer trace multi nodes */
2216  if (a->buffer_trace_node_index != ~0)
2217  {
2218  /* *INDENT-OFF* */
2220  {
2221  tm = &this_vlib_main->trace_main;
2222  tm->verbose = 0; /* not sure this ever did anything... */
2224  tn = tm->nodes + a->buffer_trace_node_index;
2225  tn->limit += a->buffer_traces_to_capture;
2226  tm->trace_enable = 1;
2227  }));
2228  /* *INDENT-ON* */
2230  }
2231 
2232  if (a->enable)
2233  {
2234  /* Clean up from previous run, if any */
2235  vec_free (pm->file_name);
2236  vec_free (pm->pcap_data);
2237  memset (pm, 0, sizeof (*pm));
2238 
2240  if (pm->lock == 0)
2241  clib_spinlock_init (&(pm->lock));
2242 
2243  if (a->filename == 0)
2244  a->filename = format (0, "/tmp/dispatch.pcap%c", 0);
2245 
2246  pm->file_name = (char *) a->filename;
2247  pm->n_packets_captured = 0;
2248  pm->packet_type = PCAP_PACKET_TYPE_vpp;
2250  /* *INDENT-OFF* */
2251  foreach_vlib_main (({this_vlib_main->dispatch_pcap_enable = 1;}));
2252  /* *INDENT-ON* */
2253  }
2254  else
2255  {
2256  /* *INDENT-OFF* */
2257  foreach_vlib_main (({this_vlib_main->dispatch_pcap_enable = 0;}));
2258  /* *INDENT-ON* */
2260  if (pm->n_packets_captured)
2261  {
2262  clib_error_t *error;
2264  vlib_cli_output (vm, "Write %d packets to %s, and stop capture...",
2265  pm->n_packets_captured, pm->file_name);
2266  error = pcap_write (pm);
2267  if (pm->flags & PCAP_MAIN_INIT_DONE)
2268  pcap_close (pm);
2269  /* Report I/O errors... */
2270  if (error)
2271  {
2272  clib_error_report (error);
2273  return -11; /* VNET_API_ERROR_SYSCALL_ERROR_1 */
2274  }
2275  return 0;
2276  }
2277  else
2278  return -6; /* VNET_API_ERROR_NO_SUCH_ENTRY */
2279  }
2280 
2281  return 0;
2282 }
2283 
2284 static clib_error_t *
2286  unformat_input_t * input, vlib_cli_command_t * cmd)
2287 {
2288  unformat_input_t _line_input, *line_input = &_line_input;
2290  u8 *filename = 0;
2291  u32 max = 1000;
2292  int rv;
2293  int enable = 0;
2294  int status = 0;
2295  u32 node_index = ~0, buffer_traces_to_capture = 100;
2296 
2297  /* Get a line of input. */
2298  if (!unformat_user (input, unformat_line_input, line_input))
2299  return 0;
2300 
2301  while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
2302  {
2303  if (unformat (line_input, "on %=", &enable, 1))
2304  ;
2305  else if (unformat (line_input, "enable %=", &enable, 1))
2306  ;
2307  else if (unformat (line_input, "off %=", &enable, 0))
2308  ;
2309  else if (unformat (line_input, "disable %=", &enable, 0))
2310  ;
2311  else if (unformat (line_input, "max %d", &max))
2312  ;
2313  else if (unformat (line_input, "packets-to-capture %d", &max))
2314  ;
2315  else if (unformat (line_input, "file %U", unformat_vlib_tmpfile,
2316  &filename))
2317  ;
2318  else if (unformat (line_input, "status %=", &status, 1))
2319  ;
2320  else if (unformat (line_input, "buffer-trace %U %d",
2321  unformat_vlib_node, vm, &node_index,
2322  &buffer_traces_to_capture))
2323  ;
2324  else
2325  {
2326  return clib_error_return (0, "unknown input `%U'",
2327  format_unformat_error, line_input);
2328  }
2329  }
2330 
2331  unformat_free (line_input);
2332 
2333  /* no need for memset (a, 0, sizeof (*a)), set all fields here. */
2334  a->filename = filename;
2335  a->enable = enable;
2336  a->status = status;
2337  a->packets_to_capture = max;
2338  a->buffer_trace_node_index = node_index;
2339  a->buffer_traces_to_capture = buffer_traces_to_capture;
2340 
2342 
2343  switch (rv)
2344  {
2345  case 0:
2346  break;
2347 
2348  case -7:
2349  return clib_error_return (0, "dispatch trace already enabled...");
2350 
2351  case -81:
2352  return clib_error_return (0, "dispatch trace already disabled...");
2353 
2354  case -8:
2355  return clib_error_return
2356  (0, "can't change number of records to capture while tracing...");
2357 
2358  case -11:
2359  return clib_error_return (0, "I/O writing trace capture...");
2360 
2361  case -6:
2362  return clib_error_return (0, "No packets captured...");
2363 
2364  default:
2365  vlib_cli_output (vm, "WARNING: trace configure returned %d", rv);
2366  break;
2367  }
2368  return 0;
2369 }
2370 
2371 /*?
2372  * This command is used to start or stop pcap dispatch trace capture, or show
2373  * the capture status.
2374  *
2375  * This command has the following optional parameters:
2376  *
2377  * - <b>on|off</b> - Used to start or stop capture.
2378  *
2379  * - <b>max <nn></b> - Depth of local buffer. Once '<em>nn</em>' number
2380  * of packets have been received, buffer is flushed to file. Once another
2381  * '<em>nn</em>' number of packets have been received, buffer is flushed
2382  * to file, overwriting previous write. If not entered, value defaults
2383  * to 100. Can only be updated if packet capture is off.
2384  *
2385  * - <b>file <name></b> - Used to specify the output filename. The file will
2386  * be placed in the '<em>/tmp</em>' directory, so only the filename is
2387  * supported. Directory should not be entered. If file already exists, file
2388  * will be overwritten. If no filename is provided, '<em>/tmp/vpe.pcap</em>'
2389  * will be used. Can only be updated if packet capture is off.
2390  *
2391  * - <b>status</b> - Displays the current status and configured attributes
2392  * associated with a packet capture. If packet capture is in progress,
2393  * '<em>status</em>' also will return the number of packets currently in
2394  * the local buffer. All additional attributes entered on command line
2395  * with '<em>status</em>' will be ignored and not applied.
2396  *
2397  * @cliexpar
2398  * Example of how to display the status of capture when off:
2399  * @cliexstart{pcap dispatch trace status}
2400  * max is 100, for any interface to file /tmp/vpe.pcap
2401  * pcap dispatch capture is off...
2402  * @cliexend
2403  * Example of how to start a dispatch trace capture:
2404  * @cliexstart{pcap dispatch trace on max 35 file dispatchTrace.pcap}
2405  * pcap dispatch capture on...
2406  * @cliexend
2407  * Example of how to start a dispatch trace capture with buffer tracing
2408  * @cliexstart{pcap dispatch trace on max 10000 file dispatchTrace.pcap buffer-trace dpdk-input 1000}
2409  * pcap dispatch capture on...
2410  * @cliexend
2411  * Example of how to display the status of a tx packet capture in progress:
2412  * @cliexstart{pcap tx trace status}
2413  * max is 35, dispatch trace to file /tmp/vppTest.pcap
2414  * pcap tx capture is on: 20 of 35 pkts...
2415  * @cliexend
2416  * Example of how to stop a tx packet capture:
2417  * @cliexstart{vppctl pcap dispatch trace off}
2418  * captured 21 pkts...
2419  * saved to /tmp/dispatchTrace.pcap...
2420  * @cliexend
2421 ?*/
2422 /* *INDENT-OFF* */
2424  .path = "pcap dispatch trace",
2425  .short_help =
2426  "pcap dispatch trace [on|off] [max <nn>] [file <name>] [status]\n"
2427  " [buffer-trace <input-node-name> <nn>]",
2428  .function = dispatch_trace_command_fn,
2429 };
2430 /* *INDENT-ON* */
2431 
2432 /*
2433  * fd.io coding-style-patch-verification: ON
2434  *
2435  * Local Variables:
2436  * eval: (c-set-style "gnu")
2437  * End:
2438  */
_vlib_init_function_list_elt_t * worker_init_function_registrations
Definition: main.h:224
u32 * next_nodes
Definition: node.h:335
static void elog_enable_disable(elog_main_t *em, int is_enabled)
Enable or disable event logging.
Definition: elog.h:220
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:440
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
clib_error_t * vlib_call_all_main_loop_exit_functions(vlib_main_t *vm)
Definition: init.c:398
static vlib_frame_t * vlib_frame_alloc(vlib_main_t *vm, vlib_node_runtime_t *from_node_runtime, u32 to_next_index)
Definition: main.c:172
uword * pending_rpc_requests
Definition: main.h:267
u64 internal_node_vectors
Definition: main.h:106
vlib_main_t vlib_global_main
Definition: main.c:1943
f64 time
Absolute time as floating point number in seconds.
Definition: elog.h:67
void TW() tw_timer_wheel_init(TWT(tw_timer_wheel) *tw, void *expired_timer_callback, f64 timer_interval_in_seconds, u32 max_expirations)
Initialize a tw timer wheel template instance.
u16 vector_size
Definition: node.h:321
u32 max_clock
Maximum clock cycle for an invocation.
Definition: node.h:478
u32 next_frame_index
Start of next frames for this node.
Definition: node.h:493
static clib_error_t * show_frame_stats(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:242
#define hash_set(h, key, value)
Definition: hash.h:255
char * file_name
File name of pcap output.
Definition: pcap.h:162
vlib_node_type_t
Definition: node.h:69
#define clib_min(x, y)
Definition: clib.h:295
clib_error_t * vlib_call_all_main_loop_enter_functions(vlib_main_t *vm)
Definition: init.c:391
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:102
static void vlib_next_frame_change_ownership(vlib_main_t *vm, vlib_node_runtime_t *node_runtime, u32 next_index)
Definition: main.c:272
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:80
static vlib_next_frame_t * vlib_node_get_next_frame(vlib_main_t *vm, u32 node_index, u32 next_index)
Get pointer to frame by (node_index, next_index).
Definition: node_funcs.h:299
u32 interrupt_threshold_vector_length
Definition: node.h:719
vlib_process_t ** processes
Definition: node.h:740
int vl_api_get_elog_trace_api_messages(void)
Definition: main.c:2009
u32 flags
flags
Definition: pcap.h:174
static uword vlib_process_bootstrap(uword _a)
Definition: main.c:1459
format_function_t format_vlib_node_name
Definition: node_funcs.h:1141
vlib_node_runtime_t node_runtime
Definition: node.h:554
a
Definition: bitmap.h:538
u32 n_suspends
Definition: node.h:582
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:461
u32 n_packets_to_capture
Number of packets to capture.
Definition: pcap.h:165
static u32 vlib_buffer_get_trace_index(vlib_buffer_t *b)
Extract the trace (pool) index from a trace handle.
Definition: buffer.h:388
vlib_frame_t * vlib_get_next_frame_internal(vlib_main_t *vm, vlib_node_runtime_t *node, u32 next_index, u32 allocate_new_next_frame)
Definition: main.c:355
uword * processing_rpc_requests
Definition: main.h:268
static u32 clib_get_current_numa_node()
Definition: cpu.h:172
u8 * format_vnet_buffer_opaque2(u8 *s, va_list *args)
Definition: main.c:993
#define VLIB_PCAP_MAJOR_VERSION
Definition: main.h:403
vlib_trace_node_t * nodes
Definition: trace.h:100
static void vlib_elog_main_loop_event(vlib_main_t *vm, u32 node_index, u64 time, u32 n_vectors, u32 is_return)
Definition: main.c:911
static clib_error_t * elog_save_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:713
int elog_post_mortem_dump
Definition: main.h:246
uword dispatch_pcap_enable
Definition: main.h:168
#define PREDICT_TRUE(x)
Definition: clib.h:112
uword random_seed
Definition: main.h:209
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
unsigned long u64
Definition: types.h:89
static void vlib_increment_main_loop_counter(vlib_main_t *vm)
Definition: main.h:371
elog_time_stamp_t init_time
Timestamps.
Definition: elog.h:172
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
static void vlib_process_sync_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:570
void clib_random_buffer_init(clib_random_buffer_t *b, uword seed)
Definition: random_buffer.c:62
#define NULL
Definition: clib.h:58
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 index
Definition: node.h:280
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:279
static void dummy_queue_signal_callback(vlib_main_t *vm)
Definition: main.c:1976
u32 current_process_index
Definition: node.h:743
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:110
static u32 clib_get_current_cpu_id()
Definition: cpu.h:164
u32 thread_index
Definition: main.h:218
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
static_always_inline uword vlib_process_startup(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f)
Definition: main.c:1486
u32 main_loop_exit_set
Definition: main.h:127
static u64 dispatch_pending_node(vlib_main_t *vm, uword pending_frame_index, u64 last_time_stamp)
Definition: main.c:1325
static vlib_frame_t * vlib_get_frame(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:216
u8 data[0]
Packet data.
Definition: buffer.h:181
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
static u64 clib_cpu_time_now(void)
Definition: time.h:75
u8 * format_vnet_buffer_opaque(u8 *s, va_list *args)
Definition: main.c:984
elog_track_t elog_track
Definition: threads.h:100
u32 clocks_since_last_overflow
Number of clock cycles.
Definition: node.h:476
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:561
int i
uword unformat_user(unformat_input_t *input, unformat_function_t *func,...)
Definition: unformat.c:989
static void vlib_process_update_stats(vlib_main_t *vm, vlib_process_t *p, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:687
static void elog_show_buffer_internal(vlib_main_t *vm, u32 n_events_to_show)
Definition: main.c:848
#define VLIB_MAIN_LOOP_EXIT_CLI
Definition: main.h:134
void clib_longjmp(clib_longjmp_t *save, uword return_value)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static vlib_frame_size_t * get_frame_size_info(vlib_node_main_t *nm, u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:93
static clib_error_t * elog_show_buffer(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:876
u8 * format_elog_event(u8 *s, va_list *va)
Definition: elog.c:296
clib_time_t clib_time
Definition: main.h:87
u32 numa_node
Definition: main.h:220
u8 * format_vnet_buffer_flags(u8 *s, va_list *args)
Definition: main.c:976
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:451
clib_spinlock_t pending_interrupt_lock
Definition: node.h:713
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
Definition: buffer_funcs.h:366
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:237
uword clib_setjmp(clib_longjmp_t *save, uword return_value_not_taken)
#define vec_alloc(V, N)
Allocate space for N more elements (no header, unspecified alignment)
Definition: vec.h:281
unsigned char u8
Definition: types.h:56
clib_error_t * vlib_sort_init_exit_functions(_vlib_init_function_list_elt_t **head)
Topological sorter for init function chains.
Definition: init.c:76
static uword vlib_process_stack_is_valid(vlib_process_t *p)
Definition: main.c:1445
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
u64 perf_counter0_ticks
Definition: node.h:236
double f64
Definition: types.h:142
u16 scalar_size
Definition: node.h:321
u8 state
Definition: node.h:309
u16 thread_index
thread this node runs on
Definition: node.h:521
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
u64 internal_node_calls
Definition: main.h:107
volatile uword check_frame_queues
Definition: main.h:264
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:396
static clib_error_t * dispatch_trace_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:2285
u32 * pending_interrupt_node_runtime_indices
Definition: node.h:712
u32 input_main_loops_per_call
For input nodes: decremented on each main loop interation until it reaches zero and function is calle...
Definition: node.h:498
static void vlib_main_loop(vlib_main_t *vm)
Definition: main.c:1932
#define foreach_weak_reference_stub
Definition: main.c:1980
void(** vlib_node_runtime_perf_counter_cbs)(struct vlib_main_t *, u64 *, u64 *, vlib_node_runtime_t *, vlib_frame_t *, int)
Definition: main.h:115
vlib_trace_header_t ** trace_buffer_pool
Definition: trace.h:86
#define static_always_inline
Definition: clib.h:99
uword unformat_vlib_tmpfile(unformat_input_t *input, va_list *args)
Definition: format.c:192
PCAP main state data structure.
Definition: pcap.h:156
u32 trace_enable
Definition: trace.h:97
clib_spinlock_t lock
spinlock to protect e.g.
Definition: pcap.h:159
#define VLIB_FRAME_NO_APPEND
Definition: node.h:419
vlib_node_function_t * function
Node function to call.
Definition: node.h:468
void elog_init(elog_main_t *em, u32 n_events)
Definition: elog.c:493
#define VLIB_FRAME_MAGIC
void vl_api_set_elog_main(elog_main_t *m)
Definition: main.c:1994
u32 main_loop_vectors_processed
Definition: main.h:102
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:375
u16 log2_n_stack_bytes
Definition: node.h:577
vlib_node_t ** nodes
Definition: node.h:699
u32 vectors_since_last_overflow
Number of vector elements processed by this node.
Definition: node.h:486
vlib_frame_t * frame
Definition: main.c:1454
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:185
#define clib_error_return(e, args...)
Definition: error.h:99
static void dispatch_pcap_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
Definition: main.c:1024
#define VLIB_FRAME_ALIGN
Definition: node.h:379
u64 perf_counter_vectors
Definition: node.h:238
u32 cpu_id
Definition: main.h:219
static uword vlib_timing_wheel_data_is_timed_event(u32 d)
Definition: node.h:673
#define VLIB_NODE_FLAG_SWITCH_FROM_INTERRUPT_TO_POLLING_MODE
Definition: node.h:304
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_CLOCK
Definition: node.h:568
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:243
unsigned int u32
Definition: types.h:88
u64 cpu_time_main_loop_start
Definition: main.h:96
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:709
static u32 vlib_frame_bytes(u32 n_scalar_bytes, u32 n_vector_bytes)
Definition: main.c:56
#define VLIB_MAIN_LOOP_EXIT_NONE
Definition: main.h:131
clib_error_t * vlib_node_main_init(vlib_main_t *vm)
Definition: node.c:639
#define VLIB_FRAME_SIZE
Definition: node.h:378
static vlib_next_frame_t * vlib_node_runtime_get_next_frame(vlib_main_t *vm, vlib_node_runtime_t *n, u32 next_index)
Definition: node_funcs.h:264
u32 perf_counter0_ticks_since_last_overflow
Perf counter 0 ticks.
Definition: node.h:489
u8 * pcap_buffer
Definition: main.h:170
#define VLIB_PROCESS_IS_RUNNING
Definition: node.h:574
#define VLIB_PROCESS_RETURN_LONGJMP_RETURN
Definition: node.h:559
u64 max_clock_n
Definition: node.h:235
unformat_function_t unformat_line_input
Definition: format.h:283
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
vl_api_fib_path_type_t type
Definition: fib_types.api:123
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
u32 calls_since_last_overflow
Number of calls.
Definition: node.h:484
char * name
Definition: main.h:140
clib_error_t * pcap_close(pcap_main_t *pm)
Close PCAP file.
Definition: pcap.c:74
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:37
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:519
static clib_error_t * elog_write_file(elog_main_t *em, char *clib_file, int flush_ring)
Definition: elog.h:532
u32 next_frame_index
Definition: node.h:458
static void * vlib_process_signal_event_helper(vlib_node_main_t *nm, vlib_node_t *n, vlib_process_t *p, uword t, uword n_data_elts, uword n_data_elt_bytes)
Definition: node_funcs.h:743
vlib_node_stats_t stats_total
Definition: node.h:270
static void vlib_put_next_frame_validate(vlib_main_t *vm, vlib_node_runtime_t *rt, u32 next_index, u32 n_vectors_left)
Definition: main.c:420
u16 state
Input node state.
Definition: node.h:511
static u8 * format_buffer_metadata(u8 *s, va_list *args)
Definition: main.c:1000
format_function_t format_vlib_trace
Definition: trace.h:113
vlib_signal_timed_event_data_t * signal_timed_event_data_pool
Definition: node.h:730
u16 frame_flags
Definition: node.h:385
#define VLIB_FRAME_IS_ALLOCATED
Definition: node.h:426
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:415
u8 * pcap_data
Vector of pcap data.
Definition: pcap.h:184
struct _unformat_input_t unformat_input_t
static_always_inline u64 dispatch_node(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_node_type_t type, vlib_node_state_t dispatch_state, vlib_frame_t *frame, u64 last_time_stamp)
Definition: main.c:1132
u32 n_alloc_frames
Definition: node.h:539
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:194
static uword vlib_timing_wheel_data_get_index(u32 d)
Definition: node.h:691
u32 *TW() tw_timer_expire_timers_vec(TWT(tw_timer_wheel) *tw, f64 now, u32 *vec)
void di(unformat_input_t *i)
Definition: unformat.c:163
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:287
elog_event_type_t * node_return_elog_event_types
Definition: main.h:204
#define TWT(a)
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:214
u32 polling_threshold_vector_length
Definition: node.h:718
#define ELOG_DATA(em, f)
Definition: elog.h:484
u64 * n_vectors_by_next_node
Definition: node.h:344
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:163
#define PREDICT_FALSE(x)
Definition: clib.h:111
#define always_inline
Definition: ipsec.h:28
f64 seconds_per_clock
Definition: time.h:57
void elog_post_mortem_dump(void)
Definition: main.c:750
u8 protocol_hint
Definition: node.h:315
vlib_frame_t * frame
Definition: node.h:455
u32 node_index
Node index.
Definition: node.h:496
#define foreach_vlib_main(body)
Definition: threads.h:241
uword * init_functions_called
Definition: main.h:215
static void * pcap_add_packet(pcap_main_t *pm, f64 time_now, u32 n_bytes_in_trace, u32 n_bytes_in_packet)
Add packet.
Definition: pcap_funcs.h:41
#define VLIB_PROCESS_STACK_MAGIC
Definition: node.h:625
#define VLIB_PROCESS_RESUME_LONGJMP_RESUME
Definition: node.h:565
void clib_time_init(clib_time_t *c)
Definition: time.c:178
uword * frame_size_hash
Definition: node.h:755
void elog_alloc(elog_main_t *em, u32 n_events)
Definition: elog.c:479
vlib_main_t * vm
Definition: in2out_ed.c:1810
clib_error_t * vlibmemory_init(vlib_main_t *vm)
Definition: memory_api.c:933
volatile u32 queue_signal_pending
Definition: main.h:231
static vlib_frame_t * vlib_frame_alloc_to_node(vlib_main_t *vm, u32 to_node_index, u32 frame_flags)
Definition: main.c:119
clib_error_t * map_api_segment_init(vlib_main_t *vm)
Definition: memory_api.c:476
u8 * name
Definition: node.h:264
static u64 dispatch_process(vlib_main_t *vm, vlib_process_t *p, vlib_frame_t *f, u64 last_time_stamp)
Definition: main.c:1517
static u32 vlib_node_runtime_update_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:621
u32 owner_node_index
Definition: node.h:355
void vlib_register_all_static_nodes(vlib_main_t *vm)
Definition: node.c:548
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:226
static u32 vlib_node_runtime_update_main_loop_vector_stats(vlib_main_t *vm, vlib_node_runtime_t *node, uword n_vectors)
Definition: node_funcs.h:1029
#define UNFORMAT_END_OF_INPUT
Definition: format.h:145
u32 flags
Definition: vhost_user.h:141
u8 inline_event_data[64 - 3 *sizeof(u32) - 2 *sizeof(u16)]
Definition: node.h:664
svmdb_client_t * c
u16 n_vectors
Definition: node.h:397
u32 runtime_index
Definition: node.h:283
#define CLIB_PREFETCH(addr, size, type)
Definition: cache.h:80
int elog_trace_graph_dispatch
Definition: main.h:198
static uword elog_n_events_in_buffer(elog_main_t *em)
Return number of events in the event-log buffer.
Definition: elog.h:191
static_always_inline void vlib_main_or_worker_loop(vlib_main_t *vm, int is_main)
Definition: main.c:1680
u32 node_runtime_index
Definition: node.h:452
vlib_pending_frame_t * pending_frames
Definition: node.h:725
u32 flow_id
Generic flow identifier.
Definition: buffer.h:127
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1642
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:342
clib_error_t * vpe_api_init(vlib_main_t *vm)
Definition: api.c:655
#define VLIB_NODE_FLAG_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:293
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_PROCESS_RESUME_PENDING
Definition: node.h:571
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
u32 perf_counter_vectors_since_last_overflow
Perf counter vectors.
Definition: node.h:491
#define VLIB_BUFFER_TRACE_TRAJECTORY
Compile time buffer trajectory tracing option Turn this on if you run into "bad monkey" contexts...
Definition: buffer.h:483
elog_main_t elog_main
Definition: main.h:193
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:804
#define VLIB_FRAME_PENDING
Definition: node.h:429
static u32 vlib_frame_vector_byte_offset(u32 scalar_size)
Definition: node_funcs.h:233
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:147
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
Definition: main.c:456
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:442
clib_error_t * pcap_write(pcap_main_t *pm)
Write PCAP file.
Definition: pcap.c:89
static uword round_pow2(uword x, uword pow2)
Definition: clib.h:241
u32 n_total_events_disable_limit
When count reaches limit logging is disabled.
Definition: elog.h:139
vlib_frame_t ** free_frames
Definition: node.h:542
vlib_main_t vlib_node_runtime_t * node
Definition: in2out_ed.c:1810
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:152
#define VLIB_PCAP_MINOR_VERSION
Definition: main.h:404
#define never_inline
Definition: clib.h:95
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define uword_to_pointer(u, type)
Definition: types.h:136
u16 cached_next_index
Next frame index that vector arguments were last enqueued to last time this node ran.
Definition: node.h:515
#define PCAP_MAIN_INIT_DONE
Definition: pcap.h:175
#define pool_put_index(p, i)
Free pool element with given index.
Definition: pool.h:316
#define ASSERT(truth)
u64 last_cpu_time
Definition: time.h:50
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:542
static void vlib_node_runtime_perf_counter(vlib_main_t *vm, u64 *pmc0, u64 *pmc1, vlib_node_runtime_t *node, vlib_frame_t *frame, int before_or_after)
Definition: main.c:675
static void elog_reset_buffer(elog_main_t *em)
Reset the event buffer.
Definition: elog.h:210
u64 perf_counter1_ticks
Definition: node.h:237
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:332
u8 data[128]
Definition: ipsec_types.api:87
#define ELOG_TRACK_DATA(em, f, track)
Definition: elog.h:478
vlib_frame_t * frame
Definition: node.h:406
uword event_ring_size
Power of 2 number of elements in ring.
Definition: elog.h:145
u16 flags
Definition: node.h:388
u32 main_loop_nodes_processed
Definition: main.h:103
u64 cpu
CPU cycle counter.
Definition: elog.h:126
void vlib_worker_thread_initial_barrier_sync_and_release(vlib_main_t *vm)
Definition: threads.c:1369
#define clib_error_report(e)
Definition: error.h:113
u32 elog_trace_graph_circuit_node_index
Definition: main.h:200
vlib_trace_main_t trace_main
Definition: main.h:164
int vl_api_set_elog_trace_api_messages(int enable)
Definition: main.c:2001
u8 * vnet_trace_dummy
Definition: trace.c:43
static uword pointer_to_uword(const void *p)
Definition: types.h:131
void(**volatile worker_thread_main_loop_callbacks)(struct vlib_main_t *)
Definition: main.h:237
#define A(x)
Definition: main.c:1021
void(* queue_signal_callback)(struct vlib_main_t *)
Definition: main.h:233
void vlib_gdb_show_event_log(void)
Definition: main.c:905
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
static_always_inline uword vlib_process_resume(vlib_process_t *p)
Definition: main.c:1504
#define VLIB_FRAME_OWNER
Definition: node.h:423
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:584
#define vec_elt(v, i)
Get vector value at index i.
int vlib_main(vlib_main_t *volatile vm, unformat_input_t *input)
Definition: main.c:2017
#define unformat_parse_error(input)
Definition: format.h:269
typedef key
Definition: ipsec_types.api:83
pcap_packet_type_t packet_type
Packet type.
Definition: pcap.h:168
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1938
static vlib_node_t * vlib_get_next_node(vlib_main_t *vm, u32 node_index, u32 next_index)
Get vlib node by graph arc (next) index.
Definition: node_funcs.h:72
void vl_api_send_pending_rpc_requests(vlib_main_t *)
Definition: main.c:1674
#define clib_mem_alloc_aligned_no_fail(size, align)
Definition: mem.h:187
CJ_GLOBAL_LOG_PROTOTYPE
Definition: main.c:49
u32 * data_from_advancing_timing_wheel
Definition: node.h:733
clib_error_t * vlib_physmem_init(vlib_main_t *vm)
Definition: physmem.c:94
u32 perf_counter1_ticks_since_last_overflow
Perf counter 1 ticks.
Definition: node.h:490
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u32 input_node_counts_by_state[VLIB_N_NODE_STATE]
Definition: node.h:752
vlib_pending_frame_t * suspended_process_frames
Definition: node.h:746
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
vlib_node_main_t node_main
Definition: main.h:158
static u32 * vlib_frame_find_magic(vlib_frame_t *f, vlib_node_t *node)
Definition: main.c:81
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
u32 owner_next_index
Definition: node.h:355
u8 vector_size
Definition: node.h:394
vlib_next_frame_t * next_frames
Definition: node.h:722
static clib_error_t * elog_stop(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:776
static void unformat_free(unformat_input_t *i)
Definition: format.h:163
clib_error_t * vlib_call_all_config_functions(vlib_main_t *vm, unformat_input_t *input, int is_early)
Definition: init.c:405
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
static uword elog_is_enabled(elog_main_t *em)
event logging enabled predicate
Definition: elog.h:276
#define VLIB_FRAME_TRACE
Definition: node.h:435
u32 stop_timer_handle
Definition: node.h:607
clib_error_t * vlib_call_all_init_functions(vlib_main_t *vm)
Definition: init.c:378
#define VLIB_FRAME_SIZE_EXTRA
unformat_function_t unformat_vlib_node
Definition: node_funcs.h:1147
static void elog_track(elog_main_t *em, elog_event_type_t *type, elog_track_t *track, u32 data)
Log a single-datum event to a specific track, non-inline version.
Definition: elog.h:398
static u64 dispatch_suspended_process(vlib_main_t *vm, uword process_index, u64 last_time_stamp)
Definition: main.c:1598
vlib_frame_size_t * frame_sizes
Definition: node.h:758
u64 resume_clock_interval
Definition: node.h:604
static void add_trajectory_trace(vlib_buffer_t *b, u32 node_index)
Definition: main.c:964
u16 flags
Definition: node.h:567
elog_event_type_t * node_call_elog_event_types
Definition: main.h:203
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:161
vlib_node_type_t type
Definition: node.h:277
static vlib_process_t * vlib_get_process_from_node(vlib_main_t *vm, vlib_node_t *node)
Definition: node_funcs.h:208
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
u8 * format_unformat_error(u8 *s, va_list *va)
Definition: unformat.c:91
u32 name_elog_string
Definition: node.h:267
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1496
#define clib_call_callbacks(h,...)
call the specified callback set
Definition: callback.h:67
vlib_node_state_t
Definition: node.h:250
pcap_main_t dispatch_pcap_main
Definition: main.h:167
u64 init_cpu_time
Definition: time.h:62
u32 TW() tw_timer_start(TWT(tw_timer_wheel) *tw, u32 user_id, u32 timer_id, u64 interval)
Start a Tw Timer.
u32 suspended_process_frame_index
Definition: node.h:579
static void validate_frame_magic(vlib_main_t *vm, vlib_frame_t *f, vlib_node_t *n, uword next_index)
Definition: main.c:346
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static clib_error_t * vlib_main_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: main.c:1946
static vlib_cli_command_t pcap_dispatch_trace_command
(constructor) VLIB_CLI_COMMAND (pcap_dispatch_trace_command)
Definition: main.c:2423
u32 internal_node_last_vectors_per_main_loop
Definition: main.h:112
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
static clib_error_t * vlib_cli_elog_clear(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:696
#define VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT
Definition: node.h:569
#define vec_foreach(var, vec)
Vector iterator.
vlib_main_t vlib_node_runtime_t vlib_frame_t * frame
Definition: in2out_ed.c:1811
elog_track_t default_track
Default track.
Definition: elog.h:166
clib_longjmp_t return_longjmp
Definition: node.h:557
u16 flags
Copy of main node flags.
Definition: node.h:509
u32 node_runtime_index
Definition: node.h:409
u32 n_total_events
Total number of events in buffer.
Definition: elog.h:135
u8 scalar_size
Definition: node.h:391
clib_longjmp_t main_loop_exit
Definition: main.h:130
clib_longjmp_t resume_longjmp
Definition: node.h:563
void clib_mem_trace(int enable)
Definition: mem_dlmalloc.c:423
static clib_error_t * elog_resize(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:816
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
Definition: buffer_funcs.h:244
#define VLIB_NODE_FLAG_TRACE
Definition: node.h:302
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
#define TW(a)
clib_error_t * vlib_map_stat_segment_init(void)
Definition: stat_segment.c:262
static u32 vlib_timing_wheel_data_set_suspended_process(u32 i)
Definition: node.h:679
void vlib_start_process(vlib_main_t *vm, uword process_index)
Definition: main.c:1590
int vlib_pcap_dispatch_trace_configure(vlib_pcap_dispatch_trace_args_t *a)
Definition: main.c:2169
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:689
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:95
void * timing_wheel
Definition: node.h:728
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
int elog_trace_graph_circuit
Definition: main.h:199
u32 n_packets_captured
Number of packets currently captured.
Definition: pcap.h:171
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
clib_random_buffer_t random_buffer
Definition: main.h:212
static clib_error_t * elog_restart(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: main.c:796
#define VLIB_NODE_FLAG_SWITCH_FROM_POLLING_TO_INTERRUPT_MODE
Definition: node.h:305
u8 *(* validate_frame)(struct vlib_main_t *vm, struct vlib_node_runtime_t *, struct vlib_frame_t *f)
Definition: node.h:365
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
u32 * dispatch_buffer_trace_nodes
Definition: main.h:169
clib_error_t * main_loop_error
Definition: main.h:137
vlib_process_t * process
Definition: main.c:1453
elog_event_t * elog_peek_events(elog_main_t *em)
convert event ring events to events, and return them as a vector.
Definition: elog.c:537
static uword elog_buffer_capacity(elog_main_t *em)
Return number of events which can fit in the event buffer.
Definition: elog.h:201
u64 cpu_time_last_node_dispatch
Definition: main.h:93
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:171
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:432
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:210
u32 vectors_since_last_overflow
Definition: node.h:438
u32 max_clock_n
Number of vectors in the recorded max_clock.
Definition: node.h:481
#define VLIB_PROCESS_RETURN_LONGJMP_SUSPEND
Definition: node.h:560