FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
threads.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #define _GNU_SOURCE
16 
17 #include <signal.h>
18 #include <math.h>
19 #include <vppinfra/format.h>
20 #include <vppinfra/linux/sysfs.h>
21 #include <vlib/vlib.h>
22 
23 #include <vlib/threads.h>
24 #include <vlib/unix/cj.h>
25 
27 
28 #define FRAME_QUEUE_NELTS 64
29 
30 u32
31 vl (void *p)
32 {
33  return vec_len (p);
34 }
35 
38 
39 /*
40  * Barrier tracing can be enabled on a normal build to collect information
41  * on barrier use, including timings and call stacks. Deliberately not
42  * keyed off CLIB_DEBUG, because that can add significant overhead which
43  * imapacts observed timings.
44  */
45 
46 u32
47 elog_global_id_for_msg_name (const char *msg_name)
48 {
49  uword *p, r;
50  static uword *h;
51  u8 *name_copy;
52 
53  if (!h)
54  h = hash_create_string (0, sizeof (uword));
55 
56  p = hash_get_mem (h, msg_name);
57  if (p)
58  return p[0];
59  r = elog_string (&vlib_global_main.elog_main, "%s", msg_name);
60 
61  name_copy = format (0, "%s%c", msg_name, 0);
62 
63  hash_set_mem (h, name_copy, r);
64 
65  return r;
66 }
67 
68 static inline void
69 barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed)
70 {
71  if (!vlib_worker_threads->barrier_elog_enabled)
72  return;
73 
74  /* *INDENT-OFF* */
75  ELOG_TYPE_DECLARE (e) =
76  {
77  .format = "bar-trace-%s-#%d",
78  .format_args = "T4i4",
79  };
80  /* *INDENT-ON* */
81  struct
82  {
83  u32 caller, count, t_entry, t_open, t_closed;
84  } *ed = 0;
85 
87  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
88  ed->caller = elog_global_id_for_msg_name
89  (vlib_worker_threads[0].barrier_caller);
90  ed->t_entry = (int) (1000000.0 * t_entry);
91  ed->t_open = (int) (1000000.0 * t_open);
92  ed->t_closed = (int) (1000000.0 * t_closed);
93 }
94 
95 static inline void
97 {
98  if (!vlib_worker_threads->barrier_elog_enabled)
99  return;
100 
101  /* *INDENT-OFF* */
102  ELOG_TYPE_DECLARE (e) =
103  {
104  .format = "bar-syncrec-%s-#%d",
105  .format_args = "T4i4",
106  };
107  /* *INDENT-ON* */
108  struct
109  {
110  u32 caller, depth;
111  } *ed = 0;
112 
114  ed->depth = (int) vlib_worker_threads[0].recursion_level - 1;
115  ed->caller = elog_global_id_for_msg_name
116  (vlib_worker_threads[0].barrier_caller);
117 }
118 
119 static inline void
121 {
122  if (!vlib_worker_threads->barrier_elog_enabled)
123  return;
124 
125  /* *INDENT-OFF* */
126  ELOG_TYPE_DECLARE (e) =
127  {
128  .format = "bar-relrrec-#%d",
129  .format_args = "i4",
130  };
131  /* *INDENT-ON* */
132  struct
133  {
134  u32 depth;
135  } *ed = 0;
136 
138  ed->depth = (int) vlib_worker_threads[0].recursion_level;
139 }
140 
141 static inline void
142 barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main)
143 {
144  if (!vlib_worker_threads->barrier_elog_enabled)
145  return;
146 
147  /* *INDENT-OFF* */
148  ELOG_TYPE_DECLARE (e) =
149  {
150  .format = "bar-rel-#%d-e%d-u%d-t%d",
151  .format_args = "i4i4i4i4",
152  };
153  /* *INDENT-ON* */
154  struct
155  {
156  u32 count, t_entry, t_update_main, t_closed_total;
157  } *ed = 0;
158 
160  ed->t_entry = (int) (1000000.0 * t_entry);
161  ed->t_update_main = (int) (1000000.0 * t_update_main);
162  ed->t_closed_total = (int) (1000000.0 * t_closed_total);
163  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
164 
165  /* Reset context for next trace */
166  vlib_worker_threads[0].barrier_context = NULL;
167 }
168 
169 uword
171 {
172  u32 len;
173 
174  len = vec_len (vlib_thread_stacks);
175  if (len == 0)
176  return 1;
177  else
178  return len;
179 }
180 
181 void
183 {
184  int pthread_setname_np (pthread_t __target_thread, const char *__name);
185  int rv;
186  pthread_t thread = pthread_self ();
187 
188  if (thread)
189  {
190  rv = pthread_setname_np (thread, name);
191  if (rv)
192  clib_warning ("pthread_setname_np returned %d", rv);
193  }
194 }
195 
196 static int
197 sort_registrations_by_no_clone (void *a0, void *a1)
198 {
199  vlib_thread_registration_t **tr0 = a0;
200  vlib_thread_registration_t **tr1 = a1;
201 
202  return ((i32) ((*tr0)->no_data_structure_clone)
203  - ((i32) ((*tr1)->no_data_structure_clone)));
204 }
205 
206 static uword *
208 {
209  FILE *fp;
210  uword *r = 0;
211 
212  fp = fopen (filename, "r");
213 
214  if (fp != NULL)
215  {
216  u8 *buffer = 0;
217  vec_validate (buffer, 256 - 1);
218  if (fgets ((char *) buffer, 256, fp))
219  {
220  unformat_input_t in;
221  unformat_init_string (&in, (char *) buffer,
222  strlen ((char *) buffer));
223  if (unformat (&in, "%U", unformat_bitmap_list, &r) != 1)
224  clib_warning ("unformat_bitmap_list failed");
225  unformat_free (&in);
226  }
227  vec_free (buffer);
228  fclose (fp);
229  }
230  return r;
231 }
232 
233 
234 /* Called early in the init sequence */
235 
236 clib_error_t *
238 {
242  u32 n_vlib_mains = 1;
243  u32 first_index = 1;
244  u32 i;
245  uword *avail_cpu;
246 
247  /* get bitmaps of active cpu cores and sockets */
248  tm->cpu_core_bitmap =
249  clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online");
250  tm->cpu_socket_bitmap =
251  clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online");
252 
253  avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
254 
255  /* skip cores */
256  for (i = 0; i < tm->skip_cores; i++)
257  {
258  uword c = clib_bitmap_first_set (avail_cpu);
259  if (c == ~0)
260  return clib_error_return (0, "no available cpus to skip");
261 
262  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
263  }
264 
265  /* grab cpu for main thread */
266  if (tm->main_lcore == ~0)
267  {
268  /* if main-lcore is not set, we try to use lcore 1 */
269  if (clib_bitmap_get (avail_cpu, 1))
270  tm->main_lcore = 1;
271  else
272  tm->main_lcore = clib_bitmap_first_set (avail_cpu);
273  if (tm->main_lcore == (u8) ~ 0)
274  return clib_error_return (0, "no available cpus to be used for the"
275  " main thread");
276  }
277  else
278  {
279  if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0)
280  return clib_error_return (0, "cpu %u is not available to be used"
281  " for the main thread", tm->main_lcore);
282  }
283  avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
284 
285  /* assume that there is socket 0 only if there is no data from sysfs */
286  if (!tm->cpu_socket_bitmap)
287  tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1);
288 
289  /* pin main thread to main_lcore */
291  {
293  }
294  else
295  {
296  cpu_set_t cpuset;
297  CPU_ZERO (&cpuset);
298  CPU_SET (tm->main_lcore, &cpuset);
299  pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
300  }
301 
302  /* as many threads as stacks... */
303  vec_validate_aligned (vlib_worker_threads, vec_len (vlib_thread_stacks) - 1,
305 
306  /* Preallocate thread 0 */
307  _vec_len (vlib_worker_threads) = 1;
311  w->cpu_id = tm->main_lcore;
312  w->lwp = syscall (SYS_gettid);
313  w->thread_id = pthread_self ();
314  tm->n_vlib_mains = 1;
315 
316  if (tm->sched_policy != ~0)
317  {
318  struct sched_param sched_param;
319  if (!sched_getparam (w->lwp, &sched_param))
320  {
321  if (tm->sched_priority != ~0)
322  sched_param.sched_priority = tm->sched_priority;
323  sched_setscheduler (w->lwp, tm->sched_policy, &sched_param);
324  }
325  }
326 
327  /* assign threads to cores and set n_vlib_mains */
328  tr = tm->next;
329 
330  while (tr)
331  {
332  vec_add1 (tm->registrations, tr);
333  tr = tr->next;
334  }
335 
337 
338  for (i = 0; i < vec_len (tm->registrations); i++)
339  {
340  int j;
341  tr = tm->registrations[i];
342  tr->first_index = first_index;
343  first_index += tr->count;
344  n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0;
345 
346  /* construct coremask */
347  if (tr->use_pthreads || !tr->count)
348  continue;
349 
350  if (tr->coremask)
351  {
352  uword c;
353  /* *INDENT-OFF* */
354  clib_bitmap_foreach (c, tr->coremask, ({
355  if (clib_bitmap_get(avail_cpu, c) == 0)
356  return clib_error_return (0, "cpu %u is not available to be used"
357  " for the '%s' thread",c, tr->name);
358 
359  avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
360  }));
361 /* *INDENT-ON* */
362 
363  }
364  else
365  {
366  for (j = 0; j < tr->count; j++)
367  {
368  uword c = clib_bitmap_first_set (avail_cpu);
369  if (c == ~0)
370  return clib_error_return (0,
371  "no available cpus to be used for"
372  " the '%s' thread", tr->name);
373 
374  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
375  tr->coremask = clib_bitmap_set (tr->coremask, c, 1);
376  }
377  }
378  }
379 
380  clib_bitmap_free (avail_cpu);
381 
382  tm->n_vlib_mains = n_vlib_mains;
383 
384  vec_validate_aligned (vlib_worker_threads, first_index - 1,
386 
387  return 0;
388 }
389 
392 {
393  vlib_frame_queue_t *fq;
394 
395  fq = clib_mem_alloc_aligned (sizeof (*fq), CLIB_CACHE_LINE_BYTES);
396  clib_memset (fq, 0, sizeof (*fq));
397  fq->nelts = nelts;
398  fq->vector_threshold = 128; // packets
400 
401  if (1)
402  {
403  if (((uword) & fq->tail) & (CLIB_CACHE_LINE_BYTES - 1))
404  fformat (stderr, "WARNING: fq->tail unaligned\n");
405  if (((uword) & fq->head) & (CLIB_CACHE_LINE_BYTES - 1))
406  fformat (stderr, "WARNING: fq->head unaligned\n");
407  if (((uword) fq->elts) & (CLIB_CACHE_LINE_BYTES - 1))
408  fformat (stderr, "WARNING: fq->elts unaligned\n");
409 
410  if (sizeof (fq->elts[0]) % CLIB_CACHE_LINE_BYTES)
411  fformat (stderr, "WARNING: fq->elts[0] size %d\n",
412  sizeof (fq->elts[0]));
413  if (nelts & (nelts - 1))
414  {
415  fformat (stderr, "FATAL: nelts MUST be a power of 2\n");
416  abort ();
417  }
418  }
419 
420  return (fq);
421 }
422 
423 void vl_msg_api_handler_no_free (void *) __attribute__ ((weak));
424 void
426 {
427 }
428 
429 /* Turned off, save as reference material... */
430 #if 0
431 static inline int
432 vlib_frame_queue_dequeue_internal (int thread_id,
433  vlib_main_t * vm, vlib_node_main_t * nm)
434 {
435  vlib_frame_queue_t *fq = vlib_frame_queues[thread_id];
437  vlib_frame_t *f;
440  u32 node_runtime_index;
441  int msg_type;
442  u64 before;
443  int processed = 0;
444 
445  ASSERT (vm == vlib_mains[thread_id]);
446 
447  while (1)
448  {
449  if (fq->head == fq->tail)
450  return processed;
451 
452  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
453 
454  if (!elt->valid)
455  return processed;
456 
457  before = clib_cpu_time_now ();
458 
459  f = elt->frame;
460  node_runtime_index = elt->node_runtime_index;
461  msg_type = elt->msg_type;
462 
463  switch (msg_type)
464  {
465  case VLIB_FRAME_QUEUE_ELT_FREE_BUFFERS:
467  /* note fallthrough... */
468  case VLIB_FRAME_QUEUE_ELT_FREE_FRAME:
470  node_runtime_index);
471  vlib_frame_free (vm, r, f);
472  break;
474  vec_add2 (vm->node_main.pending_frames, p, 1);
476  p->node_runtime_index = elt->node_runtime_index;
477  p->frame_index = vlib_frame_index (vm, f);
479  fq->dequeue_vectors += (u64) f->n_vectors;
480  break;
481  case VLIB_FRAME_QUEUE_ELT_API_MSG:
483  break;
484  default:
485  clib_warning ("bogus frame queue message, type %d", msg_type);
486  break;
487  }
488  elt->valid = 0;
489  fq->dequeues++;
490  fq->dequeue_ticks += clib_cpu_time_now () - before;
492  fq->head++;
493  processed++;
494  }
495  ASSERT (0);
496  return processed;
497 }
498 
499 int
500 vlib_frame_queue_dequeue (int thread_id,
501  vlib_main_t * vm, vlib_node_main_t * nm)
502 {
503  return vlib_frame_queue_dequeue_internal (thread_id, vm, nm);
504 }
505 
506 int
507 vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index,
508  u32 frame_queue_index, vlib_frame_t * frame,
510 {
511  vlib_frame_queue_t *fq = vlib_frame_queues[frame_queue_index];
513  u32 save_count;
514  u64 new_tail;
515  u64 before = clib_cpu_time_now ();
516 
517  ASSERT (fq);
518 
519  new_tail = clib_atomic_add_fetch (&fq->tail, 1);
520 
521  /* Wait until a ring slot is available */
522  while (new_tail >= fq->head + fq->nelts)
523  {
524  f64 b4 = vlib_time_now_ticks (vm, before);
526  /* Bad idea. Dequeue -> enqueue -> dequeue -> trouble */
527  // vlib_frame_queue_dequeue (vm->thread_index, vm, nm);
528  }
529 
530  elt = fq->elts + (new_tail & (fq->nelts - 1));
531 
532  /* this would be very bad... */
533  while (elt->valid)
534  {
535  }
536 
537  /* Once we enqueue the frame, frame->n_vectors is owned elsewhere... */
538  save_count = frame->n_vectors;
539 
540  elt->frame = frame;
541  elt->node_runtime_index = node_runtime_index;
542  elt->msg_type = type;
544  elt->valid = 1;
545 
546  return save_count;
547 }
548 #endif /* 0 */
549 
550 /* To be called by vlib worker threads upon startup */
551 void
553 {
555 
556  /*
557  * Note: disabling signals in worker threads as follows
558  * prevents the api post-mortem dump scheme from working
559  * {
560  * sigset_t s;
561  * sigfillset (&s);
562  * pthread_sigmask (SIG_SETMASK, &s, 0);
563  * }
564  */
565 
567 
568  if (vec_len (tm->thread_prefix) && w->registration->short_name)
569  {
570  w->name = format (0, "%v_%s_%d%c", tm->thread_prefix,
571  w->registration->short_name, w->instance_id, '\0');
572  vlib_set_thread_name ((char *) w->name);
573  }
574 
575  if (!w->registration->use_pthreads)
576  {
577 
578  /* Initial barrier sync, for both worker and i/o threads */
579  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
580 
581  while (*vlib_worker_threads->wait_at_barrier)
582  ;
583 
584  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
585  }
586 }
587 
588 void *
590 {
591  void *rv;
592  vlib_worker_thread_t *w = arg;
593 
594  w->lwp = syscall (SYS_gettid);
595  w->thread_id = pthread_self ();
596 
597  __os_thread_index = w - vlib_worker_threads;
598 
599  rv = (void *) clib_calljmp
600  ((uword (*)(uword)) w->thread_function,
602  /* NOTREACHED, we hope */
603  return rv;
604 }
605 
606 static void
608 {
609  const char *sys_cpu_path = "/sys/devices/system/cpu/cpu";
610  u8 *p = 0;
611  int core_id = -1, socket_id = -1;
612 
613  p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, cpu_id, 0);
614  clib_sysfs_read ((char *) p, "%d", &core_id);
615  vec_reset_length (p);
616  p =
617  format (p, "%s%u/topology/physical_package_id%c", sys_cpu_path, cpu_id,
618  0);
619  clib_sysfs_read ((char *) p, "%d", &socket_id);
620  vec_free (p);
621 
622  w->core_id = core_id;
623  w->socket_id = socket_id;
624 }
625 
626 static clib_error_t *
627 vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
628 {
630  void *(*fp_arg) (void *) = fp;
631 
632  w->cpu_id = cpu_id;
633  vlib_get_thread_core_socket (w, cpu_id);
635  return tm->cb.vlib_launch_thread_cb (fp, (void *) w, cpu_id);
636  else
637  {
638  pthread_t worker;
639  cpu_set_t cpuset;
640  CPU_ZERO (&cpuset);
641  CPU_SET (cpu_id, &cpuset);
642 
643  if (pthread_create (&worker, NULL /* attr */ , fp_arg, (void *) w))
644  return clib_error_return_unix (0, "pthread_create");
645 
646  if (pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset))
647  return clib_error_return_unix (0, "pthread_setaffinity_np");
648 
649  return 0;
650  }
651 }
652 
653 static clib_error_t *
655 {
656  int i, j;
658  vlib_main_t *vm_clone;
659  void *oldheap;
663  u32 n_vlib_mains = tm->n_vlib_mains;
664  u32 worker_thread_index;
665  u8 *main_heap = clib_mem_get_per_cpu_heap ();
666 
667  vec_reset_length (vlib_worker_threads);
668 
669  /* Set up the main thread */
670  vec_add2_aligned (vlib_worker_threads, w, 1, CLIB_CACHE_LINE_BYTES);
671  w->elog_track.name = "main thread";
673 
674  if (vec_len (tm->thread_prefix))
675  {
676  w->name = format (0, "%v_main%c", tm->thread_prefix, '\0');
677  vlib_set_thread_name ((char *) w->name);
678  }
679 
680  vm->elog_main.lock =
682  vm->elog_main.lock[0] = 0;
683 
684  if (n_vlib_mains > 1)
685  {
686  /* Replace hand-crafted length-1 vector with a real vector */
687  vlib_mains = 0;
688 
691  _vec_len (vlib_mains) = 0;
693 
694  vlib_worker_threads->wait_at_barrier =
696  vlib_worker_threads->workers_at_barrier =
698 
699  vlib_worker_threads->node_reforks_required =
701 
702  /* We'll need the rpc vector lock... */
704 
705  /* Ask for an initial barrier sync */
706  *vlib_worker_threads->workers_at_barrier = 0;
707  *vlib_worker_threads->wait_at_barrier = 1;
708 
709  /* Without update or refork */
710  *vlib_worker_threads->node_reforks_required = 0;
712 
713  /* init timing */
714  vm->barrier_epoch = 0;
715  vm->barrier_no_close_before = 0;
716 
717  worker_thread_index = 1;
718 
719  for (i = 0; i < vec_len (tm->registrations); i++)
720  {
721  vlib_node_main_t *nm, *nm_clone;
722  vlib_buffer_free_list_t *fl_clone, *fl_orig;
723  vlib_buffer_free_list_t *orig_freelist_pool;
724  int k;
725 
726  tr = tm->registrations[i];
727 
728  if (tr->count == 0)
729  continue;
730 
731  for (k = 0; k < tr->count; k++)
732  {
733  vlib_node_t *n;
734 
735  vec_add2 (vlib_worker_threads, w, 1);
736  /* Currently unused, may not really work */
737  if (tr->mheap_size)
738  {
739 #if USE_DLMALLOC == 0
740  w->thread_mheap =
741  mheap_alloc (0 /* use VM */ , tr->mheap_size);
742 #else
744  0 /* unlocked */ );
745 #endif
746  }
747  else
748  w->thread_mheap = main_heap;
749 
750  w->thread_stack =
751  vlib_thread_stack_init (w - vlib_worker_threads);
752  w->thread_function = tr->function;
753  w->thread_function_arg = w;
754  w->instance_id = k;
755  w->registration = tr;
756 
757  w->elog_track.name =
758  (char *) format (0, "%s %d", tr->name, k + 1);
759  vec_add1 (w->elog_track.name, 0);
761 
762  if (tr->no_data_structure_clone)
763  continue;
764 
765  /* Fork vlib_global_main et al. Look for bugs here */
766  oldheap = clib_mem_set_heap (w->thread_mheap);
767 
768  vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone),
770  clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone));
771 
772  vm_clone->thread_index = worker_thread_index;
773  vm_clone->heap_base = w->thread_mheap;
774  vm_clone->heap_aligned_base = (void *)
775  (((uword) w->thread_mheap) & ~(VLIB_FRAME_ALIGN - 1));
776  vm_clone->init_functions_called =
777  hash_create (0, /* value bytes */ 0);
778  vm_clone->pending_rpc_requests = 0;
779  vec_validate (vm_clone->pending_rpc_requests, 0);
780  _vec_len (vm_clone->pending_rpc_requests) = 0;
781  clib_memset (&vm_clone->random_buffer, 0,
782  sizeof (vm_clone->random_buffer));
783 
784  nm = &vlib_mains[0]->node_main;
785  nm_clone = &vm_clone->node_main;
786  /* fork next frames array, preserving node runtime indices */
787  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
789  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
790  {
791  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
792  u32 save_node_runtime_index;
793  u32 save_flags;
794 
795  save_node_runtime_index = nf->node_runtime_index;
796  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
798  nf->node_runtime_index = save_node_runtime_index;
799  nf->flags = save_flags;
800  }
801 
802  /* fork the frame dispatch queue */
803  nm_clone->pending_frames = 0;
804  vec_validate (nm_clone->pending_frames, 10); /* $$$$$?????? */
805  _vec_len (nm_clone->pending_frames) = 0;
806 
807  /* fork nodes */
808  nm_clone->nodes = 0;
809 
810  /* Allocate all nodes in single block for speed */
811  n = clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*n));
812 
813  for (j = 0; j < vec_len (nm->nodes); j++)
814  {
815  clib_memcpy (n, nm->nodes[j], sizeof (*n));
816  /* none of the copied nodes have enqueue rights given out */
818  clib_memset (&n->stats_total, 0, sizeof (n->stats_total));
820  sizeof (n->stats_last_clear));
821  vec_add1 (nm_clone->nodes, n);
822  n++;
823  }
827  vec_foreach (rt,
829  {
830  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
831  rt->thread_index = vm_clone->thread_index;
832  /* copy initial runtime_data from node */
833  if (n->runtime_data && n->runtime_data_bytes > 0)
836  n->runtime_data_bytes));
837  }
838 
843  {
844  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
845  rt->thread_index = vm_clone->thread_index;
846  /* copy initial runtime_data from node */
847  if (n->runtime_data && n->runtime_data_bytes > 0)
850  n->runtime_data_bytes));
851  }
852 
853  nm_clone->processes = vec_dup_aligned (nm->processes,
855 
856  /* zap the (per worker) frame freelists, etc */
857  nm_clone->frame_sizes = 0;
858  nm_clone->frame_size_hash = hash_create (0, sizeof (uword));
859  nm_clone->node_by_error = nm->node_by_error;
860 
861  /* Packet trace buffers are guaranteed to be empty, nothing to do here */
862 
863  clib_mem_set_heap (oldheap);
865 
867  (vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES);
869  (vlib_mains[0]->error_main.counters_last_clear,
871 
872  /* Fork the vlib_buffer_main_t free lists, etc. */
873  orig_freelist_pool = vm_clone->buffer_free_list_pool;
874  vm_clone->buffer_free_list_pool = 0;
875 
876  /* *INDENT-OFF* */
877  pool_foreach (fl_orig, orig_freelist_pool,
878  ({
880  fl_clone, CLIB_CACHE_LINE_BYTES);
881  ASSERT (fl_orig - orig_freelist_pool
882  == fl_clone - vm_clone->buffer_free_list_pool);
883 
884  fl_clone[0] = fl_orig[0];
885  fl_clone->buffers = 0;
886  vec_validate(fl_clone->buffers, 0);
887  vec_reset_length(fl_clone->buffers);
888  fl_clone->n_alloc = 0;
889  }));
890 /* *INDENT-ON* */
891 
892  worker_thread_index++;
893  }
894  }
895  }
896  else
897  {
898  /* only have non-data-structure copy threads to create... */
899  for (i = 0; i < vec_len (tm->registrations); i++)
900  {
901  tr = tm->registrations[i];
902 
903  for (j = 0; j < tr->count; j++)
904  {
905  vec_add2 (vlib_worker_threads, w, 1);
906  if (tr->mheap_size)
907  {
908 #if USE_DLMALLOC == 0
909  w->thread_mheap =
910  mheap_alloc (0 /* use VM */ , tr->mheap_size);
911 #else
912  w->thread_mheap =
913  create_mspace (tr->mheap_size, 0 /* locked */ );
914 #endif
915  }
916  else
917  w->thread_mheap = main_heap;
918  w->thread_stack =
919  vlib_thread_stack_init (w - vlib_worker_threads);
920  w->thread_function = tr->function;
921  w->thread_function_arg = w;
922  w->instance_id = j;
923  w->elog_track.name =
924  (char *) format (0, "%s %d", tr->name, j + 1);
925  w->registration = tr;
926  vec_add1 (w->elog_track.name, 0);
928  }
929  }
930  }
931 
932  worker_thread_index = 1;
933 
934  for (i = 0; i < vec_len (tm->registrations); i++)
935  {
936  clib_error_t *err;
937  int j;
938 
939  tr = tm->registrations[i];
940 
941  if (tr->use_pthreads || tm->use_pthreads)
942  {
943  for (j = 0; j < tr->count; j++)
944  {
945  w = vlib_worker_threads + worker_thread_index++;
947  w, 0);
948  if (err)
949  clib_error_report (err);
950  }
951  }
952  else
953  {
954  uword c;
955  /* *INDENT-OFF* */
956  clib_bitmap_foreach (c, tr->coremask, ({
957  w = vlib_worker_threads + worker_thread_index++;
958  err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
959  w, c);
960  if (err)
961  clib_error_report (err);
962  }));
963  /* *INDENT-ON* */
964  }
965  }
968  return 0;
969 }
970 
972 
973 
974 static inline void
976 {
977  int i, j;
978  vlib_main_t *vm;
979  vlib_node_main_t *nm, *nm_clone;
980  vlib_main_t *vm_clone;
982  never_inline void
985  uword n_calls,
986  uword n_vectors, uword n_clocks);
987 
988  ASSERT (vlib_get_thread_index () == 0);
989 
990  vm = vlib_mains[0];
991  nm = &vm->node_main;
992 
993  ASSERT (*vlib_worker_threads->wait_at_barrier == 1);
994 
995  /*
996  * Scrape all runtime stats, so we don't lose node runtime(s) with
997  * pending counts, or throw away worker / io thread counts.
998  */
999  for (j = 0; j < vec_len (nm->nodes); j++)
1000  {
1001  vlib_node_t *n;
1002  n = nm->nodes[j];
1003  vlib_node_sync_stats (vm, n);
1004  }
1005 
1006  for (i = 1; i < vec_len (vlib_mains); i++)
1007  {
1008  vlib_node_t *n;
1009 
1010  vm_clone = vlib_mains[i];
1011  nm_clone = &vm_clone->node_main;
1012 
1013  for (j = 0; j < vec_len (nm_clone->nodes); j++)
1014  {
1015  n = nm_clone->nodes[j];
1016 
1017  rt = vlib_node_get_runtime (vm_clone, n->index);
1018  vlib_node_runtime_sync_stats (vm_clone, rt, 0, 0, 0);
1019  }
1020  }
1021 
1022  /* Per-worker clone rebuilds are now done on each thread */
1023 }
1024 
1025 
1026 void
1028 {
1029  vlib_main_t *vm, *vm_clone;
1030  vlib_node_main_t *nm, *nm_clone;
1031  vlib_node_t **old_nodes_clone;
1032  vlib_node_runtime_t *rt, *old_rt;
1033 
1034  vlib_node_t *new_n_clone;
1035 
1036  int j;
1037 
1038  vm = vlib_mains[0];
1039  nm = &vm->node_main;
1040  vm_clone = vlib_get_main ();
1041  nm_clone = &vm_clone->node_main;
1042 
1043  /* Re-clone error heap */
1044  u64 *old_counters = vm_clone->error_main.counters;
1045  u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear;
1046 
1047  clib_memcpy_fast (&vm_clone->error_main, &vm->error_main,
1048  sizeof (vm->error_main));
1049  j = vec_len (vm->error_main.counters) - 1;
1050  vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES);
1051  vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
1052  vm_clone->error_main.counters = old_counters;
1053  vm_clone->error_main.counters_last_clear = old_counters_all_clear;
1054 
1055  nm_clone = &vm_clone->node_main;
1056  vec_free (nm_clone->next_frames);
1057  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
1059 
1060  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
1061  {
1062  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
1063  u32 save_node_runtime_index;
1064  u32 save_flags;
1065 
1066  save_node_runtime_index = nf->node_runtime_index;
1067  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
1068  vlib_next_frame_init (nf);
1069  nf->node_runtime_index = save_node_runtime_index;
1070  nf->flags = save_flags;
1071  }
1072 
1073  old_nodes_clone = nm_clone->nodes;
1074  nm_clone->nodes = 0;
1075 
1076  /* re-fork nodes */
1077 
1078  /* Allocate all nodes in single block for speed */
1079  new_n_clone =
1080  clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*new_n_clone));
1081  for (j = 0; j < vec_len (nm->nodes); j++)
1082  {
1083  vlib_node_t *old_n_clone;
1084  vlib_node_t *new_n;
1085 
1086  new_n = nm->nodes[j];
1087  old_n_clone = old_nodes_clone[j];
1088 
1089  clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n));
1090  /* none of the copied nodes have enqueue rights given out */
1092 
1093  if (j >= vec_len (old_nodes_clone))
1094  {
1095  /* new node, set to zero */
1096  clib_memset (&new_n_clone->stats_total, 0,
1097  sizeof (new_n_clone->stats_total));
1098  clib_memset (&new_n_clone->stats_last_clear, 0,
1099  sizeof (new_n_clone->stats_last_clear));
1100  }
1101  else
1102  {
1103  /* Copy stats if the old data is valid */
1104  clib_memcpy_fast (&new_n_clone->stats_total,
1105  &old_n_clone->stats_total,
1106  sizeof (new_n_clone->stats_total));
1107  clib_memcpy_fast (&new_n_clone->stats_last_clear,
1108  &old_n_clone->stats_last_clear,
1109  sizeof (new_n_clone->stats_last_clear));
1110 
1111  /* keep previous node state */
1112  new_n_clone->state = old_n_clone->state;
1113  }
1114  vec_add1 (nm_clone->nodes, new_n_clone);
1115  new_n_clone++;
1116  }
1117  /* Free the old node clones */
1118  clib_mem_free (old_nodes_clone[0]);
1119 
1120  vec_free (old_nodes_clone);
1121 
1122 
1123  /* re-clone internal nodes */
1124  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
1128 
1130  {
1131  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1132  rt->thread_index = vm_clone->thread_index;
1133  /* copy runtime_data, will be overwritten later for existing rt */
1134  if (n->runtime_data && n->runtime_data_bytes > 0)
1137  n->runtime_data_bytes));
1138  }
1139 
1140  for (j = 0; j < vec_len (old_rt); j++)
1141  {
1142  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1143  rt->state = old_rt[j].state;
1144  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1146  }
1147 
1148  vec_free (old_rt);
1149 
1150  /* re-clone input nodes */
1151  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
1152  nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
1155 
1157  {
1158  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1159  rt->thread_index = vm_clone->thread_index;
1160  /* copy runtime_data, will be overwritten later for existing rt */
1161  if (n->runtime_data && n->runtime_data_bytes > 0)
1164  n->runtime_data_bytes));
1165  }
1166 
1167  for (j = 0; j < vec_len (old_rt); j++)
1168  {
1169  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1170  rt->state = old_rt[j].state;
1171  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1173  }
1174 
1175  vec_free (old_rt);
1176 
1177  nm_clone->processes = vec_dup_aligned (nm->processes,
1179  nm_clone->node_by_error = nm->node_by_error;
1180 }
1181 
1182 void
1184 {
1185  /*
1186  * Make a note that we need to do a node runtime update
1187  * prior to releasing the barrier.
1188  */
1190 }
1191 
1192 u32
1193 unformat_sched_policy (unformat_input_t * input, va_list * args)
1194 {
1195  u32 *r = va_arg (*args, u32 *);
1196 
1197  if (0);
1198 #define _(v,f,s) else if (unformat (input, s)) *r = SCHED_POLICY_##f;
1200 #undef _
1201  else
1202  return 0;
1203  return 1;
1204 }
1205 
1206 static clib_error_t *
1208 {
1210  uword *p;
1212  u8 *name;
1213  uword *bitmap;
1214  u32 count;
1215 
1217 
1218  tm->n_thread_stacks = 1; /* account for main thread */
1219  tm->sched_policy = ~0;
1220  tm->sched_priority = ~0;
1221  tm->main_lcore = ~0;
1222 
1223  tr = tm->next;
1224 
1225  while (tr)
1226  {
1228  tr = tr->next;
1229  }
1230 
1231  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1232  {
1233  if (unformat (input, "use-pthreads"))
1234  tm->use_pthreads = 1;
1235  else if (unformat (input, "thread-prefix %v", &tm->thread_prefix))
1236  ;
1237  else if (unformat (input, "main-core %u", &tm->main_lcore))
1238  ;
1239  else if (unformat (input, "skip-cores %u", &tm->skip_cores))
1240  ;
1241  else if (unformat (input, "coremask-%s %U", &name,
1242  unformat_bitmap_mask, &bitmap) ||
1243  unformat (input, "corelist-%s %U", &name,
1244  unformat_bitmap_list, &bitmap))
1245  {
1247  if (p == 0)
1248  return clib_error_return (0, "no such thread type '%s'", name);
1249 
1250  tr = (vlib_thread_registration_t *) p[0];
1251 
1252  if (tr->use_pthreads)
1253  return clib_error_return (0,
1254  "corelist cannot be set for '%s' threads",
1255  name);
1256 
1257  tr->coremask = bitmap;
1259  }
1260  else
1261  if (unformat
1262  (input, "scheduler-policy %U", unformat_sched_policy,
1263  &tm->sched_policy))
1264  ;
1265  else if (unformat (input, "scheduler-priority %u", &tm->sched_priority))
1266  ;
1267  else if (unformat (input, "%s %u", &name, &count))
1268  {
1270  if (p == 0)
1271  return clib_error_return (0, "no such thread type 3 '%s'", name);
1272 
1273  tr = (vlib_thread_registration_t *) p[0];
1274  if (tr->fixed_count)
1275  return clib_error_return
1276  (0, "number of %s threads not configurable", tr->name);
1277  tr->count = count;
1278  }
1279  else
1280  break;
1281  }
1282 
1283  if (tm->sched_priority != ~0)
1284  {
1285  if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR)
1286  {
1287  u32 prio_max = sched_get_priority_max (tm->sched_policy);
1288  u32 prio_min = sched_get_priority_min (tm->sched_policy);
1289  if (tm->sched_priority > prio_max)
1290  tm->sched_priority = prio_max;
1291  if (tm->sched_priority < prio_min)
1292  tm->sched_priority = prio_min;
1293  }
1294  else
1295  {
1296  return clib_error_return
1297  (0,
1298  "scheduling priority (%d) is not allowed for `normal` scheduling policy",
1299  tm->sched_priority);
1300  }
1301  }
1302  tr = tm->next;
1303 
1304  if (!tm->thread_prefix)
1305  tm->thread_prefix = format (0, "vpp");
1306 
1307  while (tr)
1308  {
1309  tm->n_thread_stacks += tr->count;
1310  tm->n_pthreads += tr->count * tr->use_pthreads;
1311  tm->n_threads += tr->count * (tr->use_pthreads == 0);
1312  tr = tr->next;
1313  }
1314 
1315  return 0;
1316 }
1317 
1319 
1320 void vnet_main_fixup (vlib_fork_fixup_t which) __attribute__ ((weak));
1321 void
1323 {
1324 }
1325 
1326 void
1328 {
1329  vlib_main_t *vm = vlib_get_main ();
1330 
1331  if (vlib_mains == 0)
1332  return;
1333 
1334  ASSERT (vlib_get_thread_index () == 0);
1336 
1337  switch (which)
1338  {
1341  break;
1342 
1343  default:
1344  ASSERT (0);
1345  }
1347 }
1348 
1349  /*
1350  * Enforce minimum open time to minimize packet loss due to Rx overflow,
1351  * based on a test based heuristic that barrier should be open for at least
1352  * 3 time as long as it is closed (with an upper bound of 1ms because by that
1353  * point it is probably too late to make a difference)
1354  */
1355 
1356 #ifndef BARRIER_MINIMUM_OPEN_LIMIT
1357 #define BARRIER_MINIMUM_OPEN_LIMIT 0.001
1358 #endif
1359 
1360 #ifndef BARRIER_MINIMUM_OPEN_FACTOR
1361 #define BARRIER_MINIMUM_OPEN_FACTOR 3
1362 #endif
1363 
1364 void
1366 {
1367  f64 deadline;
1368  f64 now;
1369  f64 t_entry;
1370  f64 t_open;
1371  f64 t_closed;
1372  f64 max_vector_rate, this_vector_rate;
1373  u32 count;
1374  int i;
1375 
1376  if (vec_len (vlib_mains) < 2)
1377  return;
1378 
1379  ASSERT (vlib_get_thread_index () == 0);
1380 
1381  vlib_worker_threads[0].barrier_caller = func_name;
1382  count = vec_len (vlib_mains) - 1;
1383 
1384  /* Record entry relative to last close */
1385  now = vlib_time_now (vm);
1386  t_entry = now - vm->barrier_epoch;
1387 
1388  /* Tolerate recursive calls */
1389  if (++vlib_worker_threads[0].recursion_level > 1)
1390  {
1391  barrier_trace_sync_rec (t_entry);
1392  return;
1393  }
1394 
1395  /*
1396  * Need data to decide if we're working hard enough to honor
1397  * the barrier hold-down timer.
1398  */
1399  max_vector_rate = 0.0;
1400  for (i = 1; i < vec_len (vlib_mains); i++)
1401  {
1402  this_vector_rate =
1404  if (max_vector_rate < this_vector_rate)
1405  max_vector_rate = this_vector_rate;
1406  }
1407 
1408  vlib_worker_threads[0].barrier_sync_count++;
1409 
1410  /* Enforce minimum barrier open time to minimize packet loss */
1412 
1413  /*
1414  * If any worker thread seems busy, which we define
1415  * as a vector rate above 10, we enforce the barrier hold-down timer
1416  */
1417  if (max_vector_rate > 10.0)
1418  {
1419  while (1)
1420  {
1421  now = vlib_time_now (vm);
1422  /* Barrier hold-down timer expired? */
1423  if (now >= vm->barrier_no_close_before)
1424  break;
1425  if ((vm->barrier_no_close_before - now)
1426  > (2.0 * BARRIER_MINIMUM_OPEN_LIMIT))
1427  {
1428  clib_warning
1429  ("clock change: would have waited for %.4f seconds",
1430  (vm->barrier_no_close_before - now));
1431  break;
1432  }
1433  }
1434  }
1435  /* Record time of closure */
1436  t_open = now - vm->barrier_epoch;
1437  vm->barrier_epoch = now;
1438 
1439  deadline = now + BARRIER_SYNC_TIMEOUT;
1440 
1441  *vlib_worker_threads->wait_at_barrier = 1;
1442  while (*vlib_worker_threads->workers_at_barrier != count)
1443  {
1444  if ((now = vlib_time_now (vm)) > deadline)
1445  {
1446  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1447  os_panic ();
1448  }
1449  }
1450 
1451  t_closed = now - vm->barrier_epoch;
1452 
1453  barrier_trace_sync (t_entry, t_open, t_closed);
1454 
1455 }
1456 
1457 void vlib_stat_segment_lock (void) __attribute__ ((weak));
1458 void
1460 {
1461 }
1462 
1463 void vlib_stat_segment_unlock (void) __attribute__ ((weak));
1464 void
1466 {
1467 }
1468 
1469 void
1471 {
1472  f64 deadline;
1473  f64 now;
1474  f64 minimum_open;
1475  f64 t_entry;
1476  f64 t_closed_total;
1477  f64 t_update_main = 0.0;
1478  int refork_needed = 0;
1479 
1480  if (vec_len (vlib_mains) < 2)
1481  return;
1482 
1483  ASSERT (vlib_get_thread_index () == 0);
1484 
1485 
1486  now = vlib_time_now (vm);
1487  t_entry = now - vm->barrier_epoch;
1488 
1489  if (--vlib_worker_threads[0].recursion_level > 0)
1490  {
1491  barrier_trace_release_rec (t_entry);
1492  return;
1493  }
1494 
1495  /* Update (all) node runtimes before releasing the barrier, if needed */
1497  {
1498  /*
1499  * Lock stat segment here, so we's safe when
1500  * rebuilding the stat segment node clones from the
1501  * stat thread...
1502  */
1504 
1505  /* Do stats elements on main thread */
1508 
1509  /* Do per thread rebuilds in parallel */
1510  refork_needed = 1;
1511  clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
1512  (vec_len (vlib_mains) - 1));
1513  now = vlib_time_now (vm);
1514  t_update_main = now - vm->barrier_epoch;
1515  }
1516 
1517  deadline = now + BARRIER_SYNC_TIMEOUT;
1518 
1519  /*
1520  * Note when we let go of the barrier.
1521  * Workers can use this to derive a reasonably accurate
1522  * time offset. See vlib_time_now(...)
1523  */
1526 
1527  *vlib_worker_threads->wait_at_barrier = 0;
1528 
1529  while (*vlib_worker_threads->workers_at_barrier > 0)
1530  {
1531  if ((now = vlib_time_now (vm)) > deadline)
1532  {
1533  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1534  os_panic ();
1535  }
1536  }
1537 
1538  /* Wait for reforks before continuing */
1539  if (refork_needed)
1540  {
1541  now = vlib_time_now (vm);
1542 
1543  deadline = now + BARRIER_SYNC_TIMEOUT;
1544 
1545  while (*vlib_worker_threads->node_reforks_required > 0)
1546  {
1547  if ((now = vlib_time_now (vm)) > deadline)
1548  {
1549  fformat (stderr, "%s: worker thread refork deadlock\n",
1550  __FUNCTION__);
1551  os_panic ();
1552  }
1553  }
1555  }
1556 
1557  t_closed_total = now - vm->barrier_epoch;
1558 
1559  minimum_open = t_closed_total * BARRIER_MINIMUM_OPEN_FACTOR;
1560 
1561  if (minimum_open > BARRIER_MINIMUM_OPEN_LIMIT)
1562  {
1563  minimum_open = BARRIER_MINIMUM_OPEN_LIMIT;
1564  }
1565 
1566  vm->barrier_no_close_before = now + minimum_open;
1567 
1568  /* Record barrier epoch (used to enforce minimum open time) */
1569  vm->barrier_epoch = now;
1570 
1571  barrier_trace_release (t_entry, t_closed_total, t_update_main);
1572 
1573 }
1574 
1575 /*
1576  * Check the frame queue to see if any frames are available.
1577  * If so, pull the packets off the frames and put them to
1578  * the handoff node.
1579  */
1580 int
1582 {
1583  u32 thread_id = vm->thread_index;
1584  vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id];
1586  u32 *from, *to;
1587  vlib_frame_t *f;
1588  int msg_type;
1589  int processed = 0;
1590  u32 n_left_to_node;
1591  u32 vectors = 0;
1592 
1593  ASSERT (fq);
1594  ASSERT (vm == vlib_mains[thread_id]);
1595 
1596  if (PREDICT_FALSE (fqm->node_index == ~0))
1597  return 0;
1598  /*
1599  * Gather trace data for frame queues
1600  */
1601  if (PREDICT_FALSE (fq->trace))
1602  {
1603  frame_queue_trace_t *fqt;
1605  u32 elix;
1606 
1607  fqt = &fqm->frame_queue_traces[thread_id];
1608 
1609  fqt->nelts = fq->nelts;
1610  fqt->head = fq->head;
1611  fqt->head_hint = fq->head_hint;
1612  fqt->tail = fq->tail;
1613  fqt->threshold = fq->vector_threshold;
1614  fqt->n_in_use = fqt->tail - fqt->head;
1615  if (fqt->n_in_use >= fqt->nelts)
1616  {
1617  // if beyond max then use max
1618  fqt->n_in_use = fqt->nelts - 1;
1619  }
1620 
1621  /* Record the number of elements in use in the histogram */
1622  fqh = &fqm->frame_queue_histogram[thread_id];
1623  fqh->count[fqt->n_in_use]++;
1624 
1625  /* Record a snapshot of the elements in use */
1626  for (elix = 0; elix < fqt->nelts; elix++)
1627  {
1628  elt = fq->elts + ((fq->head + 1 + elix) & (fq->nelts - 1));
1629  if (1 || elt->valid)
1630  {
1631  fqt->n_vectors[elix] = elt->n_vectors;
1632  }
1633  }
1634  fqt->written = 1;
1635  }
1636 
1637  while (1)
1638  {
1639  if (fq->head == fq->tail)
1640  {
1641  fq->head_hint = fq->head;
1642  return processed;
1643  }
1644 
1645  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
1646 
1647  if (!elt->valid)
1648  {
1649  fq->head_hint = fq->head;
1650  return processed;
1651  }
1652 
1653  from = elt->buffer_index;
1654  msg_type = elt->msg_type;
1655 
1657  ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE);
1658 
1659  f = vlib_get_frame_to_node (vm, fqm->node_index);
1660 
1661  to = vlib_frame_vector_args (f);
1662 
1663  n_left_to_node = elt->n_vectors;
1664 
1665  while (n_left_to_node >= 4)
1666  {
1667  to[0] = from[0];
1668  to[1] = from[1];
1669  to[2] = from[2];
1670  to[3] = from[3];
1671  to += 4;
1672  from += 4;
1673  n_left_to_node -= 4;
1674  }
1675 
1676  while (n_left_to_node > 0)
1677  {
1678  to[0] = from[0];
1679  to++;
1680  from++;
1681  n_left_to_node--;
1682  }
1683 
1684  vectors += elt->n_vectors;
1685  f->n_vectors = elt->n_vectors;
1686  vlib_put_frame_to_node (vm, fqm->node_index, f);
1687 
1688  elt->valid = 0;
1689  elt->n_vectors = 0;
1690  elt->msg_type = 0xfefefefe;
1692  fq->head++;
1693  processed++;
1694 
1695  /*
1696  * Limit the number of packets pushed into the graph
1697  */
1698  if (vectors >= fq->vector_threshold)
1699  {
1700  fq->head_hint = fq->head;
1701  return processed;
1702  }
1703  }
1704  ASSERT (0);
1705  return processed;
1706 }
1707 
1708 void
1710 {
1713  vlib_main_t *vm = vlib_get_main ();
1714  clib_error_t *e;
1715 
1717 
1719  clib_time_init (&vm->clib_time);
1721 
1722  /* Wait until the dpdk init sequence is complete */
1723  while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
1725 
1727  (vm, vm->worker_init_function_registrations, 1 /* call_once */ );
1728  if (e)
1729  clib_error_report (e);
1730 
1731  vlib_worker_loop (vm);
1732 }
1733 
1734 /* *INDENT-OFF* */
1735 VLIB_REGISTER_THREAD (worker_thread_reg, static) = {
1736  .name = "workers",
1737  .short_name = "wk",
1738  .function = vlib_worker_thread_fn,
1739 };
1740 /* *INDENT-ON* */
1741 
1742 u32
1743 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts)
1744 {
1747  vlib_frame_queue_t *fq;
1748  int i;
1749 
1750  if (frame_queue_nelts == 0)
1751  frame_queue_nelts = FRAME_QUEUE_NELTS;
1752 
1753  ASSERT (frame_queue_nelts >= 8);
1754 
1755  vec_add2 (tm->frame_queue_mains, fqm, 1);
1756 
1757  fqm->node_index = node_index;
1758  fqm->frame_queue_nelts = frame_queue_nelts;
1759  fqm->queue_hi_thresh = frame_queue_nelts - 2;
1760 
1762  vec_validate (fqm->per_thread_data, tm->n_vlib_mains - 1);
1763  _vec_len (fqm->vlib_frame_queues) = 0;
1764  for (i = 0; i < tm->n_vlib_mains; i++)
1765  {
1767  fq = vlib_frame_queue_alloc (frame_queue_nelts);
1768  vec_add1 (fqm->vlib_frame_queues, fq);
1769 
1770  ptd = vec_elt_at_index (fqm->per_thread_data, i);
1772  tm->n_vlib_mains - 1);
1774  tm->n_vlib_mains - 1,
1775  (vlib_frame_queue_t *) (~0));
1776  }
1777 
1778  return (fqm - tm->frame_queue_mains);
1779 }
1780 
1781 int
1783 {
1785 
1786  if (tm->extern_thread_mgmt)
1787  return -1;
1788 
1790  tm->extern_thread_mgmt = 1;
1791  return 0;
1792 }
1793 
1794 void
1796  args)
1797 {
1798  ASSERT (vlib_get_thread_index () == 0);
1800  args->type_opaque, args->data);
1801 }
1802 
1804 
1805 void
1806 vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size)
1807 {
1809  {
1810  void (*fp) (void *, u8 *, u32) = rpc_call_main_thread_cb_fn;
1811  (*fp) (callback, args, arg_size);
1812  }
1813  else
1814  clib_warning ("BUG: rpc_call_main_thread_cb_fn NULL!");
1815 }
1816 
1817 clib_error_t *
1819 {
1820  return 0;
1821 }
1822 
1824 
1825 
1826 static clib_error_t *
1828  unformat_input_t * input, vlib_cli_command_t * cmd)
1829 {
1830  int i;
1831  f64 now;
1832 
1833  now = vlib_time_now (vm);
1834 
1835  vlib_cli_output (vm, "Time now %.9f", now);
1836 
1837  if (vec_len (vlib_mains) == 1)
1838  return 0;
1839 
1840  vlib_cli_output (vm, "Time last barrier release %.9f",
1842 
1843  for (i = 1; i < vec_len (vlib_mains); i++)
1844  {
1845  if (vlib_mains[i] == 0)
1846  continue;
1847  vlib_cli_output (vm, "Thread %d offset %.9f error %.9f", i,
1848  vlib_mains[i]->time_offset,
1851  }
1852  return 0;
1853 }
1854 
1855 /* *INDENT-OFF* */
1856 VLIB_CLI_COMMAND (f_command, static) =
1857 {
1858  .path = "show clock",
1859  .short_help = "show clock",
1860  .function = show_clock_command_fn,
1861 };
1862 /* *INDENT-ON* */
1863 
1864 /*
1865  * fd.io coding-style-patch-verification: ON
1866  *
1867  * Local Variables:
1868  * eval: (c-set-style "gnu")
1869  * End:
1870  */
_vlib_init_function_list_elt_t * worker_init_function_registrations
Definition: main.h:183
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
static void barrier_trace_release_rec(f64 t_entry)
Definition: threads.c:120
static void barrier_trace_sync(f64 t_entry, f64 t_open, f64 t_closed)
Definition: threads.c:69
u32 vl(void *p)
Definition: threads.c:31
static f64 vlib_last_vectors_per_main_loop_as_f64(vlib_main_t *vm)
Definition: main.h:314
uword * pending_rpc_requests
Definition: main.h:220
vlib_main_t vlib_global_main
Definition: main.c:1850
#define clib_min(x, y)
Definition: clib.h:295
clib_spinlock_t pending_rpc_lock
Definition: main.h:222
vlib_process_t ** processes
Definition: node.h:762
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:484
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:529
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:545
int vlib_frame_queue_enqueue(vlib_main_t *vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t *frame, vlib_frame_queue_msg_type_t type)
#define clib_atomic_add_fetch(a, b)
Definition: atomics.h:30
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x)
Definition: init.h:166
word elog_track_register(elog_main_t *em, elog_track_t *t)
register an event track
Definition: elog.c:198
unsigned long u64
Definition: types.h:89
void * mheap_alloc(void *memory, uword size)
Definition: mheap.c:963
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:118
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
#define NULL
Definition: clib.h:58
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 index
Definition: node.h:304
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:232
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:576
clib_error_t * threads_init(vlib_main_t *vm)
Definition: threads.c:1818
void os_panic(void)
Definition: unix-misc.c:174
u32 vlib_frame_queue_main_init(u32 node_index, u32 frame_queue_nelts)
Definition: threads.c:1743
u32 thread_index
Definition: main.h:179
void * thread_function_arg
Definition: threads.h:98
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:525
static int sort_registrations_by_no_clone(void *a0, void *a1)
Definition: threads.c:197
static u64 clib_cpu_time_now(void)
Definition: time.h:75
frame_queue_trace_t * frame_queue_traces
Definition: threads.h:165
void vlib_process_signal_event_mt_helper(vlib_process_signal_event_mt_args_t *args)
Definition: threads.c:1795
elog_track_t elog_track
Definition: threads.h:100
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:564
int i
void vnet_main_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1322
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
#define hash_set_mem(h, key, value)
Definition: hash.h:275
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
clib_time_t clib_time
Definition: main.h:65
void vlib_worker_thread_fn(void *arg)
Definition: threads.c:1709
u32 unformat_sched_policy(unformat_input_t *input, va_list *args)
Definition: threads.c:1193
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
struct vlib_thread_registration_ * next
Definition: threads.h:31
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:82
void * runtime_data
Definition: node.h:310
volatile u32 valid
Definition: threads.h:76
void vlib_worker_thread_barrier_sync_int(vlib_main_t *vm, const char *func_name)
Definition: threads.c:1365
vlib_main_t ** vlib_mains
Definition: buffer.c:310
static void vlib_get_thread_core_socket(vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:607
unsigned char u8
Definition: types.h:56
#define clib_bitmap_dup(v)
Duplicate a bitmap.
Definition: bitmap.h:87
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
u8 state
Definition: node.h:332
u16 thread_index
thread this node runs on
Definition: node.h:543
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
#define clib_memcpy(d, s, n)
Definition: string.h:180
u64 * counters_last_clear
Definition: error.h:51
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:390
vlib_thread_registration_t * next
Definition: threads.h:283
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
Definition: vec.h:535
#define VLIB_NODE_RUNTIME_DATA_SIZE
Definition: node.h:556
vlib_node_stats_t stats_last_clear
Definition: node.h:298
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:490
void vlib_worker_thread_node_runtime_update(void)
Definition: threads.c:1183
u64 count[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:825
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:398
void * heap_aligned_base
Definition: main.h:117
vlib_frame_queue_msg_type_t
Definition: threads.h:68
vlib_node_t ** nodes
Definition: node.h:721
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
Definition: threads.h:151
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:181
#define clib_error_return(e, args...)
Definition: error.h:99
#define VLIB_FRAME_ALIGN
Definition: node.h:402
uword * lock
SMP lock, non-zero means locking required.
Definition: elog.h:175
uword * cpu_core_bitmap
Definition: threads.h:320
#define BARRIER_MINIMUM_OPEN_FACTOR
Definition: threads.c:1361
clib_error_t * vlib_call_init_exit_functions(vlib_main_t *vm, _vlib_init_function_list_elt_t *head, int call_once)
Definition: init.c:43
vlib_frame_queue_elt_t * elts
Definition: threads.h:144
pthread_t thread[MAX_CONNS]
Definition: main.c:142
f64 time_last_barrier_release
Definition: main.h:68
unsigned int u32
Definition: types.h:88
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:731
#define VLIB_FRAME_SIZE
Definition: node.h:401
void vlib_set_thread_name(char *name)
Definition: threads.c:182
void vl_msg_api_handler_no_free(void *)
Definition: threads.c:425
#define hash_create_string(elts, value_bytes)
Definition: hash.h:690
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:1023
vlib_fork_fixup_t
Definition: threads.h:227
#define BARRIER_SYNC_TIMEOUT
Definition: threads.h:199
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
void * rpc_call_main_thread_cb_fn
Definition: threads.c:1803
VLIB_REGISTER_THREAD(worker_thread_reg, static)
int extern_thread_mgmt
Definition: threads.h:339
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:36
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
void * thread_mheap
Definition: threads.h:95
u32 next_frame_index
Definition: node.h:481
vlib_node_stats_t stats_total
Definition: node.h:294
volatile u64 head
Definition: threads.h:131
u16 state
Input node state.
Definition: node.h:533
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:934
static uword clib_bitmap_first_set(uword *ai)
Return the lowest numbered set bit in a bitmap.
Definition: bitmap.h:385
u8 * vlib_thread_stack_init(uword thread_index)
Definition: main.c:618
static void vlib_next_frame_init(vlib_next_frame_t *nf)
Definition: node.h:465
vlib_error_main_t error_main
Definition: main.h:143
vlib_thread_callbacks_t cb
Definition: threads.h:338
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:438
int vlib_thread_cb_register(struct vlib_main_t *vm, vlib_thread_callbacks_t *cb)
Definition: threads.c:1782
struct _unformat_input_t unformat_input_t
const char * barrier_context
Definition: threads.h:107
char * name
Track name vector.
Definition: elog.h:116
#define clib_error_return_unix(e, args...)
Definition: error.h:102
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:190
static void * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:64
u32 * node_by_error
Definition: node.h:789
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:210
#define ELOG_DATA(em, f)
Definition: elog.h:484
#define PREDICT_FALSE(x)
Definition: clib.h:111
static clib_error_t * vlib_launch_thread_int(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:627
void vlib_worker_thread_node_refork(void)
Definition: threads.c:1027
clib_error_t *(* vlib_thread_set_lcore_cb)(u32 thread, u16 cpu)
Definition: threads.h:277
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: main.h:123
u32 node_index
Node index.
Definition: node.h:518
u32 elog_global_id_for_msg_name(const char *msg_name)
Definition: threads.c:47
uword * init_functions_called
Definition: main.h:176
void clib_time_init(clib_time_t *c)
Definition: time.c:178
uword * frame_size_hash
Definition: node.h:777
u8 name[64]
Definition: memclnt.api:152
vlib_thread_main_t vlib_thread_main
Definition: threads.c:37
clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
Definition: sysfs.c:50
word fformat(FILE *f, char *fmt,...)
Definition: format.c:453
void(* thread_function)(void *)
Definition: threads.h:97
static clib_error_t * cpu_config(vlib_main_t *vm, unformat_input_t *input)
Definition: threads.c:1207
u8 len
Definition: ip_types.api:49
#define pool_get_aligned(P, E, A)
Allocate an object E from a pool P with alignment A.
Definition: pool.h:230
i32 n_vectors[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:820
u64 * counters
Definition: error.h:48
u32 owner_node_index
Definition: node.h:378
vlib_frame_queue_t * vlib_frame_queue_alloc(int nelts)
Definition: threads.c:391
volatile u64 tail
Definition: threads.h:123
#define clib_mem_alloc_no_fail(size)
Definition: mem.h:176
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:216
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
svmdb_client_t * c
u16 n_vectors
Definition: node.h:420
vlib_frame_queue_t ** vlib_frame_queues
Definition: threads.h:161
DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked)
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:212
vlib_main_t * vm
Definition: buffer.c:301
u32 node_runtime_index
Definition: node.h:475
vlib_pending_frame_t * pending_frames
Definition: node.h:747
vlib_thread_function_t * function
Definition: threads.h:36
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1581
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
void * heap_base
Definition: main.h:114
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:261
#define clib_warning(format, args...)
Definition: error.h:59
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
elog_main_t elog_main
Definition: main.h:157
frame_queue_nelt_counter_t * frame_queue_histogram
Definition: threads.h:166
#define VLIB_FRAME_PENDING
Definition: node.h:452
static uword clib_bitmap_get(uword *ai, uword i)
Gets the ith bit value from a bitmap.
Definition: bitmap.h:197
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:442
void vlib_worker_thread_init(vlib_worker_thread_t *w)
Definition: threads.c:552
uword os_get_nthreads(void)
Definition: threads.c:170
static void * clib_mem_get_heap(void)
Definition: mem.h:255
volatile u32 * wait_at_barrier
Definition: threads.h:90
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:155
#define FRAME_QUEUE_NELTS
Definition: threads.c:28
vlib_frame_queue_per_thread_data_t * per_thread_data
Definition: threads.h:162
void vlib_stat_segment_unlock(void)
Definition: threads.c:1465
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
Definition: threads.h:152
#define never_inline
Definition: clib.h:95
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define ASSERT(truth)
static clib_error_t * show_clock_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: threads.c:1827
static void barrier_trace_sync_rec(f64 t_entry)
Definition: threads.c:96
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:326
u16 flags
Definition: node.h:411
static void clib_mem_free(void *p)
Definition: mem.h:205
#define clib_error_report(e)
Definition: error.h:113
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
size_t count
Definition: vapi.c:47
int need_vlib_worker_thread_node_runtime_update
Definition: main.h:208
uword * thread_registrations_by_name
Definition: threads.h:288
#define BARRIER_MINIMUM_OPEN_LIMIT
Definition: threads.c:1357
clib_error_t *(* vlib_launch_thread_cb)(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.h:275
volatile u32 * node_reforks_required
Definition: threads.h:108
const char * barrier_caller
Definition: threads.h:106
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
#define clib_atomic_fetch_add(a, b)
Definition: atomics.h:23
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:577
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:462
static void barrier_trace_release(f64 t_entry, f64 t_closed_total, f64 t_update_main)
Definition: threads.c:142
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1845
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks)
Definition: main.c:538
#define vec_dup_aligned(V, A)
Return copy of vector (no header, alignment specified).
Definition: vec.h:384
u32 elog_string(elog_main_t *em, char *fmt,...)
add a string to the event-log string table
Definition: elog.c:562
f64 barrier_no_close_before
Definition: main.h:217
static clib_error_t * start_workers(vlib_main_t *vm)
Definition: threads.c:654
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void vlib_rpc_call_main_thread(void *callback, u8 *args, u32 arg_size)
Definition: threads.c:1806
DECLARE_CJ_GLOBAL_LOG
Definition: threads.c:26
vlib_node_main_t node_main
Definition: main.h:129
u64 uword
Definition: types.h:112
vlib_next_frame_t * next_frames
Definition: node.h:744
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
Definition: vec.h:984
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
volatile u64 head_hint
Definition: threads.h:140
#define VLIB_THREAD_STACK_SIZE
Definition: threads.h:66
f64 barrier_epoch
Definition: main.h:214
vlib_frame_size_t * frame_sizes
Definition: node.h:780
#define hash_get_mem(h, key)
Definition: hash.h:269
static void worker_thread_node_runtime_update_internal(void)
Definition: threads.c:975
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:140
volatile u32 * workers_at_barrier
Definition: threads.h:91
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
static uword * clib_sysfs_list_to_bitmap(char *filename)
Definition: threads.c:207
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1470
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static f64 vlib_time_now_ticks(vlib_main_t *vm, u64 n)
Definition: main.h:238
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
void vlib_stat_segment_lock(void)
Definition: threads.c:1459
#define vec_foreach(var, vec)
Vector iterator.
void * vlib_worker_thread_bootstrap_fn(void *arg)
Definition: threads.c:589
#define SYS_gettid
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:115
u32 node_runtime_index
Definition: node.h:432
uword * cpu_socket_bitmap
Definition: threads.h:323
#define foreach_sched_policy
Definition: threads.h:258
vlib_thread_registration_t ** registrations
Definition: threads.h:286
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:488
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u8 ** vlib_thread_stacks
Definition: main.c:601
pthread_t thread_id
Definition: threads.h:114
vlib_thread_registration_t * registration
Definition: threads.h:102
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:762
volatile u32 worker_thread_release
Definition: threads.h:329
void vlib_worker_thread_fork_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1327
clib_random_buffer_t random_buffer
Definition: main.h:173
u8 runtime_data_bytes
Definition: node.h:335
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:455
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:237