FD.io VPP  v21.06-1-gbb7418cf9
Vector Packet Processing
threads.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #define _GNU_SOURCE
16 
17 #include <signal.h>
18 #include <math.h>
19 #include <vppinfra/format.h>
20 #include <vppinfra/time_range.h>
21 #include <vppinfra/interrupt.h>
22 #include <vppinfra/linux/sysfs.h>
23 #include <vlib/vlib.h>
24 
25 #include <vlib/threads.h>
26 
27 #include <vlib/stat_weak_inlines.h>
28 
29 u32
30 vl (void *p)
31 {
32  return vec_len (p);
33 }
34 
37 
38 /*
39  * Barrier tracing can be enabled on a normal build to collect information
40  * on barrier use, including timings and call stacks. Deliberately not
41  * keyed off CLIB_DEBUG, because that can add significant overhead which
42  * imapacts observed timings.
43  */
44 
45 static inline void
46 barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed)
47 {
48  if (!vlib_worker_threads->barrier_elog_enabled)
49  return;
50 
51  ELOG_TYPE_DECLARE (e) = {
52  .format = "bar-trace-%s-#%d",
53  .format_args = "T4i4",
54  };
55 
56  struct
57  {
58  u32 caller, count, t_entry, t_open, t_closed;
59  } *ed = 0;
60 
62  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
63  ed->caller = elog_string (&vlib_global_main.elog_main,
64  (char *) vlib_worker_threads[0].barrier_caller);
65  ed->t_entry = (int) (1000000.0 * t_entry);
66  ed->t_open = (int) (1000000.0 * t_open);
67  ed->t_closed = (int) (1000000.0 * t_closed);
68 }
69 
70 static inline void
72 {
73  if (!vlib_worker_threads->barrier_elog_enabled)
74  return;
75 
76  ELOG_TYPE_DECLARE (e) = {
77  .format = "bar-syncrec-%s-#%d",
78  .format_args = "T4i4",
79  };
80 
81  struct
82  {
83  u32 caller, depth;
84  } *ed = 0;
85 
87  ed->depth = (int) vlib_worker_threads[0].recursion_level - 1;
88  ed->caller = elog_string (&vlib_global_main.elog_main,
89  (char *) vlib_worker_threads[0].barrier_caller);
90 }
91 
92 static inline void
94 {
95  if (!vlib_worker_threads->barrier_elog_enabled)
96  return;
97 
98  ELOG_TYPE_DECLARE (e) = {
99  .format = "bar-relrrec-#%d",
100  .format_args = "i4",
101  };
102 
103  struct
104  {
105  u32 depth;
106  } *ed = 0;
107 
109  ed->depth = (int) vlib_worker_threads[0].recursion_level;
110 }
111 
112 static inline void
113 barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main)
114 {
115  if (!vlib_worker_threads->barrier_elog_enabled)
116  return;
117 
118  ELOG_TYPE_DECLARE (e) = {
119  .format = "bar-rel-#%d-e%d-u%d-t%d",
120  .format_args = "i4i4i4i4",
121  };
122 
123  struct
124  {
125  u32 count, t_entry, t_update_main, t_closed_total;
126  } *ed = 0;
127 
129  ed->t_entry = (int) (1000000.0 * t_entry);
130  ed->t_update_main = (int) (1000000.0 * t_update_main);
131  ed->t_closed_total = (int) (1000000.0 * t_closed_total);
132  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
133 
134  /* Reset context for next trace */
135  vlib_worker_threads[0].barrier_context = NULL;
136 }
137 
138 uword
140 {
141  return vec_len (vlib_thread_stacks);
142 }
143 
144 void
146 {
147  int pthread_setname_np (pthread_t __target_thread, const char *__name);
148  int rv;
149  pthread_t thread = pthread_self ();
150 
151  if (thread)
152  {
153  rv = pthread_setname_np (thread, name);
154  if (rv)
155  clib_warning ("pthread_setname_np returned %d", rv);
156  }
157 }
158 
159 static int
160 sort_registrations_by_no_clone (void *a0, void *a1)
161 {
162  vlib_thread_registration_t **tr0 = a0;
163  vlib_thread_registration_t **tr1 = a1;
164 
165  return ((i32) ((*tr0)->no_data_structure_clone)
166  - ((i32) ((*tr1)->no_data_structure_clone)));
167 }
168 
169 
170 /* Called early in the init sequence */
171 
172 clib_error_t *
174 {
178  u32 n_vlib_mains = 1;
179  u32 first_index = 1;
180  u32 i;
181  uword *avail_cpu;
182 
183  /* get bitmaps of active cpu cores and sockets */
184  tm->cpu_core_bitmap =
185  clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online");
186  tm->cpu_socket_bitmap =
187  clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online");
188 
189  avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
190 
191  /* skip cores */
192  for (i = 0; i < tm->skip_cores; i++)
193  {
194  uword c = clib_bitmap_first_set (avail_cpu);
195  if (c == ~0)
196  return clib_error_return (0, "no available cpus to skip");
197 
198  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
199  }
200 
201  /* grab cpu for main thread */
202  if (tm->main_lcore == ~0)
203  {
204  /* if main-lcore is not set, we try to use lcore 1 */
205  if (clib_bitmap_get (avail_cpu, 1))
206  tm->main_lcore = 1;
207  else
208  tm->main_lcore = clib_bitmap_first_set (avail_cpu);
209  if (tm->main_lcore == (u8) ~ 0)
210  return clib_error_return (0, "no available cpus to be used for the"
211  " main thread");
212  }
213  else
214  {
215  if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0)
216  return clib_error_return (0, "cpu %u is not available to be used"
217  " for the main thread", tm->main_lcore);
218  }
219  avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
220 
221  /* assume that there is socket 0 only if there is no data from sysfs */
222  if (!tm->cpu_socket_bitmap)
223  tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1);
224 
225  /* pin main thread to main_lcore */
227  {
229  }
230  else
231  {
232  cpu_set_t cpuset;
233  CPU_ZERO (&cpuset);
234  CPU_SET (tm->main_lcore, &cpuset);
235  pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
236  }
237 
238  /* Set up thread 0 */
239  vec_validate_aligned (vlib_worker_threads, 0, CLIB_CACHE_LINE_BYTES);
240  _vec_len (vlib_worker_threads) = 1;
244  w->cpu_id = tm->main_lcore;
245  w->lwp = syscall (SYS_gettid);
246  w->thread_id = pthread_self ();
247  tm->n_vlib_mains = 1;
248 
250 
251  if (tm->sched_policy != ~0)
252  {
253  struct sched_param sched_param;
254  if (!sched_getparam (w->lwp, &sched_param))
255  {
256  if (tm->sched_priority != ~0)
257  sched_param.sched_priority = tm->sched_priority;
258  sched_setscheduler (w->lwp, tm->sched_policy, &sched_param);
259  }
260  }
261 
262  /* assign threads to cores and set n_vlib_mains */
263  tr = tm->next;
264 
265  while (tr)
266  {
267  vec_add1 (tm->registrations, tr);
268  tr = tr->next;
269  }
270 
272 
273  for (i = 0; i < vec_len (tm->registrations); i++)
274  {
275  int j;
276  tr = tm->registrations[i];
277  tr->first_index = first_index;
278  first_index += tr->count;
279  n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0;
280 
281  /* construct coremask */
282  if (tr->use_pthreads || !tr->count)
283  continue;
284 
285  if (tr->coremask)
286  {
287  uword c;
288  /* *INDENT-OFF* */
289  clib_bitmap_foreach (c, tr->coremask) {
290  if (clib_bitmap_get(avail_cpu, c) == 0)
291  return clib_error_return (0, "cpu %u is not available to be used"
292  " for the '%s' thread",c, tr->name);
293 
294  avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
295  }
296  /* *INDENT-ON* */
297  }
298  else
299  {
300  for (j = 0; j < tr->count; j++)
301  {
302  /* Do not use CPU 0 by default - leave it to the host and IRQs */
303  uword avail_c0 = clib_bitmap_get (avail_cpu, 0);
304  avail_cpu = clib_bitmap_set (avail_cpu, 0, 0);
305 
306  uword c = clib_bitmap_first_set (avail_cpu);
307  /* Use CPU 0 as a last resort */
308  if (c == ~0 && avail_c0)
309  {
310  c = 0;
311  avail_c0 = 0;
312  }
313 
314  if (c == ~0)
315  return clib_error_return (0,
316  "no available cpus to be used for"
317  " the '%s' thread", tr->name);
318 
319  avail_cpu = clib_bitmap_set (avail_cpu, 0, avail_c0);
320  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
321  tr->coremask = clib_bitmap_set (tr->coremask, c, 1);
322  }
323  }
324  }
325 
326  clib_bitmap_free (avail_cpu);
327 
328  tm->n_vlib_mains = n_vlib_mains;
329 
330  /*
331  * Allocate the remaining worker threads, and thread stack vector slots
332  * from now on, calls to os_get_nthreads() will return the correct
333  * answer.
334  */
335  vec_validate_aligned (vlib_worker_threads, first_index - 1,
337  vec_validate (vlib_thread_stacks, vec_len (vlib_worker_threads) - 1);
338  return 0;
339 }
340 
343 {
344  vlib_frame_queue_t *fq;
345 
346  fq = clib_mem_alloc_aligned (sizeof (*fq), CLIB_CACHE_LINE_BYTES);
347  clib_memset (fq, 0, sizeof (*fq));
348  fq->nelts = nelts;
349  fq->vector_threshold = 128; // packets
351 
352  if (1)
353  {
354  if (((uword) & fq->tail) & (CLIB_CACHE_LINE_BYTES - 1))
355  fformat (stderr, "WARNING: fq->tail unaligned\n");
356  if (((uword) & fq->head) & (CLIB_CACHE_LINE_BYTES - 1))
357  fformat (stderr, "WARNING: fq->head unaligned\n");
358  if (((uword) fq->elts) & (CLIB_CACHE_LINE_BYTES - 1))
359  fformat (stderr, "WARNING: fq->elts unaligned\n");
360 
361  if (sizeof (fq->elts[0]) % CLIB_CACHE_LINE_BYTES)
362  fformat (stderr, "WARNING: fq->elts[0] size %d\n",
363  sizeof (fq->elts[0]));
364  if (nelts & (nelts - 1))
365  {
366  fformat (stderr, "FATAL: nelts MUST be a power of 2\n");
367  abort ();
368  }
369  }
370 
371  return (fq);
372 }
373 
374 void vl_msg_api_handler_no_free (void *) __attribute__ ((weak));
375 void
377 {
378 }
379 
380 /* To be called by vlib worker threads upon startup */
381 void
383 {
385 
386  /*
387  * Note: disabling signals in worker threads as follows
388  * prevents the api post-mortem dump scheme from working
389  * {
390  * sigset_t s;
391  * sigfillset (&s);
392  * pthread_sigmask (SIG_SETMASK, &s, 0);
393  * }
394  */
395 
397 
398  if (vec_len (tm->thread_prefix) && w->registration->short_name)
399  {
400  w->name = format (0, "%v_%s_%d%c", tm->thread_prefix,
401  w->registration->short_name, w->instance_id, '\0');
402  vlib_set_thread_name ((char *) w->name);
403  }
404 
405  if (!w->registration->use_pthreads)
406  {
407 
408  /* Initial barrier sync, for both worker and i/o threads */
409  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
410 
411  while (*vlib_worker_threads->wait_at_barrier)
412  ;
413 
414  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
415  }
416 }
417 
418 void *
420 {
421  void *rv;
422  vlib_worker_thread_t *w = arg;
423 
424  w->lwp = syscall (SYS_gettid);
425  w->thread_id = pthread_self ();
426 
427  __os_thread_index = w - vlib_worker_threads;
428 
430  0);
431  rv = (void *) clib_calljmp
432  ((uword (*)(uword)) w->thread_function,
434  /* NOTREACHED, we hope */
435  return rv;
436 }
437 
438 void
440 {
441  const char *sys_cpu_path = "/sys/devices/system/cpu/cpu";
442  const char *sys_node_path = "/sys/devices/system/node/node";
443  clib_bitmap_t *nbmp = 0, *cbmp = 0;
444  u32 node;
445  u8 *p = 0;
446  int core_id = -1, numa_id = -1;
447 
448  p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, cpu_id, 0);
449  clib_sysfs_read ((char *) p, "%d", &core_id);
450  vec_reset_length (p);
451 
452  /* *INDENT-OFF* */
453  clib_sysfs_read ("/sys/devices/system/node/online", "%U",
454  unformat_bitmap_list, &nbmp);
455  clib_bitmap_foreach (node, nbmp) {
456  p = format (p, "%s%u/cpulist%c", sys_node_path, node, 0);
457  clib_sysfs_read ((char *) p, "%U", unformat_bitmap_list, &cbmp);
458  if (clib_bitmap_get (cbmp, cpu_id))
459  numa_id = node;
460  vec_reset_length (cbmp);
461  vec_reset_length (p);
462  }
463  /* *INDENT-ON* */
464  vec_free (nbmp);
465  vec_free (cbmp);
466  vec_free (p);
467 
468  w->core_id = core_id;
469  w->numa_id = numa_id;
470 }
471 
472 static clib_error_t *
474 {
477  void *(*fp_arg) (void *) = fp;
478  void *numa_heap;
479 
480  w->cpu_id = cpu_id;
481  vlib_get_thread_core_numa (w, cpu_id);
482 
483  /* Set up NUMA-bound heap if indicated */
484  if (mm->per_numa_mheaps[w->numa_id] == 0)
485  {
486  /* If the user requested a NUMA heap, create it... */
487  if (tm->numa_heap_size)
488  {
489  clib_mem_set_numa_affinity (w->numa_id, 1 /* force */ );
490  numa_heap = clib_mem_create_heap (0 /* DIY */ , tm->numa_heap_size,
491  1 /* is_locked */ ,
492  "numa %u heap", w->numa_id);
494  mm->per_numa_mheaps[w->numa_id] = numa_heap;
495  }
496  else
497  {
498  /* Or, use the main heap */
499  mm->per_numa_mheaps[w->numa_id] = w->thread_mheap;
500  }
501  }
502 
504  return tm->cb.vlib_launch_thread_cb (fp, (void *) w, cpu_id);
505  else
506  {
507  pthread_t worker;
508  cpu_set_t cpuset;
509  CPU_ZERO (&cpuset);
510  CPU_SET (cpu_id, &cpuset);
511 
512  if (pthread_create (&worker, NULL /* attr */ , fp_arg, (void *) w))
513  return clib_error_return_unix (0, "pthread_create");
514 
515  if (pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset))
516  return clib_error_return_unix (0, "pthread_setaffinity_np");
517 
518  return 0;
519  }
520 }
521 
522 static clib_error_t *
524 {
526  int i, j;
528  vlib_main_t *vm_clone;
529  void *oldheap;
533  u32 n_vlib_mains = tm->n_vlib_mains;
534  u32 worker_thread_index;
536  vlib_stats_register_mem_heap (main_heap);
537 
538  vec_reset_length (vlib_worker_threads);
539 
540  /* Set up the main thread */
541  vec_add2_aligned (vlib_worker_threads, w, 1, CLIB_CACHE_LINE_BYTES);
542  w->elog_track.name = "main thread";
544 
545  if (vec_len (tm->thread_prefix))
546  {
547  w->name = format (0, "%v_main%c", tm->thread_prefix, '\0');
548  vlib_set_thread_name ((char *) w->name);
549  }
550 
551  vgm->elog_main.lock =
553  vgm->elog_main.lock[0] = 0;
554 
557 
558  vec_validate_aligned (vgm->vlib_mains, n_vlib_mains - 1,
560  _vec_len (vgm->vlib_mains) = 0;
562 
563  if (n_vlib_mains > 1)
564  {
565  vlib_worker_threads->wait_at_barrier =
567  vlib_worker_threads->workers_at_barrier =
569 
570  vlib_worker_threads->node_reforks_required =
572 
573  /* We'll need the rpc vector lock... */
575 
576  /* Ask for an initial barrier sync */
577  *vlib_worker_threads->workers_at_barrier = 0;
578  *vlib_worker_threads->wait_at_barrier = 1;
579 
580  /* Without update or refork */
581  *vlib_worker_threads->node_reforks_required = 0;
583 
584  /* init timing */
585  vm->barrier_epoch = 0;
586  vm->barrier_no_close_before = 0;
587 
588  worker_thread_index = 1;
590 
591  for (i = 0; i < vec_len (tm->registrations); i++)
592  {
593  vlib_node_main_t *nm, *nm_clone;
594  int k;
595 
596  tr = tm->registrations[i];
597 
598  if (tr->count == 0)
599  continue;
600 
601  for (k = 0; k < tr->count; k++)
602  {
603  vlib_node_t *n;
604 
605  vec_add2 (vlib_worker_threads, w, 1);
606  /* Currently unused, may not really work */
607  if (tr->mheap_size)
609  /* unlocked */ 0,
610  "%s%d heap",
611  tr->name, k);
612  else
613  w->thread_mheap = main_heap;
614 
615  w->thread_stack =
616  vlib_thread_stack_init (w - vlib_worker_threads);
617  w->thread_function = tr->function;
618  w->thread_function_arg = w;
619  w->instance_id = k;
620  w->registration = tr;
621 
622  w->elog_track.name =
623  (char *) format (0, "%s %d", tr->name, k + 1);
624  vec_add1 (w->elog_track.name, 0);
626 
627  if (tr->no_data_structure_clone)
628  continue;
629 
630  /* Fork vlib_global_main et al. Look for bugs here */
631  oldheap = clib_mem_set_heap (w->thread_mheap);
632 
633  vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone),
635  clib_memcpy (vm_clone, vlib_get_first_main (),
636  sizeof (*vm_clone));
637 
638  vm_clone->thread_index = worker_thread_index;
639  vm_clone->heap_base = w->thread_mheap;
640  vm_clone->heap_aligned_base = (void *)
641  (((uword) w->thread_mheap) & ~(VLIB_FRAME_ALIGN - 1));
642  vm_clone->pending_rpc_requests = 0;
643  vec_validate (vm_clone->pending_rpc_requests, 0);
644  _vec_len (vm_clone->pending_rpc_requests) = 0;
645  clib_memset (&vm_clone->random_buffer, 0,
646  sizeof (vm_clone->random_buffer));
652 
653  nm = &vlib_get_first_main ()->node_main;
654  nm_clone = &vm_clone->node_main;
655  /* fork next frames array, preserving node runtime indices */
656  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
658  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
659  {
660  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
661  u32 save_node_runtime_index;
662  u32 save_flags;
663 
664  save_node_runtime_index = nf->node_runtime_index;
665  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
667  nf->node_runtime_index = save_node_runtime_index;
668  nf->flags = save_flags;
669  }
670 
671  /* fork the frame dispatch queue */
672  nm_clone->pending_frames = 0;
673  vec_validate (nm_clone->pending_frames, 10);
674  _vec_len (nm_clone->pending_frames) = 0;
675 
676  /* fork nodes */
677  nm_clone->nodes = 0;
678 
679  /* Allocate all nodes in single block for speed */
680  n = clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*n));
681 
682  for (j = 0; j < vec_len (nm->nodes); j++)
683  {
684  clib_memcpy (n, nm->nodes[j], sizeof (*n));
685  /* none of the copied nodes have enqueue rights given out */
687  clib_memset (&n->stats_total, 0, sizeof (n->stats_total));
689  sizeof (n->stats_last_clear));
690  vec_add1 (nm_clone->nodes, n);
691  n++;
692  }
696  vec_foreach (rt,
698  {
699  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
700  rt->thread_index = vm_clone->thread_index;
701  /* copy initial runtime_data from node */
702  if (n->runtime_data && n->runtime_data_bytes > 0)
705  n->runtime_data_bytes));
706  }
707 
712  &nm_clone->interrupts,
713  vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
715  {
716  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
717  rt->thread_index = vm_clone->thread_index;
718  /* copy initial runtime_data from node */
719  if (n->runtime_data && n->runtime_data_bytes > 0)
722  n->runtime_data_bytes));
723  }
724 
728  vec_foreach (rt,
730  {
731  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
732  rt->thread_index = vm_clone->thread_index;
733  /* copy initial runtime_data from node */
734  if (n->runtime_data && n->runtime_data_bytes > 0)
737  n->runtime_data_bytes));
738  }
739 
740  nm_clone->processes = vec_dup_aligned (nm->processes,
742 
743  /* Create per-thread frame freelist */
744  nm_clone->frame_sizes = vec_new (vlib_frame_size_t, 1);
745 #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
746  nm_clone->frame_size_hash = hash_create (0, sizeof (uword));
747 #endif
748  nm_clone->node_by_error = nm->node_by_error;
749 
750  /* Packet trace buffers are guaranteed to be empty, nothing to do here */
751 
752  clib_mem_set_heap (oldheap);
753  vec_add1_aligned (vgm->vlib_mains, vm_clone,
755 
756  /* Switch to the stats segment ... */
757  void *oldheap = vlib_stats_push_heap (0);
758  vm_clone->error_main.counters =
759  vec_dup_aligned (vlib_get_first_main ()->error_main.counters,
762  worker_thread_index, oldheap, 1);
763 
765  vlib_get_first_main ()->error_main.counters_last_clear,
767 
768  worker_thread_index++;
769  }
770  }
771  }
772  else
773  {
774  /* only have non-data-structure copy threads to create... */
775  for (i = 0; i < vec_len (tm->registrations); i++)
776  {
777  tr = tm->registrations[i];
778 
779  for (j = 0; j < tr->count; j++)
780  {
781  vec_add2 (vlib_worker_threads, w, 1);
782  if (tr->mheap_size)
783  {
785  /* locked */ 0,
786  "%s%d heap",
787  tr->name, j);
788  }
789  else
790  w->thread_mheap = main_heap;
791  w->thread_stack =
792  vlib_thread_stack_init (w - vlib_worker_threads);
793  w->thread_function = tr->function;
794  w->thread_function_arg = w;
795  w->instance_id = j;
796  w->elog_track.name =
797  (char *) format (0, "%s %d", tr->name, j + 1);
798  w->registration = tr;
799  vec_add1 (w->elog_track.name, 0);
801  }
802  }
803  }
804 
805  worker_thread_index = 1;
806 
807  for (i = 0; i < vec_len (tm->registrations); i++)
808  {
809  clib_error_t *err;
810  int j;
811 
812  tr = tm->registrations[i];
813 
814  if (tr->use_pthreads || tm->use_pthreads)
815  {
816  for (j = 0; j < tr->count; j++)
817  {
818  w = vlib_worker_threads + worker_thread_index++;
820  w, 0);
821  if (err)
822  clib_error_report (err);
823  }
824  }
825  else
826  {
827  uword c;
828  /* *INDENT-OFF* */
829  clib_bitmap_foreach (c, tr->coremask) {
830  w = vlib_worker_threads + worker_thread_index++;
832  w, c);
833  if (err)
834  clib_error_report (err);
835  }
836  /* *INDENT-ON* */
837  }
838  }
841  return 0;
842 }
843 
845 
846 
847 static inline void
849 {
850  int i, j;
851  vlib_main_t *vm;
852  vlib_node_main_t *nm, *nm_clone;
853  vlib_main_t *vm_clone;
855 
856  ASSERT (vlib_get_thread_index () == 0);
857 
858  vm = vlib_get_first_main ();
859  nm = &vm->node_main;
860 
861  ASSERT (*vlib_worker_threads->wait_at_barrier == 1);
862 
863  /*
864  * Scrape all runtime stats, so we don't lose node runtime(s) with
865  * pending counts, or throw away worker / io thread counts.
866  */
867  for (j = 0; j < vec_len (nm->nodes); j++)
868  {
869  vlib_node_t *n;
870  n = nm->nodes[j];
871  vlib_node_sync_stats (vm, n);
872  }
873 
874  for (i = 1; i < vlib_get_n_threads (); i++)
875  {
876  vlib_node_t *n;
877 
878  vm_clone = vlib_get_main_by_index (i);
879  nm_clone = &vm_clone->node_main;
880 
881  for (j = 0; j < vec_len (nm_clone->nodes); j++)
882  {
883  n = nm_clone->nodes[j];
884 
885  rt = vlib_node_get_runtime (vm_clone, n->index);
886  vlib_node_runtime_sync_stats (vm_clone, rt, 0, 0, 0);
887  }
888  }
889 
890  /* Per-worker clone rebuilds are now done on each thread */
891 }
892 
893 
894 void
896 {
897  vlib_main_t *vm, *vm_clone;
898  vlib_node_main_t *nm, *nm_clone;
899  vlib_node_t **old_nodes_clone;
900  vlib_node_runtime_t *rt, *old_rt;
901 
902  vlib_node_t *new_n_clone;
903 
904  int j;
905 
906  vm = vlib_get_first_main ();
907  nm = &vm->node_main;
908  vm_clone = vlib_get_main ();
909  nm_clone = &vm_clone->node_main;
910 
911  /* Re-clone error heap */
912  u64 *old_counters = vm_clone->error_main.counters;
913  u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear;
914 
915  clib_memcpy_fast (&vm_clone->error_main, &vm->error_main,
916  sizeof (vm->error_main));
917  j = vec_len (vm->error_main.counters) - 1;
918 
919  /* Switch to the stats segment ... */
920  void *oldheap = vlib_stats_push_heap (0);
921  vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES);
922  vm_clone->error_main.counters = old_counters;
924  oldheap, 0);
925 
926  vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
927  vm_clone->error_main.counters_last_clear = old_counters_all_clear;
928 
929  nm_clone = &vm_clone->node_main;
930  vec_free (nm_clone->next_frames);
931  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
933 
934  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
935  {
936  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
937  u32 save_node_runtime_index;
938  u32 save_flags;
939 
940  save_node_runtime_index = nf->node_runtime_index;
941  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
943  nf->node_runtime_index = save_node_runtime_index;
944  nf->flags = save_flags;
945  }
946 
947  old_nodes_clone = nm_clone->nodes;
948  nm_clone->nodes = 0;
949 
950  /* re-fork nodes */
951 
952  /* Allocate all nodes in single block for speed */
953  new_n_clone =
954  clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*new_n_clone));
955  for (j = 0; j < vec_len (nm->nodes); j++)
956  {
957  vlib_node_t *new_n = nm->nodes[j];
958 
959  clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n));
960  /* none of the copied nodes have enqueue rights given out */
962 
963  if (j >= vec_len (old_nodes_clone))
964  {
965  /* new node, set to zero */
966  clib_memset (&new_n_clone->stats_total, 0,
967  sizeof (new_n_clone->stats_total));
968  clib_memset (&new_n_clone->stats_last_clear, 0,
969  sizeof (new_n_clone->stats_last_clear));
970  }
971  else
972  {
973  vlib_node_t *old_n_clone = old_nodes_clone[j];
974  /* Copy stats if the old data is valid */
975  clib_memcpy_fast (&new_n_clone->stats_total,
976  &old_n_clone->stats_total,
977  sizeof (new_n_clone->stats_total));
978  clib_memcpy_fast (&new_n_clone->stats_last_clear,
979  &old_n_clone->stats_last_clear,
980  sizeof (new_n_clone->stats_last_clear));
981 
982  /* keep previous node state */
983  new_n_clone->state = old_n_clone->state;
984  new_n_clone->flags = old_n_clone->flags;
985  }
986  vec_add1 (nm_clone->nodes, new_n_clone);
987  new_n_clone++;
988  }
989  /* Free the old node clones */
990  clib_mem_free (old_nodes_clone[0]);
991 
992  vec_free (old_nodes_clone);
993 
994 
995  /* re-clone internal nodes */
996  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
1000 
1002  {
1003  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1004  rt->thread_index = vm_clone->thread_index;
1005  /* copy runtime_data, will be overwritten later for existing rt */
1006  if (n->runtime_data && n->runtime_data_bytes > 0)
1009  n->runtime_data_bytes));
1010  }
1011 
1012  for (j = 0; j < vec_len (old_rt); j++)
1013  {
1014  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1015  rt->state = old_rt[j].state;
1016  rt->flags = old_rt[j].flags;
1017  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1019  }
1020 
1021  vec_free (old_rt);
1022 
1023  /* re-clone input nodes */
1024  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
1025  nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
1029  &nm_clone->interrupts,
1030  vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
1031 
1033  {
1034  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1035  rt->thread_index = vm_clone->thread_index;
1036  /* copy runtime_data, will be overwritten later for existing rt */
1037  if (n->runtime_data && n->runtime_data_bytes > 0)
1040  n->runtime_data_bytes));
1041  }
1042 
1043  for (j = 0; j < vec_len (old_rt); j++)
1044  {
1045  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1046  rt->state = old_rt[j].state;
1047  rt->flags = old_rt[j].flags;
1048  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1050  }
1051 
1052  vec_free (old_rt);
1053 
1054  /* re-clone pre-input nodes */
1055  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT];
1059 
1061  {
1062  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1063  rt->thread_index = vm_clone->thread_index;
1064  /* copy runtime_data, will be overwritten later for existing rt */
1065  if (n->runtime_data && n->runtime_data_bytes > 0)
1068  n->runtime_data_bytes));
1069  }
1070 
1071  for (j = 0; j < vec_len (old_rt); j++)
1072  {
1073  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1074  rt->state = old_rt[j].state;
1075  rt->flags = old_rt[j].flags;
1076  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1078  }
1079 
1080  vec_free (old_rt);
1081 
1082  nm_clone->processes = vec_dup_aligned (nm->processes,
1084  nm_clone->node_by_error = nm->node_by_error;
1085 }
1086 
1087 void
1089 {
1090  /*
1091  * Make a note that we need to do a node runtime update
1092  * prior to releasing the barrier.
1093  */
1095 }
1096 
1097 u32
1098 unformat_sched_policy (unformat_input_t * input, va_list * args)
1099 {
1100  u32 *r = va_arg (*args, u32 *);
1101 
1102  if (0);
1103 #define _(v,f,s) else if (unformat (input, s)) *r = SCHED_POLICY_##f;
1105 #undef _
1106  else
1107  return 0;
1108  return 1;
1109 }
1110 
1111 static clib_error_t *
1113 {
1115  uword *p;
1117  u8 *name;
1118  uword *bitmap;
1119  u32 count;
1120 
1122 
1123  tm->n_thread_stacks = 1; /* account for main thread */
1124  tm->sched_policy = ~0;
1125  tm->sched_priority = ~0;
1126  tm->main_lcore = ~0;
1127 
1128  tr = tm->next;
1129 
1130  while (tr)
1131  {
1133  tr = tr->next;
1134  }
1135 
1136  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1137  {
1138  if (unformat (input, "use-pthreads"))
1139  tm->use_pthreads = 1;
1140  else if (unformat (input, "thread-prefix %v", &tm->thread_prefix))
1141  ;
1142  else if (unformat (input, "main-core %u", &tm->main_lcore))
1143  ;
1144  else if (unformat (input, "skip-cores %u", &tm->skip_cores))
1145  ;
1146  else if (unformat (input, "numa-heap-size %U",
1148  ;
1149  else if (unformat (input, "coremask-%s %U", &name,
1150  unformat_bitmap_mask, &bitmap) ||
1151  unformat (input, "corelist-%s %U", &name,
1152  unformat_bitmap_list, &bitmap))
1153  {
1155  if (p == 0)
1156  return clib_error_return (0, "no such thread type '%s'", name);
1157 
1158  tr = (vlib_thread_registration_t *) p[0];
1159 
1160  if (tr->use_pthreads)
1161  return clib_error_return (0,
1162  "corelist cannot be set for '%s' threads",
1163  name);
1164  if (tr->count)
1165  return clib_error_return
1166  (0, "core placement of '%s' threads is already configured",
1167  name);
1168 
1169  tr->coremask = bitmap;
1171  }
1172  else
1173  if (unformat
1174  (input, "scheduler-policy %U", unformat_sched_policy,
1175  &tm->sched_policy))
1176  ;
1177  else if (unformat (input, "scheduler-priority %u", &tm->sched_priority))
1178  ;
1179  else if (unformat (input, "%s %u", &name, &count))
1180  {
1182  if (p == 0)
1183  return clib_error_return (0, "no such thread type 3 '%s'", name);
1184 
1185  tr = (vlib_thread_registration_t *) p[0];
1186 
1187  if (tr->fixed_count)
1188  return clib_error_return
1189  (0, "number of '%s' threads not configurable", name);
1190  if (tr->count)
1191  return clib_error_return
1192  (0, "number of '%s' threads is already configured", name);
1193 
1194  tr->count = count;
1195  }
1196  else
1197  break;
1198  }
1199 
1200  if (tm->sched_priority != ~0)
1201  {
1202  if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR)
1203  {
1204  u32 prio_max = sched_get_priority_max (tm->sched_policy);
1205  u32 prio_min = sched_get_priority_min (tm->sched_policy);
1206  if (tm->sched_priority > prio_max)
1207  tm->sched_priority = prio_max;
1208  if (tm->sched_priority < prio_min)
1209  tm->sched_priority = prio_min;
1210  }
1211  else
1212  {
1213  return clib_error_return
1214  (0,
1215  "scheduling priority (%d) is not allowed for `normal` scheduling policy",
1216  tm->sched_priority);
1217  }
1218  }
1219  tr = tm->next;
1220 
1221  if (!tm->thread_prefix)
1222  tm->thread_prefix = format (0, "vpp");
1223 
1224  while (tr)
1225  {
1226  tm->n_thread_stacks += tr->count;
1227  tm->n_pthreads += tr->count * tr->use_pthreads;
1228  tm->n_threads += tr->count * (tr->use_pthreads == 0);
1229  tr = tr->next;
1230  }
1231 
1232  return 0;
1233 }
1234 
1236 
1237  /*
1238  * Enforce minimum open time to minimize packet loss due to Rx overflow,
1239  * based on a test based heuristic that barrier should be open for at least
1240  * 3 time as long as it is closed (with an upper bound of 1ms because by that
1241  * point it is probably too late to make a difference)
1242  */
1243 
1244 #ifndef BARRIER_MINIMUM_OPEN_LIMIT
1245 #define BARRIER_MINIMUM_OPEN_LIMIT 0.001
1246 #endif
1247 
1248 #ifndef BARRIER_MINIMUM_OPEN_FACTOR
1249 #define BARRIER_MINIMUM_OPEN_FACTOR 3
1250 #endif
1251 
1252 void
1254 {
1255  f64 deadline;
1256  f64 now = vlib_time_now (vm);
1257  u32 count = vlib_get_n_threads () - 1;
1258 
1259  /* No worker threads? */
1260  if (count == 0)
1261  return;
1262 
1263  deadline = now + BARRIER_SYNC_TIMEOUT;
1264  *vlib_worker_threads->wait_at_barrier = 1;
1265  while (*vlib_worker_threads->workers_at_barrier != count)
1266  {
1267  if ((now = vlib_time_now (vm)) > deadline)
1268  {
1269  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1270  os_panic ();
1271  }
1272  CLIB_PAUSE ();
1273  }
1274  *vlib_worker_threads->wait_at_barrier = 0;
1275 }
1276 
1277 /**
1278  * Return true if the wroker thread barrier is held
1279  */
1280 u8
1282 {
1283  if (vlib_get_n_threads () < 2)
1284  return (1);
1285 
1286  return (*vlib_worker_threads->wait_at_barrier == 1);
1287 }
1288 
1289 void
1291 {
1292  f64 deadline;
1293  f64 now;
1294  f64 t_entry;
1295  f64 t_open;
1296  f64 t_closed;
1297  f64 max_vector_rate;
1298  u32 count;
1299  int i;
1300 
1301  if (vlib_get_n_threads () < 2)
1302  return;
1303 
1304  ASSERT (vlib_get_thread_index () == 0);
1305 
1306  vlib_worker_threads[0].barrier_caller = func_name;
1307  count = vlib_get_n_threads () - 1;
1308 
1309  /* Record entry relative to last close */
1310  now = vlib_time_now (vm);
1311  t_entry = now - vm->barrier_epoch;
1312 
1313  /* Tolerate recursive calls */
1314  if (++vlib_worker_threads[0].recursion_level > 1)
1315  {
1316  barrier_trace_sync_rec (t_entry);
1317  return;
1318  }
1319 
1320  if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
1322  vm->clib_time.last_cpu_time, 0 /* enter */ );
1323 
1324  /*
1325  * Need data to decide if we're working hard enough to honor
1326  * the barrier hold-down timer.
1327  */
1328  max_vector_rate = 0.0;
1329  for (i = 1; i < vlib_get_n_threads (); i++)
1330  {
1332  max_vector_rate = clib_max (max_vector_rate,
1334  }
1335 
1336  vlib_worker_threads[0].barrier_sync_count++;
1337 
1338  /* Enforce minimum barrier open time to minimize packet loss */
1340 
1341  /*
1342  * If any worker thread seems busy, which we define
1343  * as a vector rate above 10, we enforce the barrier hold-down timer
1344  */
1345  if (max_vector_rate > 10.0)
1346  {
1347  while (1)
1348  {
1349  now = vlib_time_now (vm);
1350  /* Barrier hold-down timer expired? */
1351  if (now >= vm->barrier_no_close_before)
1352  break;
1353  if ((vm->barrier_no_close_before - now)
1354  > (2.0 * BARRIER_MINIMUM_OPEN_LIMIT))
1355  {
1356  clib_warning
1357  ("clock change: would have waited for %.4f seconds",
1358  (vm->barrier_no_close_before - now));
1359  break;
1360  }
1361  }
1362  }
1363  /* Record time of closure */
1364  t_open = now - vm->barrier_epoch;
1365  vm->barrier_epoch = now;
1366 
1367  deadline = now + BARRIER_SYNC_TIMEOUT;
1368 
1369  *vlib_worker_threads->wait_at_barrier = 1;
1370  while (*vlib_worker_threads->workers_at_barrier != count)
1371  {
1372  if ((now = vlib_time_now (vm)) > deadline)
1373  {
1374  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1375  os_panic ();
1376  }
1377  }
1378 
1379  t_closed = now - vm->barrier_epoch;
1380 
1381  barrier_trace_sync (t_entry, t_open, t_closed);
1382 
1383 }
1384 
1385 void
1387 {
1389  f64 deadline;
1390  f64 now;
1391  f64 minimum_open;
1392  f64 t_entry;
1393  f64 t_closed_total;
1394  f64 t_update_main = 0.0;
1395  int refork_needed = 0;
1396 
1397  if (vlib_get_n_threads () < 2)
1398  return;
1399 
1400  ASSERT (vlib_get_thread_index () == 0);
1401 
1402 
1403  now = vlib_time_now (vm);
1404  t_entry = now - vm->barrier_epoch;
1405 
1406  if (--vlib_worker_threads[0].recursion_level > 0)
1407  {
1408  barrier_trace_release_rec (t_entry);
1409  return;
1410  }
1411 
1412  /* Update (all) node runtimes before releasing the barrier, if needed */
1414  {
1415  /*
1416  * Lock stat segment here, so we's safe when
1417  * rebuilding the stat segment node clones from the
1418  * stat thread...
1419  */
1421 
1422  /* Do stats elements on main thread */
1425 
1426  /* Do per thread rebuilds in parallel */
1427  refork_needed = 1;
1428  clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
1429  (vlib_get_n_threads () - 1));
1430  now = vlib_time_now (vm);
1431  t_update_main = now - vm->barrier_epoch;
1432  }
1433 
1434  deadline = now + BARRIER_SYNC_TIMEOUT;
1435 
1436  /*
1437  * Note when we let go of the barrier.
1438  * Workers can use this to derive a reasonably accurate
1439  * time offset. See vlib_time_now(...)
1440  */
1443 
1444  *vlib_worker_threads->wait_at_barrier = 0;
1445 
1446  while (*vlib_worker_threads->workers_at_barrier > 0)
1447  {
1448  if ((now = vlib_time_now (vm)) > deadline)
1449  {
1450  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1451  os_panic ();
1452  }
1453  }
1454 
1455  /* Wait for reforks before continuing */
1456  if (refork_needed)
1457  {
1458  now = vlib_time_now (vm);
1459 
1460  deadline = now + BARRIER_SYNC_TIMEOUT;
1461 
1462  while (*vlib_worker_threads->node_reforks_required > 0)
1463  {
1464  if ((now = vlib_time_now (vm)) > deadline)
1465  {
1466  fformat (stderr, "%s: worker thread refork deadlock\n",
1467  __FUNCTION__);
1468  os_panic ();
1469  }
1470  }
1472  }
1473 
1474  t_closed_total = now - vm->barrier_epoch;
1475 
1476  minimum_open = t_closed_total * BARRIER_MINIMUM_OPEN_FACTOR;
1477 
1478  if (minimum_open > BARRIER_MINIMUM_OPEN_LIMIT)
1479  {
1480  minimum_open = BARRIER_MINIMUM_OPEN_LIMIT;
1481  }
1482 
1483  vm->barrier_no_close_before = now + minimum_open;
1484 
1485  /* Record barrier epoch (used to enforce minimum open time) */
1486  vm->barrier_epoch = now;
1487 
1488  barrier_trace_release (t_entry, t_closed_total, t_update_main);
1489 
1490  if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
1492  vm->clib_time.last_cpu_time, 1 /* leave */ );
1493 }
1494 
1495 /**
1496  * Wait until each of the workers has been once around the track
1497  */
1498 void
1500 {
1502  ASSERT (vlib_get_thread_index () == 0);
1503 
1504  if (vlib_get_n_threads () < 2)
1505  return;
1506 
1508  return;
1509 
1510  u32 *counts = 0;
1511  u32 ii;
1512 
1513  vec_validate (counts, vlib_get_n_threads () - 1);
1514 
1515  /* record the current loop counts */
1516  vec_foreach_index (ii, vgm->vlib_mains)
1517  counts[ii] = vgm->vlib_mains[ii]->main_loop_count;
1518 
1519  /* spin until each changes, apart from the main thread, or we'd be
1520  * a while */
1521  for (ii = 1; ii < vec_len (counts); ii++)
1522  {
1523  while (counts[ii] == vgm->vlib_mains[ii]->main_loop_count)
1524  CLIB_PAUSE ();
1525  }
1526 
1527  vec_free (counts);
1528  return;
1529 }
1530 
1531 void
1533 {
1538  clib_error_t *e;
1539 
1541 
1543 
1545  clib_time_init (&vm->clib_time);
1547 
1549 
1551  vm, &vgm->worker_init_function_registrations, 1 /* call_once */,
1552  0 /* is_global */);
1553  if (e)
1554  clib_error_report (e);
1555 
1556  /* Wait until the dpdk init sequence is complete */
1557  while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
1559 
1560  vlib_worker_loop (vm);
1561 }
1562 
1563 /* *INDENT-OFF* */
1564 VLIB_REGISTER_THREAD (worker_thread_reg, static) = {
1565  .name = "workers",
1566  .short_name = "wk",
1567  .function = vlib_worker_thread_fn,
1568 };
1569 /* *INDENT-ON* */
1570 
1571 u32
1573 {
1576  vlib_frame_queue_t *fq;
1577  int i;
1578  u32 num_threads;
1579 
1580  if (frame_queue_nelts == 0)
1581  frame_queue_nelts = FRAME_QUEUE_MAX_NELTS;
1582 
1583  num_threads = 1 /* main thread */ + tm->n_threads;
1584  ASSERT (frame_queue_nelts >= 8 + num_threads);
1585 
1586  vec_add2 (tm->frame_queue_mains, fqm, 1);
1587 
1588  fqm->node_index = node_index;
1589  fqm->frame_queue_nelts = frame_queue_nelts;
1590  fqm->queue_hi_thresh = frame_queue_nelts - num_threads;
1591 
1593  vec_validate (fqm->per_thread_data, tm->n_vlib_mains - 1);
1594  _vec_len (fqm->vlib_frame_queues) = 0;
1595  for (i = 0; i < tm->n_vlib_mains; i++)
1596  {
1598  fq = vlib_frame_queue_alloc (frame_queue_nelts);
1599  vec_add1 (fqm->vlib_frame_queues, fq);
1600 
1601  ptd = vec_elt_at_index (fqm->per_thread_data, i);
1603  tm->n_vlib_mains - 1);
1605  tm->n_vlib_mains - 1,
1606  (vlib_frame_queue_t *) (~0));
1607  }
1608 
1609  return (fqm - tm->frame_queue_mains);
1610 }
1611 
1612 int
1614 {
1616 
1617  if (tm->extern_thread_mgmt)
1618  return -1;
1619 
1621  tm->extern_thread_mgmt = 1;
1622  return 0;
1623 }
1624 
1625 void
1627  args)
1628 {
1629  ASSERT (vlib_get_thread_index () == 0);
1631  args->type_opaque, args->data);
1632 }
1633 
1635 
1636 void
1637 vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size)
1638 {
1640  {
1641  void (*fp) (void *, u8 *, u32) = rpc_call_main_thread_cb_fn;
1642  (*fp) (callback, args, arg_size);
1643  }
1644  else
1645  clib_warning ("BUG: rpc_call_main_thread_cb_fn NULL!");
1646 }
1647 
1648 clib_error_t *
1650 {
1651  return 0;
1652 }
1653 
1655 
1656 
1657 static clib_error_t *
1659  unformat_input_t * input, vlib_cli_command_t * cmd)
1660 {
1661  int i;
1662  int verbose = 0;
1663  clib_timebase_t _tb, *tb = &_tb;
1664 
1665  (void) unformat (input, "verbose %=", &verbose, 1);
1666 
1668  &vm->clib_time);
1669 
1670  vlib_cli_output (vm, "%U, %U GMT", format_clib_time, &vm->clib_time,
1671  verbose, format_clib_timebase_time,
1672  clib_timebase_now (tb));
1673 
1674  if (vlib_get_n_threads () == 1)
1675  return 0;
1676 
1677  vlib_cli_output (vm, "Time last barrier release %.9f",
1679 
1680  for (i = 1; i < vlib_get_n_threads (); i++)
1681  {
1683  if (ovm == 0)
1684  continue;
1685 
1686  vlib_cli_output (vm, "%d: %U", i, format_clib_time, &ovm->clib_time,
1687  verbose);
1688 
1689  vlib_cli_output (
1690  vm, "Thread %d offset %.9f error %.9f", i, ovm->time_offset,
1692  }
1693  return 0;
1694 }
1695 
1696 /* *INDENT-OFF* */
1697 VLIB_CLI_COMMAND (f_command, static) =
1698 {
1699  .path = "show clock",
1700  .short_help = "show clock",
1701  .function = show_clock_command_fn,
1702 };
1703 /* *INDENT-ON* */
1704 
1707 {
1708  return vlib_get_thread_main ();
1709 }
1710 
1711 /*
1712  * fd.io coding-style-patch-verification: ON
1713  *
1714  * Local Variables:
1715  * eval: (c-set-style "gnu")
1716  * End:
1717  */
f64 time_offset
Definition: main.h:108
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:524
volatile u32 main_loop_count
Definition: main.h:118
static void barrier_trace_release_rec(f64 t_entry)
Definition: threads.c:93
static void barrier_trace_sync(f64 t_entry, f64 t_open, f64 t_closed)
Definition: threads.c:46
#define CLIB_PAUSE()
Definition: lock.h:23
u32 vl(void *p)
Definition: threads.c:30
#define clib_callback_data_init(set_, lock_)
Initialize a callback set.
Definition: callback_data.h:41
uword * pending_rpc_requests
Definition: main.h:260
#define vec_foreach_index(var, v)
Iterate over vector indices.
vnet_interface_output_runtime_t * rt
void vlib_worker_wait_one_loop(void)
Wait until each of the workers has been once around the track.
Definition: threads.c:1499
#define clib_min(x, y)
Definition: clib.h:342
void vlib_stats_pop_heap2(u64 *error_vector, u32 thread_index, void *oldheap, int lock)
Definition: stat_segment.c:363
clib_spinlock_t pending_rpc_lock
Definition: main.h:262
void vlib_stats_register_mem_heap(clib_mem_heap_t *heap)
vlib_process_t ** processes
Definition: node.h:710
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:506
vnet_hw_if_output_node_runtime_t * r
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x)
Definition: init.h:175
__clib_export uword unformat_bitmap_mask(unformat_input_t *input, va_list *va)
unformat an any sized hexadecimal bitmask into a bitmap
Definition: bitmap.c:20
clib_mem_heap_t * clib_mem_create_heap(void *base, uword size, int is_locked, char *fmt,...)
Definition: mem_dlmalloc.c:536
unsigned long u64
Definition: types.h:89
clib_spinlock_t worker_thread_main_loop_callback_lock
Definition: main.h:228
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:140
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 index
Definition: node.h:270
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:325
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:657
clib_error_t * threads_init(vlib_main_t *vm)
Definition: threads.c:1649
static clib_mem_heap_t * clib_mem_set_heap(clib_mem_heap_t *heap)
Definition: mem.h:365
format_function_t format_clib_time
Definition: time.h:76
#define clib_bitmap_foreach(i, ai)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
uword numa_heap_size
Definition: threads.h:326
u32 vlib_frame_queue_main_init(u32 node_index, u32 frame_queue_nelts)
Definition: threads.c:1572
u32 thread_index
Definition: main.h:213
u16 flags
Definition: node.h:279
void * thread_function_arg
Definition: threads.h:97
__clib_export uword * clib_sysfs_list_to_bitmap(char *filename)
Definition: sysfs.c:267
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:607
static int sort_registrations_by_no_clone(void *a0, void *a1)
Definition: threads.c:160
void vlib_process_signal_event_mt_helper(vlib_process_signal_event_mt_args_t *args)
Definition: threads.c:1626
elog_track_t elog_track
Definition: threads.h:99
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:645
void * vlib_stats_push_heap(void *old)
Definition: stat_segment.c:50
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
#define hash_set_mem(h, key, value)
Definition: hash.h:275
string name[64]
Definition: fib.api:25
void vlib_worker_thread_fn(void *arg)
Definition: threads.c:1532
clib_time_t clib_time
Definition: main.h:106
u32 unformat_sched_policy(unformat_input_t *input, va_list *args)
Definition: threads.c:1098
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:535
struct vlib_thread_registration_ * next
Definition: threads.h:30
void * runtime_data
Definition: node.h:276
void vlib_worker_thread_barrier_sync_int(vlib_main_t *vm, const char *func_name)
Definition: threads.c:1290
static u32 vlib_last_vectors_per_main_loop(vlib_main_t *vm)
Definition: main.h:427
unsigned char u8
Definition: types.h:56
#define clib_bitmap_dup(v)
Duplicate a bitmap.
Definition: bitmap.h:87
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
u8 state
Definition: node.h:300
u16 thread_index
thread this node runs on
Definition: node.h:504
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:194
unsigned int u32
Definition: types.h:88
#define clib_memcpy(d, s, n)
Definition: string.h:197
u64 * counters_last_clear
Definition: error.h:67
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:378
vlib_thread_registration_t * next
Definition: threads.h:267
__clib_export word fformat(FILE *f, char *fmt,...)
Definition: format.c:466
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
Definition: vec.h:616
#define VLIB_NODE_RUNTIME_DATA_SIZE
Definition: node.h:517
vlib_node_stats_t stats_last_clear
Definition: node.h:264
if(node->flags &VLIB_NODE_FLAG_TRACE) vnet_interface_output_trace(vm
void vlib_worker_thread_node_runtime_update(void)
Definition: threads.c:1088
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:172
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:366
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:365
void * heap_aligned_base
Definition: main.h:159
static vlib_global_main_t * vlib_get_global_main(void)
Definition: global_funcs.h:50
static f64 clib_timebase_now(clib_timebase_t *tb)
Definition: time_range.h:88
vlib_node_t ** nodes
Definition: node.h:669
description fragment has unexpected format
Definition: map.api:433
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
Definition: threads.h:144
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
#define VLIB_FRAME_ALIGN
Definition: node.h:370
__clib_export int clib_mem_set_numa_affinity(u8 numa_node, int force)
Definition: mem.c:638
uword * lock
SMP lock, non-zero means locking required.
Definition: elog.h:175
uword * cpu_core_bitmap
Definition: threads.h:304
#define BARRIER_MINIMUM_OPEN_FACTOR
Definition: threads.c:1249
vlib_frame_queue_elt_t * elts
Definition: threads.h:137
pthread_t thread[MAX_CONNS]
Definition: main.c:142
f64 time_last_barrier_release
Definition: main.h:109
static_always_inline void vlib_process_start_switch_stack(vlib_main_t *vm, vlib_process_t *p)
Definition: node_funcs.h:57
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:679
int __clib_unused rv
Definition: application.c:491
void vlib_set_thread_name(char *name)
Definition: threads.c:145
void vl_msg_api_handler_no_free(void *)
Definition: threads.c:376
u32 cpu_id
Definition: vpe.api:222
format_function_t format_clib_timebase_time
Definition: time_range.h:70
#define hash_create_string(elts, value_bytes)
Definition: hash.h:690
#define BARRIER_SYNC_TIMEOUT
Definition: threads.h:189
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:65
void * rpc_call_main_thread_cb_fn
Definition: threads.c:1634
VLIB_REGISTER_THREAD(worker_thread_reg, static)
int extern_thread_mgmt
Definition: threads.h:323
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:35
void * thread_mheap
Definition: threads.h:94
vlib_node_stats_t stats_total
Definition: node.h:260
volatile u64 head
Definition: threads.h:127
u16 state
Input node state.
Definition: node.h:494
static elog_main_t * vlib_get_elog_main()
Definition: global_funcs.h:62
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:1019
static uword clib_bitmap_first_set(uword *ai)
Return the lowest numbered set bit in a bitmap.
Definition: bitmap.h:391
u8 * vlib_thread_stack_init(uword thread_index)
Definition: main.c:678
static void vlib_next_frame_init(vlib_next_frame_t *nf)
Definition: node.h:433
vlib_error_main_t error_main
Definition: main.h:177
vlib_thread_callbacks_t cb
Definition: threads.h:322
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:406
int vlib_thread_cb_register(struct vlib_main_t *vm, vlib_thread_callbacks_t *cb)
Definition: threads.c:1613
void vlib_stat_segment_unlock(void)
Definition: stat_segment.c:38
u8 vlib_worker_thread_barrier_held(void)
Return true if the wroker thread barrier is held.
Definition: threads.c:1281
struct _unformat_input_t unformat_input_t
const char * barrier_context
Definition: threads.h:106
char * name
Track name vector.
Definition: elog.h:116
#define clib_error_return_unix(e, args...)
Definition: error.h:102
vlib_main_t ** vlib_mains
Definition: main.h:279
u32 * node_by_error
Definition: node.h:734
#define ELOG_DATA(em, f)
Definition: elog.h:484
__clib_export void clib_interrupt_resize(void **data, uword n_int)
Definition: interrupt.c:38
#define PREDICT_FALSE(x)
Definition: clib.h:124
_vlib_init_function_list_elt_t * worker_init_function_registrations
Definition: main.h:310
static clib_error_t * vlib_launch_thread_int(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:473
void vlib_worker_thread_node_refork(void)
Definition: threads.c:895
clib_error_t *(* vlib_thread_set_lcore_cb)(u32 thread, u16 cpu)
Definition: threads.h:261
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
Definition: nat44_ei.c:3047
u32 node_index
Node index.
Definition: node.h:479
static clib_mem_heap_t * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:161
uword * worker_init_functions_called
Definition: main.h:189
uword * frame_size_hash
Definition: node.h:725
vlib_thread_main_t vlib_thread_main
Definition: threads.c:36
__clib_export word elog_track_register(elog_main_t *em, elog_track_t *t)
register an event track
Definition: elog.c:198
void(* thread_function)(void *)
Definition: threads.h:96
static clib_error_t * cpu_config(vlib_main_t *vm, unformat_input_t *input)
Definition: threads.c:1112
u64 * counters
Definition: error.h:64
u32 owner_node_index
Definition: node.h:346
vlib_frame_queue_t * vlib_frame_queue_alloc(int nelts)
Definition: threads.c:342
volatile u64 tail
Definition: threads.h:122
#define clib_mem_alloc_no_fail(size)
Definition: mem.h:297
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:220
__clib_export void clib_timebase_init(clib_timebase_t *tb, i32 timezone_offset_in_hours, clib_timebase_daylight_time_t daylight_type, clib_time_t *clib_time)
Definition: time_range.c:19
#define UNFORMAT_END_OF_INPUT
Definition: format.h:137
svmdb_client_t * c
vlib_frame_queue_t ** vlib_frame_queues
Definition: threads.h:154
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:208
clib_error_t * vlib_call_init_exit_functions_no_sort(vlib_main_t *vm, _vlib_init_function_list_elt_t **headp, int call_once, int is_global)
Definition: init.c:382
vlib_pending_frame_t * pending_frames
Definition: node.h:695
vlib_thread_function_t * function
Definition: threads.h:35
__clib_export void clib_interrupt_init(void **data, uword n_int)
Definition: interrupt.c:24
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:261
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:395
void os_panic(void)
void * heap_base
Definition: main.h:156
#define clib_warning(format, args...)
Definition: error.h:59
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:116
void vlib_stat_segment_lock(void)
Definition: stat_segment.c:30
__clib_export uword unformat_bitmap_list(unformat_input_t *input, va_list *va)
unformat a list of bit ranges into a bitmap (eg "0-3,5-7,11" )
Definition: bitmap.c:55
static uword clib_bitmap_get(uword *ai, uword i)
Gets the ith bit value from a bitmap.
Definition: bitmap.h:197
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:442
vlib_node_runtime_perf_callback_set_t vlib_node_runtime_perf_callbacks
Definition: main.h:134
void vlib_worker_thread_init(vlib_worker_thread_t *w)
Definition: threads.c:382
uword os_get_nthreads(void)
Definition: threads.c:139
volatile u32 * wait_at_barrier
Definition: threads.h:89
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:163
static clib_mem_heap_t * clib_mem_get_heap(void)
Definition: mem.h:359
vlib_frame_queue_per_thread_data_t * per_thread_data
Definition: threads.h:155
static vlib_main_t * vlib_get_first_main(void)
Definition: global_funcs.h:44
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
Definition: threads.h:145
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define ASSERT(truth)
u64 last_cpu_time
Definition: time.h:51
static clib_error_t * show_clock_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: threads.c:1658
static void barrier_trace_sync_rec(f64 t_entry)
Definition: threads.c:71
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:716
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:310
__clib_export void clib_time_init(clib_time_t *c)
Definition: time.c:207
void * interrupts
Definition: node.h:682
static void clib_mem_free(void *p)
Definition: mem.h:311
void vlib_worker_thread_initial_barrier_sync_and_release(vlib_main_t *vm)
Definition: threads.c:1253
static_always_inline void vlib_process_finish_switch_stack(vlib_main_t *vm)
Definition: node_funcs.h:67
#define clib_error_report(e)
Definition: error.h:113
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
static u32 vlib_get_n_threads()
Definition: global_funcs.h:23
uword * thread_registrations_by_name
Definition: threads.h:272
static_always_inline void * clib_memcpy_fast(void *restrict dst, const void *restrict src, size_t n)
Definition: string.h:92
#define BARRIER_MINIMUM_OPEN_LIMIT
Definition: threads.c:1245
clib_error_t *(* vlib_launch_thread_cb)(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.h:259
volatile u32 * node_reforks_required
Definition: threads.h:107
const char * barrier_caller
Definition: threads.h:105
#define clib_max(x, y)
Definition: clib.h:335
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:38
#define clib_atomic_fetch_add(a, b)
Definition: atomics.h:23
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:610
void vlib_get_thread_core_numa(vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:439
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:468
static void barrier_trace_release(f64 t_entry, f64 t_closed_total, f64 t_update_main)
Definition: threads.c:113
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1781
#define SYS_gettid
static vlib_main_t * vlib_get_main_by_index(u32 thread_index)
Definition: global_funcs.h:29
void(**volatile barrier_perf_callbacks)(struct vlib_main_t *, u64 t, int leave)
Definition: main.h:252
vlib_thread_main_t * vlib_get_thread_main_not_inline(void)
Definition: threads.c:1706
#define vec_dup_aligned(V, A)
Return copy of vector (no header, alignment specified).
Definition: vec.h:453
f64 barrier_no_close_before
Definition: main.h:248
static clib_error_t * start_workers(vlib_main_t *vm)
Definition: threads.c:523
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void vlib_rpc_call_main_thread(void *callback, u8 *args, u32 arg_size)
Definition: threads.c:1637
vlib_main_t vlib_node_runtime_t * node
Definition: nat44_ei.c:3047
vlib_node_main_t node_main
Definition: main.h:171
void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks)
Definition: main.c:590
u64 uword
Definition: types.h:112
vlib_next_frame_t * next_frames
Definition: node.h:692
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
Definition: vec.h:1098
node node_index
__clib_export clib_mem_main_t clib_mem_main
Definition: mem.c:22
#define VLIB_THREAD_STACK_SIZE
Definition: threads.h:65
f64 barrier_epoch
Definition: main.h:245
vlib_frame_size_t * frame_sizes
Definition: node.h:728
#define hash_get_mem(h, key)
Definition: hash.h:269
unformat_function_t unformat_memory_size
Definition: format.h:288
static void worker_thread_node_runtime_update_internal(void)
Definition: threads.c:848
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:261
volatile u32 * workers_at_barrier
Definition: threads.h:90
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1386
#define clib_call_callbacks(h,...)
call the specified callback set
Definition: callback.h:67
nat44_ei_main_t * nm
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:56
uword clib_bitmap_t
Definition: bitmap.h:50
f64 now
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:86
#define vec_foreach(var, vec)
Vector iterator.
__clib_export clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
Definition: sysfs.c:51
void * vlib_worker_thread_bootstrap_fn(void *arg)
Definition: threads.c:419
u8 count
Definition: dhcp.api:208
u16 flags
Copy of main node flags.
Definition: node.h:492
u32 node_runtime_index
Definition: node.h:400
uword * cpu_socket_bitmap
Definition: threads.h:307
#define foreach_sched_policy
Definition: threads.h:242
void * per_numa_mheaps[CLIB_MAX_NUMAS]
Definition: mem.h:143
vlib_thread_registration_t ** registrations
Definition: threads.h:270
elog_main_t elog_main
Definition: main.h:300
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:571
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u8 ** vlib_thread_stacks
Definition: main.c:659
pthread_t thread_id
Definition: threads.h:113
vlib_thread_registration_t * registration
Definition: threads.h:101
volatile u32 worker_thread_release
Definition: threads.h:313
vlib_global_main_t vlib_global_main
Definition: main.c:1786
__clib_export int clib_mem_set_default_numa_affinity()
Definition: mem.c:674
clib_random_buffer_t random_buffer
Definition: main.h:210
u8 runtime_data_bytes
Definition: node.h:303
__clib_export u32 elog_string(elog_main_t *em, char *fmt,...)
add a string to the event-log string table
Definition: elog.c:582
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
#define FRAME_QUEUE_MAX_NELTS
Definition: node.h:763
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:163
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:173
int need_vlib_worker_thread_node_runtime_update
Definition: main.h:291