FD.io VPP  v19.04-6-g6f05f72
Vector Packet Processing
threads.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #define _GNU_SOURCE
16 
17 #include <signal.h>
18 #include <math.h>
19 #include <vppinfra/format.h>
20 #include <vppinfra/linux/sysfs.h>
21 #include <vlib/vlib.h>
22 
23 #include <vlib/threads.h>
24 #include <vlib/unix/cj.h>
25 
27 
28 #define FRAME_QUEUE_NELTS 64
29 
30 u32
31 vl (void *p)
32 {
33  return vec_len (p);
34 }
35 
38 
39 /*
40  * Barrier tracing can be enabled on a normal build to collect information
41  * on barrier use, including timings and call stacks. Deliberately not
42  * keyed off CLIB_DEBUG, because that can add significant overhead which
43  * imapacts observed timings.
44  */
45 
46 u32
47 elog_global_id_for_msg_name (const char *msg_name)
48 {
49  uword *p, r;
50  static uword *h;
51  u8 *name_copy;
52 
53  if (!h)
54  h = hash_create_string (0, sizeof (uword));
55 
56  p = hash_get_mem (h, msg_name);
57  if (p)
58  return p[0];
59  r = elog_string (&vlib_global_main.elog_main, "%s", msg_name);
60 
61  name_copy = format (0, "%s%c", msg_name, 0);
62 
63  hash_set_mem (h, name_copy, r);
64 
65  return r;
66 }
67 
68 static inline void
69 barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed)
70 {
71  if (!vlib_worker_threads->barrier_elog_enabled)
72  return;
73 
74  /* *INDENT-OFF* */
75  ELOG_TYPE_DECLARE (e) =
76  {
77  .format = "bar-trace-%s-#%d",
78  .format_args = "T4i4",
79  };
80  /* *INDENT-ON* */
81  struct
82  {
83  u32 caller, count, t_entry, t_open, t_closed;
84  } *ed = 0;
85 
87  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
88  ed->caller = elog_global_id_for_msg_name
89  (vlib_worker_threads[0].barrier_caller);
90  ed->t_entry = (int) (1000000.0 * t_entry);
91  ed->t_open = (int) (1000000.0 * t_open);
92  ed->t_closed = (int) (1000000.0 * t_closed);
93 }
94 
95 static inline void
97 {
98  if (!vlib_worker_threads->barrier_elog_enabled)
99  return;
100 
101  /* *INDENT-OFF* */
102  ELOG_TYPE_DECLARE (e) =
103  {
104  .format = "bar-syncrec-%s-#%d",
105  .format_args = "T4i4",
106  };
107  /* *INDENT-ON* */
108  struct
109  {
110  u32 caller, depth;
111  } *ed = 0;
112 
114  ed->depth = (int) vlib_worker_threads[0].recursion_level - 1;
115  ed->caller = elog_global_id_for_msg_name
116  (vlib_worker_threads[0].barrier_caller);
117 }
118 
119 static inline void
121 {
122  if (!vlib_worker_threads->barrier_elog_enabled)
123  return;
124 
125  /* *INDENT-OFF* */
126  ELOG_TYPE_DECLARE (e) =
127  {
128  .format = "bar-relrrec-#%d",
129  .format_args = "i4",
130  };
131  /* *INDENT-ON* */
132  struct
133  {
134  u32 depth;
135  } *ed = 0;
136 
138  ed->depth = (int) vlib_worker_threads[0].recursion_level;
139 }
140 
141 static inline void
142 barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main)
143 {
144  if (!vlib_worker_threads->barrier_elog_enabled)
145  return;
146 
147  /* *INDENT-OFF* */
148  ELOG_TYPE_DECLARE (e) =
149  {
150  .format = "bar-rel-#%d-e%d-u%d-t%d",
151  .format_args = "i4i4i4i4",
152  };
153  /* *INDENT-ON* */
154  struct
155  {
156  u32 count, t_entry, t_update_main, t_closed_total;
157  } *ed = 0;
158 
160  ed->t_entry = (int) (1000000.0 * t_entry);
161  ed->t_update_main = (int) (1000000.0 * t_update_main);
162  ed->t_closed_total = (int) (1000000.0 * t_closed_total);
163  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
164 
165  /* Reset context for next trace */
166  vlib_worker_threads[0].barrier_context = NULL;
167 }
168 
169 uword
171 {
172  u32 len;
173 
174  len = vec_len (vlib_thread_stacks);
175  if (len == 0)
176  return 1;
177  else
178  return len;
179 }
180 
181 void
183 {
184  int pthread_setname_np (pthread_t __target_thread, const char *__name);
185  int rv;
186  pthread_t thread = pthread_self ();
187 
188  if (thread)
189  {
190  rv = pthread_setname_np (thread, name);
191  if (rv)
192  clib_warning ("pthread_setname_np returned %d", rv);
193  }
194 }
195 
196 static int
197 sort_registrations_by_no_clone (void *a0, void *a1)
198 {
199  vlib_thread_registration_t **tr0 = a0;
200  vlib_thread_registration_t **tr1 = a1;
201 
202  return ((i32) ((*tr0)->no_data_structure_clone)
203  - ((i32) ((*tr1)->no_data_structure_clone)));
204 }
205 
206 static uword *
208 {
209  FILE *fp;
210  uword *r = 0;
211 
212  fp = fopen (filename, "r");
213 
214  if (fp != NULL)
215  {
216  u8 *buffer = 0;
217  vec_validate (buffer, 256 - 1);
218  if (fgets ((char *) buffer, 256, fp))
219  {
220  unformat_input_t in;
221  unformat_init_string (&in, (char *) buffer,
222  strlen ((char *) buffer));
223  if (unformat (&in, "%U", unformat_bitmap_list, &r) != 1)
224  clib_warning ("unformat_bitmap_list failed");
225  unformat_free (&in);
226  }
227  vec_free (buffer);
228  fclose (fp);
229  }
230  return r;
231 }
232 
233 
234 /* Called early in the init sequence */
235 
236 clib_error_t *
238 {
242  u32 n_vlib_mains = 1;
243  u32 first_index = 1;
244  u32 i;
245  uword *avail_cpu;
246 
247  /* get bitmaps of active cpu cores and sockets */
248  tm->cpu_core_bitmap =
249  clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online");
250  tm->cpu_socket_bitmap =
251  clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online");
252 
253  avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
254 
255  /* skip cores */
256  for (i = 0; i < tm->skip_cores; i++)
257  {
258  uword c = clib_bitmap_first_set (avail_cpu);
259  if (c == ~0)
260  return clib_error_return (0, "no available cpus to skip");
261 
262  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
263  }
264 
265  /* grab cpu for main thread */
266  if (tm->main_lcore == ~0)
267  {
268  /* if main-lcore is not set, we try to use lcore 1 */
269  if (clib_bitmap_get (avail_cpu, 1))
270  tm->main_lcore = 1;
271  else
272  tm->main_lcore = clib_bitmap_first_set (avail_cpu);
273  if (tm->main_lcore == (u8) ~ 0)
274  return clib_error_return (0, "no available cpus to be used for the"
275  " main thread");
276  }
277  else
278  {
279  if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0)
280  return clib_error_return (0, "cpu %u is not available to be used"
281  " for the main thread", tm->main_lcore);
282  }
283  avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
284 
285  /* assume that there is socket 0 only if there is no data from sysfs */
286  if (!tm->cpu_socket_bitmap)
287  tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1);
288 
289  /* pin main thread to main_lcore */
291  {
293  }
294  else
295  {
296  cpu_set_t cpuset;
297  CPU_ZERO (&cpuset);
298  CPU_SET (tm->main_lcore, &cpuset);
299  pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
300  }
301 
302  /* as many threads as stacks... */
303  vec_validate_aligned (vlib_worker_threads, vec_len (vlib_thread_stacks) - 1,
305 
306  /* Preallocate thread 0 */
307  _vec_len (vlib_worker_threads) = 1;
311  w->cpu_id = tm->main_lcore;
312  w->lwp = syscall (SYS_gettid);
313  w->thread_id = pthread_self ();
314  tm->n_vlib_mains = 1;
315 
316  if (tm->sched_policy != ~0)
317  {
318  struct sched_param sched_param;
319  if (!sched_getparam (w->lwp, &sched_param))
320  {
321  if (tm->sched_priority != ~0)
322  sched_param.sched_priority = tm->sched_priority;
323  sched_setscheduler (w->lwp, tm->sched_policy, &sched_param);
324  }
325  }
326 
327  /* assign threads to cores and set n_vlib_mains */
328  tr = tm->next;
329 
330  while (tr)
331  {
332  vec_add1 (tm->registrations, tr);
333  tr = tr->next;
334  }
335 
337 
338  for (i = 0; i < vec_len (tm->registrations); i++)
339  {
340  int j;
341  tr = tm->registrations[i];
342  tr->first_index = first_index;
343  first_index += tr->count;
344  n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0;
345 
346  /* construct coremask */
347  if (tr->use_pthreads || !tr->count)
348  continue;
349 
350  if (tr->coremask)
351  {
352  uword c;
353  /* *INDENT-OFF* */
354  clib_bitmap_foreach (c, tr->coremask, ({
355  if (clib_bitmap_get(avail_cpu, c) == 0)
356  return clib_error_return (0, "cpu %u is not available to be used"
357  " for the '%s' thread",c, tr->name);
358 
359  avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
360  }));
361 /* *INDENT-ON* */
362 
363  }
364  else
365  {
366  for (j = 0; j < tr->count; j++)
367  {
368  uword c = clib_bitmap_first_set (avail_cpu);
369  if (c == ~0)
370  return clib_error_return (0,
371  "no available cpus to be used for"
372  " the '%s' thread", tr->name);
373 
374  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
375  tr->coremask = clib_bitmap_set (tr->coremask, c, 1);
376  }
377  }
378  }
379 
380  clib_bitmap_free (avail_cpu);
381 
382  tm->n_vlib_mains = n_vlib_mains;
383 
384  vec_validate_aligned (vlib_worker_threads, first_index - 1,
386 
387  return 0;
388 }
389 
392 {
393  vlib_frame_queue_t *fq;
394 
395  fq = clib_mem_alloc_aligned (sizeof (*fq), CLIB_CACHE_LINE_BYTES);
396  clib_memset (fq, 0, sizeof (*fq));
397  fq->nelts = nelts;
398  fq->vector_threshold = 128; // packets
400 
401  if (1)
402  {
403  if (((uword) & fq->tail) & (CLIB_CACHE_LINE_BYTES - 1))
404  fformat (stderr, "WARNING: fq->tail unaligned\n");
405  if (((uword) & fq->head) & (CLIB_CACHE_LINE_BYTES - 1))
406  fformat (stderr, "WARNING: fq->head unaligned\n");
407  if (((uword) fq->elts) & (CLIB_CACHE_LINE_BYTES - 1))
408  fformat (stderr, "WARNING: fq->elts unaligned\n");
409 
410  if (sizeof (fq->elts[0]) % CLIB_CACHE_LINE_BYTES)
411  fformat (stderr, "WARNING: fq->elts[0] size %d\n",
412  sizeof (fq->elts[0]));
413  if (nelts & (nelts - 1))
414  {
415  fformat (stderr, "FATAL: nelts MUST be a power of 2\n");
416  abort ();
417  }
418  }
419 
420  return (fq);
421 }
422 
423 void vl_msg_api_handler_no_free (void *) __attribute__ ((weak));
424 void
426 {
427 }
428 
429 /* Turned off, save as reference material... */
430 #if 0
431 static inline int
432 vlib_frame_queue_dequeue_internal (int thread_id,
433  vlib_main_t * vm, vlib_node_main_t * nm)
434 {
435  vlib_frame_queue_t *fq = vlib_frame_queues[thread_id];
437  vlib_frame_t *f;
440  u32 node_runtime_index;
441  int msg_type;
442  u64 before;
443  int processed = 0;
444 
445  ASSERT (vm == vlib_mains[thread_id]);
446 
447  while (1)
448  {
449  if (fq->head == fq->tail)
450  return processed;
451 
452  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
453 
454  if (!elt->valid)
455  return processed;
456 
457  before = clib_cpu_time_now ();
458 
459  f = elt->frame;
460  node_runtime_index = elt->node_runtime_index;
461  msg_type = elt->msg_type;
462 
463  switch (msg_type)
464  {
465  case VLIB_FRAME_QUEUE_ELT_FREE_BUFFERS:
467  /* note fallthrough... */
468  case VLIB_FRAME_QUEUE_ELT_FREE_FRAME:
470  node_runtime_index);
471  vlib_frame_free (vm, r, f);
472  break;
474  vec_add2 (vm->node_main.pending_frames, p, 1);
476  p->node_runtime_index = elt->node_runtime_index;
477  p->frame_index = vlib_frame_index (vm, f);
479  fq->dequeue_vectors += (u64) f->n_vectors;
480  break;
481  case VLIB_FRAME_QUEUE_ELT_API_MSG:
483  break;
484  default:
485  clib_warning ("bogus frame queue message, type %d", msg_type);
486  break;
487  }
488  elt->valid = 0;
489  fq->dequeues++;
490  fq->dequeue_ticks += clib_cpu_time_now () - before;
492  fq->head++;
493  processed++;
494  }
495  ASSERT (0);
496  return processed;
497 }
498 
499 int
500 vlib_frame_queue_dequeue (int thread_id,
501  vlib_main_t * vm, vlib_node_main_t * nm)
502 {
503  return vlib_frame_queue_dequeue_internal (thread_id, vm, nm);
504 }
505 
506 int
507 vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index,
508  u32 frame_queue_index, vlib_frame_t * frame,
510 {
511  vlib_frame_queue_t *fq = vlib_frame_queues[frame_queue_index];
513  u32 save_count;
514  u64 new_tail;
515  u64 before = clib_cpu_time_now ();
516 
517  ASSERT (fq);
518 
519  new_tail = clib_atomic_add_fetch (&fq->tail, 1);
520 
521  /* Wait until a ring slot is available */
522  while (new_tail >= fq->head + fq->nelts)
523  {
524  f64 b4 = vlib_time_now_ticks (vm, before);
526  /* Bad idea. Dequeue -> enqueue -> dequeue -> trouble */
527  // vlib_frame_queue_dequeue (vm->thread_index, vm, nm);
528  }
529 
530  elt = fq->elts + (new_tail & (fq->nelts - 1));
531 
532  /* this would be very bad... */
533  while (elt->valid)
534  {
535  }
536 
537  /* Once we enqueue the frame, frame->n_vectors is owned elsewhere... */
538  save_count = frame->n_vectors;
539 
540  elt->frame = frame;
541  elt->node_runtime_index = node_runtime_index;
542  elt->msg_type = type;
544  elt->valid = 1;
545 
546  return save_count;
547 }
548 #endif /* 0 */
549 
550 /* To be called by vlib worker threads upon startup */
551 void
553 {
555 
556  /*
557  * Note: disabling signals in worker threads as follows
558  * prevents the api post-mortem dump scheme from working
559  * {
560  * sigset_t s;
561  * sigfillset (&s);
562  * pthread_sigmask (SIG_SETMASK, &s, 0);
563  * }
564  */
565 
567 
568  if (vec_len (tm->thread_prefix) && w->registration->short_name)
569  {
570  w->name = format (0, "%v_%s_%d%c", tm->thread_prefix,
571  w->registration->short_name, w->instance_id, '\0');
572  vlib_set_thread_name ((char *) w->name);
573  }
574 
575  if (!w->registration->use_pthreads)
576  {
577 
578  /* Initial barrier sync, for both worker and i/o threads */
579  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
580 
581  while (*vlib_worker_threads->wait_at_barrier)
582  ;
583 
584  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
585  }
586 }
587 
588 void *
590 {
591  void *rv;
592  vlib_worker_thread_t *w = arg;
593 
594  w->lwp = syscall (SYS_gettid);
595  w->thread_id = pthread_self ();
596 
597  __os_thread_index = w - vlib_worker_threads;
598 
599  rv = (void *) clib_calljmp
600  ((uword (*)(uword)) w->thread_function,
602  /* NOTREACHED, we hope */
603  return rv;
604 }
605 
606 static void
608 {
609  const char *sys_cpu_path = "/sys/devices/system/cpu/cpu";
610  u8 *p = 0;
611  int core_id = -1, socket_id = -1;
612 
613  p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, cpu_id, 0);
614  clib_sysfs_read ((char *) p, "%d", &core_id);
615  vec_reset_length (p);
616  p =
617  format (p, "%s%u/topology/physical_package_id%c", sys_cpu_path, cpu_id,
618  0);
619  clib_sysfs_read ((char *) p, "%d", &socket_id);
620  vec_free (p);
621 
622  w->core_id = core_id;
623  w->socket_id = socket_id;
624 }
625 
626 static clib_error_t *
627 vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
628 {
630  void *(*fp_arg) (void *) = fp;
631 
632  w->cpu_id = cpu_id;
633  vlib_get_thread_core_socket (w, cpu_id);
635  return tm->cb.vlib_launch_thread_cb (fp, (void *) w, cpu_id);
636  else
637  {
638  pthread_t worker;
639  cpu_set_t cpuset;
640  CPU_ZERO (&cpuset);
641  CPU_SET (cpu_id, &cpuset);
642 
643  if (pthread_create (&worker, NULL /* attr */ , fp_arg, (void *) w))
644  return clib_error_return_unix (0, "pthread_create");
645 
646  if (pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset))
647  return clib_error_return_unix (0, "pthread_setaffinity_np");
648 
649  return 0;
650  }
651 }
652 
653 static clib_error_t *
655 {
656  int i, j;
658  vlib_main_t *vm_clone;
659  void *oldheap;
663  u32 n_vlib_mains = tm->n_vlib_mains;
664  u32 worker_thread_index;
665  u8 *main_heap = clib_mem_get_per_cpu_heap ();
666 
667  vec_reset_length (vlib_worker_threads);
668 
669  /* Set up the main thread */
670  vec_add2_aligned (vlib_worker_threads, w, 1, CLIB_CACHE_LINE_BYTES);
671  w->elog_track.name = "main thread";
673 
674  if (vec_len (tm->thread_prefix))
675  {
676  w->name = format (0, "%v_main%c", tm->thread_prefix, '\0');
677  vlib_set_thread_name ((char *) w->name);
678  }
679 
680  vm->elog_main.lock =
682  vm->elog_main.lock[0] = 0;
683 
684  if (n_vlib_mains > 1)
685  {
686  /* Replace hand-crafted length-1 vector with a real vector */
687  vlib_mains = 0;
688 
691  _vec_len (vlib_mains) = 0;
693 
694  vlib_worker_threads->wait_at_barrier =
696  vlib_worker_threads->workers_at_barrier =
698 
699  vlib_worker_threads->node_reforks_required =
701 
702  /* We'll need the rpc vector lock... */
704 
705  /* Ask for an initial barrier sync */
706  *vlib_worker_threads->workers_at_barrier = 0;
707  *vlib_worker_threads->wait_at_barrier = 1;
708 
709  /* Without update or refork */
710  *vlib_worker_threads->node_reforks_required = 0;
712 
713  /* init timing */
714  vm->barrier_epoch = 0;
715  vm->barrier_no_close_before = 0;
716 
717  worker_thread_index = 1;
718 
719  for (i = 0; i < vec_len (tm->registrations); i++)
720  {
721  vlib_node_main_t *nm, *nm_clone;
722  int k;
723 
724  tr = tm->registrations[i];
725 
726  if (tr->count == 0)
727  continue;
728 
729  for (k = 0; k < tr->count; k++)
730  {
731  vlib_node_t *n;
732 
733  vec_add2 (vlib_worker_threads, w, 1);
734  /* Currently unused, may not really work */
735  if (tr->mheap_size)
736  {
737 #if USE_DLMALLOC == 0
738  w->thread_mheap =
739  mheap_alloc (0 /* use VM */ , tr->mheap_size);
740 #else
742  0 /* unlocked */ );
743 #endif
744  }
745  else
746  w->thread_mheap = main_heap;
747 
748  w->thread_stack =
749  vlib_thread_stack_init (w - vlib_worker_threads);
750  w->thread_function = tr->function;
751  w->thread_function_arg = w;
752  w->instance_id = k;
753  w->registration = tr;
754 
755  w->elog_track.name =
756  (char *) format (0, "%s %d", tr->name, k + 1);
757  vec_add1 (w->elog_track.name, 0);
759 
760  if (tr->no_data_structure_clone)
761  continue;
762 
763  /* Fork vlib_global_main et al. Look for bugs here */
764  oldheap = clib_mem_set_heap (w->thread_mheap);
765 
766  vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone),
768  clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone));
769 
770  vm_clone->thread_index = worker_thread_index;
771  vm_clone->heap_base = w->thread_mheap;
772  vm_clone->heap_aligned_base = (void *)
773  (((uword) w->thread_mheap) & ~(VLIB_FRAME_ALIGN - 1));
774  vm_clone->init_functions_called =
775  hash_create (0, /* value bytes */ 0);
776  vm_clone->pending_rpc_requests = 0;
777  vec_validate (vm_clone->pending_rpc_requests, 0);
778  _vec_len (vm_clone->pending_rpc_requests) = 0;
779  clib_memset (&vm_clone->random_buffer, 0,
780  sizeof (vm_clone->random_buffer));
781 
782  nm = &vlib_mains[0]->node_main;
783  nm_clone = &vm_clone->node_main;
784  /* fork next frames array, preserving node runtime indices */
785  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
787  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
788  {
789  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
790  u32 save_node_runtime_index;
791  u32 save_flags;
792 
793  save_node_runtime_index = nf->node_runtime_index;
794  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
796  nf->node_runtime_index = save_node_runtime_index;
797  nf->flags = save_flags;
798  }
799 
800  /* fork the frame dispatch queue */
801  nm_clone->pending_frames = 0;
802  vec_validate (nm_clone->pending_frames, 10);
803  _vec_len (nm_clone->pending_frames) = 0;
804 
805  /* fork nodes */
806  nm_clone->nodes = 0;
807 
808  /* Allocate all nodes in single block for speed */
809  n = clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*n));
810 
811  for (j = 0; j < vec_len (nm->nodes); j++)
812  {
813  clib_memcpy (n, nm->nodes[j], sizeof (*n));
814  /* none of the copied nodes have enqueue rights given out */
816  clib_memset (&n->stats_total, 0, sizeof (n->stats_total));
818  sizeof (n->stats_last_clear));
819  vec_add1 (nm_clone->nodes, n);
820  n++;
821  }
825  vec_foreach (rt,
827  {
828  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
829  rt->thread_index = vm_clone->thread_index;
830  /* copy initial runtime_data from node */
831  if (n->runtime_data && n->runtime_data_bytes > 0)
834  n->runtime_data_bytes));
835  }
836 
841  {
842  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
843  rt->thread_index = vm_clone->thread_index;
844  /* copy initial runtime_data from node */
845  if (n->runtime_data && n->runtime_data_bytes > 0)
848  n->runtime_data_bytes));
849  }
850 
854  vec_foreach (rt,
856  {
857  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
858  rt->thread_index = vm_clone->thread_index;
859  /* copy initial runtime_data from node */
860  if (n->runtime_data && n->runtime_data_bytes > 0)
863  n->runtime_data_bytes));
864  }
865 
866  nm_clone->processes = vec_dup_aligned (nm->processes,
868 
869  /* Create per-thread frame freelist */
870  nm_clone->frame_sizes = vec_new (vlib_frame_size_t, 1);
871 #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
872  nm_clone->frame_size_hash = hash_create (0, sizeof (uword));
873 #endif
874 
875  /* Packet trace buffers are guaranteed to be empty, nothing to do here */
876 
877  clib_mem_set_heap (oldheap);
879 
881  (vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES);
883  (vlib_mains[0]->error_main.counters_last_clear,
885 
886  worker_thread_index++;
887  }
888  }
889  }
890  else
891  {
892  /* only have non-data-structure copy threads to create... */
893  for (i = 0; i < vec_len (tm->registrations); i++)
894  {
895  tr = tm->registrations[i];
896 
897  for (j = 0; j < tr->count; j++)
898  {
899  vec_add2 (vlib_worker_threads, w, 1);
900  if (tr->mheap_size)
901  {
902 #if USE_DLMALLOC == 0
903  w->thread_mheap =
904  mheap_alloc (0 /* use VM */ , tr->mheap_size);
905 #else
906  w->thread_mheap =
907  create_mspace (tr->mheap_size, 0 /* locked */ );
908 #endif
909  }
910  else
911  w->thread_mheap = main_heap;
912  w->thread_stack =
913  vlib_thread_stack_init (w - vlib_worker_threads);
914  w->thread_function = tr->function;
915  w->thread_function_arg = w;
916  w->instance_id = j;
917  w->elog_track.name =
918  (char *) format (0, "%s %d", tr->name, j + 1);
919  w->registration = tr;
920  vec_add1 (w->elog_track.name, 0);
922  }
923  }
924  }
925 
926  worker_thread_index = 1;
927 
928  for (i = 0; i < vec_len (tm->registrations); i++)
929  {
930  clib_error_t *err;
931  int j;
932 
933  tr = tm->registrations[i];
934 
935  if (tr->use_pthreads || tm->use_pthreads)
936  {
937  for (j = 0; j < tr->count; j++)
938  {
939  w = vlib_worker_threads + worker_thread_index++;
941  w, 0);
942  if (err)
943  clib_error_report (err);
944  }
945  }
946  else
947  {
948  uword c;
949  /* *INDENT-OFF* */
950  clib_bitmap_foreach (c, tr->coremask, ({
951  w = vlib_worker_threads + worker_thread_index++;
952  err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
953  w, c);
954  if (err)
955  clib_error_report (err);
956  }));
957  /* *INDENT-ON* */
958  }
959  }
962  return 0;
963 }
964 
966 
967 
968 static inline void
970 {
971  int i, j;
972  vlib_main_t *vm;
973  vlib_node_main_t *nm, *nm_clone;
974  vlib_main_t *vm_clone;
976  never_inline void
979  uword n_calls,
980  uword n_vectors, uword n_clocks);
981 
982  ASSERT (vlib_get_thread_index () == 0);
983 
984  vm = vlib_mains[0];
985  nm = &vm->node_main;
986 
987  ASSERT (*vlib_worker_threads->wait_at_barrier == 1);
988 
989  /*
990  * Scrape all runtime stats, so we don't lose node runtime(s) with
991  * pending counts, or throw away worker / io thread counts.
992  */
993  for (j = 0; j < vec_len (nm->nodes); j++)
994  {
995  vlib_node_t *n;
996  n = nm->nodes[j];
997  vlib_node_sync_stats (vm, n);
998  }
999 
1000  for (i = 1; i < vec_len (vlib_mains); i++)
1001  {
1002  vlib_node_t *n;
1003 
1004  vm_clone = vlib_mains[i];
1005  nm_clone = &vm_clone->node_main;
1006 
1007  for (j = 0; j < vec_len (nm_clone->nodes); j++)
1008  {
1009  n = nm_clone->nodes[j];
1010 
1011  rt = vlib_node_get_runtime (vm_clone, n->index);
1012  vlib_node_runtime_sync_stats (vm_clone, rt, 0, 0, 0);
1013  }
1014  }
1015 
1016  /* Per-worker clone rebuilds are now done on each thread */
1017 }
1018 
1019 
1020 void
1022 {
1023  vlib_main_t *vm, *vm_clone;
1024  vlib_node_main_t *nm, *nm_clone;
1025  vlib_node_t **old_nodes_clone;
1026  vlib_node_runtime_t *rt, *old_rt;
1027 
1028  vlib_node_t *new_n_clone;
1029 
1030  int j;
1031 
1032  vm = vlib_mains[0];
1033  nm = &vm->node_main;
1034  vm_clone = vlib_get_main ();
1035  nm_clone = &vm_clone->node_main;
1036 
1037  /* Re-clone error heap */
1038  u64 *old_counters = vm_clone->error_main.counters;
1039  u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear;
1040 
1041  clib_memcpy_fast (&vm_clone->error_main, &vm->error_main,
1042  sizeof (vm->error_main));
1043  j = vec_len (vm->error_main.counters) - 1;
1044  vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES);
1045  vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
1046  vm_clone->error_main.counters = old_counters;
1047  vm_clone->error_main.counters_last_clear = old_counters_all_clear;
1048 
1049  nm_clone = &vm_clone->node_main;
1050  vec_free (nm_clone->next_frames);
1051  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
1053 
1054  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
1055  {
1056  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
1057  u32 save_node_runtime_index;
1058  u32 save_flags;
1059 
1060  save_node_runtime_index = nf->node_runtime_index;
1061  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
1062  vlib_next_frame_init (nf);
1063  nf->node_runtime_index = save_node_runtime_index;
1064  nf->flags = save_flags;
1065  }
1066 
1067  old_nodes_clone = nm_clone->nodes;
1068  nm_clone->nodes = 0;
1069 
1070  /* re-fork nodes */
1071 
1072  /* Allocate all nodes in single block for speed */
1073  new_n_clone =
1074  clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*new_n_clone));
1075  for (j = 0; j < vec_len (nm->nodes); j++)
1076  {
1077  vlib_node_t *old_n_clone;
1078  vlib_node_t *new_n;
1079 
1080  new_n = nm->nodes[j];
1081  old_n_clone = old_nodes_clone[j];
1082 
1083  clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n));
1084  /* none of the copied nodes have enqueue rights given out */
1086 
1087  if (j >= vec_len (old_nodes_clone))
1088  {
1089  /* new node, set to zero */
1090  clib_memset (&new_n_clone->stats_total, 0,
1091  sizeof (new_n_clone->stats_total));
1092  clib_memset (&new_n_clone->stats_last_clear, 0,
1093  sizeof (new_n_clone->stats_last_clear));
1094  }
1095  else
1096  {
1097  /* Copy stats if the old data is valid */
1098  clib_memcpy_fast (&new_n_clone->stats_total,
1099  &old_n_clone->stats_total,
1100  sizeof (new_n_clone->stats_total));
1101  clib_memcpy_fast (&new_n_clone->stats_last_clear,
1102  &old_n_clone->stats_last_clear,
1103  sizeof (new_n_clone->stats_last_clear));
1104 
1105  /* keep previous node state */
1106  new_n_clone->state = old_n_clone->state;
1107  }
1108  vec_add1 (nm_clone->nodes, new_n_clone);
1109  new_n_clone++;
1110  }
1111  /* Free the old node clones */
1112  clib_mem_free (old_nodes_clone[0]);
1113 
1114  vec_free (old_nodes_clone);
1115 
1116 
1117  /* re-clone internal nodes */
1118  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
1122 
1124  {
1125  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1126  rt->thread_index = vm_clone->thread_index;
1127  /* copy runtime_data, will be overwritten later for existing rt */
1128  if (n->runtime_data && n->runtime_data_bytes > 0)
1131  n->runtime_data_bytes));
1132  }
1133 
1134  for (j = 0; j < vec_len (old_rt); j++)
1135  {
1136  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1137  rt->state = old_rt[j].state;
1138  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1140  }
1141 
1142  vec_free (old_rt);
1143 
1144  /* re-clone input nodes */
1145  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
1146  nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
1149 
1151  {
1152  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1153  rt->thread_index = vm_clone->thread_index;
1154  /* copy runtime_data, will be overwritten later for existing rt */
1155  if (n->runtime_data && n->runtime_data_bytes > 0)
1158  n->runtime_data_bytes));
1159  }
1160 
1161  for (j = 0; j < vec_len (old_rt); j++)
1162  {
1163  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1164  rt->state = old_rt[j].state;
1165  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1167  }
1168 
1169  vec_free (old_rt);
1170 
1171  /* re-clone pre-input nodes */
1172  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT];
1176 
1178  {
1179  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1180  rt->thread_index = vm_clone->thread_index;
1181  /* copy runtime_data, will be overwritten later for existing rt */
1182  if (n->runtime_data && n->runtime_data_bytes > 0)
1185  n->runtime_data_bytes));
1186  }
1187 
1188  for (j = 0; j < vec_len (old_rt); j++)
1189  {
1190  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1191  rt->state = old_rt[j].state;
1192  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1194  }
1195 
1196  vec_free (old_rt);
1197 
1198  nm_clone->processes = vec_dup_aligned (nm->processes,
1200 }
1201 
1202 void
1204 {
1205  /*
1206  * Make a note that we need to do a node runtime update
1207  * prior to releasing the barrier.
1208  */
1210 }
1211 
1212 u32
1213 unformat_sched_policy (unformat_input_t * input, va_list * args)
1214 {
1215  u32 *r = va_arg (*args, u32 *);
1216 
1217  if (0);
1218 #define _(v,f,s) else if (unformat (input, s)) *r = SCHED_POLICY_##f;
1220 #undef _
1221  else
1222  return 0;
1223  return 1;
1224 }
1225 
1226 static clib_error_t *
1228 {
1230  uword *p;
1232  u8 *name;
1233  uword *bitmap;
1234  u32 count;
1235 
1237 
1238  tm->n_thread_stacks = 1; /* account for main thread */
1239  tm->sched_policy = ~0;
1240  tm->sched_priority = ~0;
1241  tm->main_lcore = ~0;
1242 
1243  tr = tm->next;
1244 
1245  while (tr)
1246  {
1248  tr = tr->next;
1249  }
1250 
1251  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1252  {
1253  if (unformat (input, "use-pthreads"))
1254  tm->use_pthreads = 1;
1255  else if (unformat (input, "thread-prefix %v", &tm->thread_prefix))
1256  ;
1257  else if (unformat (input, "main-core %u", &tm->main_lcore))
1258  ;
1259  else if (unformat (input, "skip-cores %u", &tm->skip_cores))
1260  ;
1261  else if (unformat (input, "coremask-%s %U", &name,
1262  unformat_bitmap_mask, &bitmap) ||
1263  unformat (input, "corelist-%s %U", &name,
1264  unformat_bitmap_list, &bitmap))
1265  {
1267  if (p == 0)
1268  return clib_error_return (0, "no such thread type '%s'", name);
1269 
1270  tr = (vlib_thread_registration_t *) p[0];
1271 
1272  if (tr->use_pthreads)
1273  return clib_error_return (0,
1274  "corelist cannot be set for '%s' threads",
1275  name);
1276 
1277  tr->coremask = bitmap;
1279  }
1280  else
1281  if (unformat
1282  (input, "scheduler-policy %U", unformat_sched_policy,
1283  &tm->sched_policy))
1284  ;
1285  else if (unformat (input, "scheduler-priority %u", &tm->sched_priority))
1286  ;
1287  else if (unformat (input, "%s %u", &name, &count))
1288  {
1290  if (p == 0)
1291  return clib_error_return (0, "no such thread type 3 '%s'", name);
1292 
1293  tr = (vlib_thread_registration_t *) p[0];
1294  if (tr->fixed_count)
1295  return clib_error_return
1296  (0, "number of %s threads not configurable", tr->name);
1297  tr->count = count;
1298  }
1299  else
1300  break;
1301  }
1302 
1303  if (tm->sched_priority != ~0)
1304  {
1305  if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR)
1306  {
1307  u32 prio_max = sched_get_priority_max (tm->sched_policy);
1308  u32 prio_min = sched_get_priority_min (tm->sched_policy);
1309  if (tm->sched_priority > prio_max)
1310  tm->sched_priority = prio_max;
1311  if (tm->sched_priority < prio_min)
1312  tm->sched_priority = prio_min;
1313  }
1314  else
1315  {
1316  return clib_error_return
1317  (0,
1318  "scheduling priority (%d) is not allowed for `normal` scheduling policy",
1319  tm->sched_priority);
1320  }
1321  }
1322  tr = tm->next;
1323 
1324  if (!tm->thread_prefix)
1325  tm->thread_prefix = format (0, "vpp");
1326 
1327  while (tr)
1328  {
1329  tm->n_thread_stacks += tr->count;
1330  tm->n_pthreads += tr->count * tr->use_pthreads;
1331  tm->n_threads += tr->count * (tr->use_pthreads == 0);
1332  tr = tr->next;
1333  }
1334 
1335  return 0;
1336 }
1337 
1339 
1340 void vnet_main_fixup (vlib_fork_fixup_t which) __attribute__ ((weak));
1341 void
1343 {
1344 }
1345 
1346 void
1348 {
1349  vlib_main_t *vm = vlib_get_main ();
1350 
1351  if (vlib_mains == 0)
1352  return;
1353 
1354  ASSERT (vlib_get_thread_index () == 0);
1356 
1357  switch (which)
1358  {
1361  break;
1362 
1363  default:
1364  ASSERT (0);
1365  }
1367 }
1368 
1369  /*
1370  * Enforce minimum open time to minimize packet loss due to Rx overflow,
1371  * based on a test based heuristic that barrier should be open for at least
1372  * 3 time as long as it is closed (with an upper bound of 1ms because by that
1373  * point it is probably too late to make a difference)
1374  */
1375 
1376 #ifndef BARRIER_MINIMUM_OPEN_LIMIT
1377 #define BARRIER_MINIMUM_OPEN_LIMIT 0.001
1378 #endif
1379 
1380 #ifndef BARRIER_MINIMUM_OPEN_FACTOR
1381 #define BARRIER_MINIMUM_OPEN_FACTOR 3
1382 #endif
1383 
1384 void
1386 {
1387  f64 deadline;
1388  f64 now;
1389  f64 t_entry;
1390  f64 t_open;
1391  f64 t_closed;
1392  f64 max_vector_rate;
1393  u32 count;
1394  int i;
1395 
1396  if (vec_len (vlib_mains) < 2)
1397  return;
1398 
1399  ASSERT (vlib_get_thread_index () == 0);
1400 
1401  vlib_worker_threads[0].barrier_caller = func_name;
1402  count = vec_len (vlib_mains) - 1;
1403 
1404  /* Record entry relative to last close */
1405  now = vlib_time_now (vm);
1406  t_entry = now - vm->barrier_epoch;
1407 
1408  /* Tolerate recursive calls */
1409  if (++vlib_worker_threads[0].recursion_level > 1)
1410  {
1411  barrier_trace_sync_rec (t_entry);
1412  return;
1413  }
1414 
1415  /*
1416  * Need data to decide if we're working hard enough to honor
1417  * the barrier hold-down timer.
1418  */
1419  max_vector_rate = 0.0;
1420  for (i = 1; i < vec_len (vlib_mains); i++)
1421  max_vector_rate =
1422  clib_max (max_vector_rate,
1424 
1425  vlib_worker_threads[0].barrier_sync_count++;
1426 
1427  /* Enforce minimum barrier open time to minimize packet loss */
1429 
1430  /*
1431  * If any worker thread seems busy, which we define
1432  * as a vector rate above 10, we enforce the barrier hold-down timer
1433  */
1434  if (max_vector_rate > 10.0)
1435  {
1436  while (1)
1437  {
1438  now = vlib_time_now (vm);
1439  /* Barrier hold-down timer expired? */
1440  if (now >= vm->barrier_no_close_before)
1441  break;
1442  if ((vm->barrier_no_close_before - now)
1443  > (2.0 * BARRIER_MINIMUM_OPEN_LIMIT))
1444  {
1445  clib_warning
1446  ("clock change: would have waited for %.4f seconds",
1447  (vm->barrier_no_close_before - now));
1448  break;
1449  }
1450  }
1451  }
1452  /* Record time of closure */
1453  t_open = now - vm->barrier_epoch;
1454  vm->barrier_epoch = now;
1455 
1456  deadline = now + BARRIER_SYNC_TIMEOUT;
1457 
1458  *vlib_worker_threads->wait_at_barrier = 1;
1459  while (*vlib_worker_threads->workers_at_barrier != count)
1460  {
1461  if ((now = vlib_time_now (vm)) > deadline)
1462  {
1463  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1464  os_panic ();
1465  }
1466  }
1467 
1468  t_closed = now - vm->barrier_epoch;
1469 
1470  barrier_trace_sync (t_entry, t_open, t_closed);
1471 
1472 }
1473 
1474 void vlib_stat_segment_lock (void) __attribute__ ((weak));
1475 void
1477 {
1478 }
1479 
1480 void vlib_stat_segment_unlock (void) __attribute__ ((weak));
1481 void
1483 {
1484 }
1485 
1486 void
1488 {
1489  f64 deadline;
1490  f64 now;
1491  f64 minimum_open;
1492  f64 t_entry;
1493  f64 t_closed_total;
1494  f64 t_update_main = 0.0;
1495  int refork_needed = 0;
1496 
1497  if (vec_len (vlib_mains) < 2)
1498  return;
1499 
1500  ASSERT (vlib_get_thread_index () == 0);
1501 
1502 
1503  now = vlib_time_now (vm);
1504  t_entry = now - vm->barrier_epoch;
1505 
1506  if (--vlib_worker_threads[0].recursion_level > 0)
1507  {
1508  barrier_trace_release_rec (t_entry);
1509  return;
1510  }
1511 
1512  /* Update (all) node runtimes before releasing the barrier, if needed */
1514  {
1515  /*
1516  * Lock stat segment here, so we's safe when
1517  * rebuilding the stat segment node clones from the
1518  * stat thread...
1519  */
1521 
1522  /* Do stats elements on main thread */
1525 
1526  /* Do per thread rebuilds in parallel */
1527  refork_needed = 1;
1528  clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
1529  (vec_len (vlib_mains) - 1));
1530  now = vlib_time_now (vm);
1531  t_update_main = now - vm->barrier_epoch;
1532  }
1533 
1534  deadline = now + BARRIER_SYNC_TIMEOUT;
1535 
1536  /*
1537  * Note when we let go of the barrier.
1538  * Workers can use this to derive a reasonably accurate
1539  * time offset. See vlib_time_now(...)
1540  */
1543 
1544  *vlib_worker_threads->wait_at_barrier = 0;
1545 
1546  while (*vlib_worker_threads->workers_at_barrier > 0)
1547  {
1548  if ((now = vlib_time_now (vm)) > deadline)
1549  {
1550  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1551  os_panic ();
1552  }
1553  }
1554 
1555  /* Wait for reforks before continuing */
1556  if (refork_needed)
1557  {
1558  now = vlib_time_now (vm);
1559 
1560  deadline = now + BARRIER_SYNC_TIMEOUT;
1561 
1562  while (*vlib_worker_threads->node_reforks_required > 0)
1563  {
1564  if ((now = vlib_time_now (vm)) > deadline)
1565  {
1566  fformat (stderr, "%s: worker thread refork deadlock\n",
1567  __FUNCTION__);
1568  os_panic ();
1569  }
1570  }
1572  }
1573 
1574  t_closed_total = now - vm->barrier_epoch;
1575 
1576  minimum_open = t_closed_total * BARRIER_MINIMUM_OPEN_FACTOR;
1577 
1578  if (minimum_open > BARRIER_MINIMUM_OPEN_LIMIT)
1579  {
1580  minimum_open = BARRIER_MINIMUM_OPEN_LIMIT;
1581  }
1582 
1583  vm->barrier_no_close_before = now + minimum_open;
1584 
1585  /* Record barrier epoch (used to enforce minimum open time) */
1586  vm->barrier_epoch = now;
1587 
1588  barrier_trace_release (t_entry, t_closed_total, t_update_main);
1589 
1590 }
1591 
1592 /*
1593  * Check the frame queue to see if any frames are available.
1594  * If so, pull the packets off the frames and put them to
1595  * the handoff node.
1596  */
1597 int
1599 {
1600  u32 thread_id = vm->thread_index;
1601  vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id];
1603  u32 *from, *to;
1604  vlib_frame_t *f;
1605  int msg_type;
1606  int processed = 0;
1607  u32 n_left_to_node;
1608  u32 vectors = 0;
1609 
1610  ASSERT (fq);
1611  ASSERT (vm == vlib_mains[thread_id]);
1612 
1613  if (PREDICT_FALSE (fqm->node_index == ~0))
1614  return 0;
1615  /*
1616  * Gather trace data for frame queues
1617  */
1618  if (PREDICT_FALSE (fq->trace))
1619  {
1620  frame_queue_trace_t *fqt;
1622  u32 elix;
1623 
1624  fqt = &fqm->frame_queue_traces[thread_id];
1625 
1626  fqt->nelts = fq->nelts;
1627  fqt->head = fq->head;
1628  fqt->head_hint = fq->head_hint;
1629  fqt->tail = fq->tail;
1630  fqt->threshold = fq->vector_threshold;
1631  fqt->n_in_use = fqt->tail - fqt->head;
1632  if (fqt->n_in_use >= fqt->nelts)
1633  {
1634  // if beyond max then use max
1635  fqt->n_in_use = fqt->nelts - 1;
1636  }
1637 
1638  /* Record the number of elements in use in the histogram */
1639  fqh = &fqm->frame_queue_histogram[thread_id];
1640  fqh->count[fqt->n_in_use]++;
1641 
1642  /* Record a snapshot of the elements in use */
1643  for (elix = 0; elix < fqt->nelts; elix++)
1644  {
1645  elt = fq->elts + ((fq->head + 1 + elix) & (fq->nelts - 1));
1646  if (1 || elt->valid)
1647  {
1648  fqt->n_vectors[elix] = elt->n_vectors;
1649  }
1650  }
1651  fqt->written = 1;
1652  }
1653 
1654  while (1)
1655  {
1656  if (fq->head == fq->tail)
1657  {
1658  fq->head_hint = fq->head;
1659  return processed;
1660  }
1661 
1662  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
1663 
1664  if (!elt->valid)
1665  {
1666  fq->head_hint = fq->head;
1667  return processed;
1668  }
1669 
1670  from = elt->buffer_index;
1671  msg_type = elt->msg_type;
1672 
1674  ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE);
1675 
1676  f = vlib_get_frame_to_node (vm, fqm->node_index);
1677 
1678  to = vlib_frame_vector_args (f);
1679 
1680  n_left_to_node = elt->n_vectors;
1681 
1682  while (n_left_to_node >= 4)
1683  {
1684  to[0] = from[0];
1685  to[1] = from[1];
1686  to[2] = from[2];
1687  to[3] = from[3];
1688  to += 4;
1689  from += 4;
1690  n_left_to_node -= 4;
1691  }
1692 
1693  while (n_left_to_node > 0)
1694  {
1695  to[0] = from[0];
1696  to++;
1697  from++;
1698  n_left_to_node--;
1699  }
1700 
1701  vectors += elt->n_vectors;
1702  f->n_vectors = elt->n_vectors;
1703  vlib_put_frame_to_node (vm, fqm->node_index, f);
1704 
1705  elt->valid = 0;
1706  elt->n_vectors = 0;
1707  elt->msg_type = 0xfefefefe;
1709  fq->head++;
1710  processed++;
1711 
1712  /*
1713  * Limit the number of packets pushed into the graph
1714  */
1715  if (vectors >= fq->vector_threshold)
1716  {
1717  fq->head_hint = fq->head;
1718  return processed;
1719  }
1720  }
1721  ASSERT (0);
1722  return processed;
1723 }
1724 
1725 void
1727 {
1730  vlib_main_t *vm = vlib_get_main ();
1731  clib_error_t *e;
1732 
1734 
1736  clib_time_init (&vm->clib_time);
1738 
1739  /* Wait until the dpdk init sequence is complete */
1740  while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
1742 
1744  (vm, vm->worker_init_function_registrations, 1 /* call_once */ );
1745  if (e)
1746  clib_error_report (e);
1747 
1748  vlib_worker_loop (vm);
1749 }
1750 
1751 /* *INDENT-OFF* */
1752 VLIB_REGISTER_THREAD (worker_thread_reg, static) = {
1753  .name = "workers",
1754  .short_name = "wk",
1755  .function = vlib_worker_thread_fn,
1756 };
1757 /* *INDENT-ON* */
1758 
1759 u32
1760 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts)
1761 {
1764  vlib_frame_queue_t *fq;
1765  int i;
1766 
1767  if (frame_queue_nelts == 0)
1768  frame_queue_nelts = FRAME_QUEUE_NELTS;
1769 
1770  ASSERT (frame_queue_nelts >= 8);
1771 
1772  vec_add2 (tm->frame_queue_mains, fqm, 1);
1773 
1774  fqm->node_index = node_index;
1775  fqm->frame_queue_nelts = frame_queue_nelts;
1776  fqm->queue_hi_thresh = frame_queue_nelts - 2;
1777 
1779  vec_validate (fqm->per_thread_data, tm->n_vlib_mains - 1);
1780  _vec_len (fqm->vlib_frame_queues) = 0;
1781  for (i = 0; i < tm->n_vlib_mains; i++)
1782  {
1784  fq = vlib_frame_queue_alloc (frame_queue_nelts);
1785  vec_add1 (fqm->vlib_frame_queues, fq);
1786 
1787  ptd = vec_elt_at_index (fqm->per_thread_data, i);
1789  tm->n_vlib_mains - 1);
1791  tm->n_vlib_mains - 1,
1792  (vlib_frame_queue_t *) (~0));
1793  }
1794 
1795  return (fqm - tm->frame_queue_mains);
1796 }
1797 
1798 int
1800 {
1802 
1803  if (tm->extern_thread_mgmt)
1804  return -1;
1805 
1807  tm->extern_thread_mgmt = 1;
1808  return 0;
1809 }
1810 
1811 void
1813  args)
1814 {
1815  ASSERT (vlib_get_thread_index () == 0);
1817  args->type_opaque, args->data);
1818 }
1819 
1821 
1822 void
1823 vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size)
1824 {
1826  {
1827  void (*fp) (void *, u8 *, u32) = rpc_call_main_thread_cb_fn;
1828  (*fp) (callback, args, arg_size);
1829  }
1830  else
1831  clib_warning ("BUG: rpc_call_main_thread_cb_fn NULL!");
1832 }
1833 
1834 clib_error_t *
1836 {
1837  return 0;
1838 }
1839 
1841 
1842 
1843 static clib_error_t *
1845  unformat_input_t * input, vlib_cli_command_t * cmd)
1846 {
1847  int i;
1848  f64 now;
1849 
1850  now = vlib_time_now (vm);
1851 
1852  vlib_cli_output (vm, "Time now %.9f", now);
1853 
1854  if (vec_len (vlib_mains) == 1)
1855  return 0;
1856 
1857  vlib_cli_output (vm, "Time last barrier release %.9f",
1859 
1860  for (i = 1; i < vec_len (vlib_mains); i++)
1861  {
1862  if (vlib_mains[i] == 0)
1863  continue;
1864  vlib_cli_output (vm, "Thread %d offset %.9f error %.9f", i,
1865  vlib_mains[i]->time_offset,
1868  }
1869  return 0;
1870 }
1871 
1872 /* *INDENT-OFF* */
1873 VLIB_CLI_COMMAND (f_command, static) =
1874 {
1875  .path = "show clock",
1876  .short_help = "show clock",
1877  .function = show_clock_command_fn,
1878 };
1879 /* *INDENT-ON* */
1880 
1881 /*
1882  * fd.io coding-style-patch-verification: ON
1883  *
1884  * Local Variables:
1885  * eval: (c-set-style "gnu")
1886  * End:
1887  */
_vlib_init_function_list_elt_t * worker_init_function_registrations
Definition: main.h:203
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
static void barrier_trace_release_rec(f64 t_entry)
Definition: threads.c:120
static void barrier_trace_sync(f64 t_entry, f64 t_open, f64 t_closed)
Definition: threads.c:69
u32 vl(void *p)
Definition: threads.c:31
static f64 vlib_last_vectors_per_main_loop_as_f64(vlib_main_t *vm)
Definition: main.h:337
uword * pending_rpc_requests
Definition: main.h:243
vlib_main_t vlib_global_main
Definition: main.c:1931
#define clib_min(x, y)
Definition: clib.h:295
clib_spinlock_t pending_rpc_lock
Definition: main.h:245
vlib_process_t ** processes
Definition: node.h:739
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:460
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:522
int vlib_frame_queue_enqueue(vlib_main_t *vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t *frame, vlib_frame_queue_msg_type_t type)
#define clib_atomic_add_fetch(a, b)
Definition: atomics.h:30
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x)
Definition: init.h:166
word elog_track_register(elog_main_t *em, elog_track_t *t)
register an event track
Definition: elog.c:198
unsigned long u64
Definition: types.h:89
void * mheap_alloc(void *memory, uword size)
Definition: mheap.c:963
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:118
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
#define NULL
Definition: clib.h:58
u32 index
Definition: node.h:279
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:255
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:572
clib_error_t * threads_init(vlib_main_t *vm)
Definition: threads.c:1835
void os_panic(void)
Definition: unix-misc.c:174
u32 vlib_frame_queue_main_init(u32 node_index, u32 frame_queue_nelts)
Definition: threads.c:1760
u32 thread_index
Definition: main.h:197
void * thread_function_arg
Definition: threads.h:98
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static int sort_registrations_by_no_clone(void *a0, void *a1)
Definition: threads.c:197
static u64 clib_cpu_time_now(void)
Definition: time.h:75
frame_queue_trace_t * frame_queue_traces
Definition: threads.h:165
void vlib_process_signal_event_mt_helper(vlib_process_signal_event_mt_args_t *args)
Definition: threads.c:1812
elog_track_t elog_track
Definition: threads.h:100
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:560
int i
void vnet_main_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1342
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
#define hash_set_mem(h, key, value)
Definition: hash.h:275
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
clib_time_t clib_time
Definition: main.h:72
void vlib_worker_thread_fn(void *arg)
Definition: threads.c:1726
u32 unformat_sched_policy(unformat_input_t *input, va_list *args)
Definition: threads.c:1213
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
struct vlib_thread_registration_ * next
Definition: threads.h:31
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:82
void * runtime_data
Definition: node.h:285
volatile u32 valid
Definition: threads.h:76
void vlib_worker_thread_barrier_sync_int(vlib_main_t *vm, const char *func_name)
Definition: threads.c:1385
vlib_main_t ** vlib_mains
Definition: buffer.c:321
static void vlib_get_thread_core_socket(vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:607
unsigned char u8
Definition: types.h:56
#define clib_bitmap_dup(v)
Duplicate a bitmap.
Definition: bitmap.h:87
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
u8 state
Definition: node.h:307
u16 thread_index
thread this node runs on
Definition: node.h:520
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
#define clib_memcpy(d, s, n)
Definition: string.h:180
u64 * counters_last_clear
Definition: error.h:81
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:390
vlib_thread_registration_t * next
Definition: threads.h:283
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
Definition: vec.h:531
#define VLIB_NODE_RUNTIME_DATA_SIZE
Definition: node.h:533
vlib_node_stats_t stats_last_clear
Definition: node.h:273
void vlib_worker_thread_node_runtime_update(void)
Definition: threads.c:1203
u64 count[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:783
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:373
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:311
void * heap_aligned_base
Definition: main.h:123
vlib_frame_queue_msg_type_t
Definition: threads.h:68
vlib_node_t ** nodes
Definition: node.h:698
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
Definition: threads.h:151
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:189
#define clib_error_return(e, args...)
Definition: error.h:99
#define VLIB_FRAME_ALIGN
Definition: node.h:377
uword * lock
SMP lock, non-zero means locking required.
Definition: elog.h:172
uword * cpu_core_bitmap
Definition: threads.h:320
#define BARRIER_MINIMUM_OPEN_FACTOR
Definition: threads.c:1381
clib_error_t * vlib_call_init_exit_functions(vlib_main_t *vm, _vlib_init_function_list_elt_t *head, int call_once)
Definition: init.c:43
vlib_frame_queue_elt_t * elts
Definition: threads.h:144
pthread_t thread[MAX_CONNS]
Definition: main.c:142
f64 time_last_barrier_release
Definition: main.h:75
unsigned int u32
Definition: types.h:88
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:708
#define VLIB_FRAME_SIZE
Definition: node.h:376
void vlib_set_thread_name(char *name)
Definition: threads.c:182
void vl_msg_api_handler_no_free(void *)
Definition: threads.c:425
#define hash_create_string(elts, value_bytes)
Definition: hash.h:690
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:1023
vlib_fork_fixup_t
Definition: threads.h:227
#define BARRIER_SYNC_TIMEOUT
Definition: threads.h:199
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
void * rpc_call_main_thread_cb_fn
Definition: threads.c:1820
VLIB_REGISTER_THREAD(worker_thread_reg, static)
int extern_thread_mgmt
Definition: threads.h:339
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:36
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
void * thread_mheap
Definition: threads.h:95
u32 next_frame_index
Definition: node.h:457
vlib_node_stats_t stats_total
Definition: node.h:269
volatile u64 head
Definition: threads.h:131
u16 state
Input node state.
Definition: node.h:510
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:964
static uword clib_bitmap_first_set(uword *ai)
Return the lowest numbered set bit in a bitmap.
Definition: bitmap.h:385
u8 * vlib_thread_stack_init(uword thread_index)
Definition: main.c:624
static void vlib_next_frame_init(vlib_next_frame_t *nf)
Definition: node.h:440
vlib_error_main_t error_main
Definition: main.h:158
static u32 vlib_frame_index(vlib_main_t *vm, vlib_frame_t *f)
Definition: node_funcs.h:252
vlib_thread_callbacks_t cb
Definition: threads.h:338
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:413
int vlib_thread_cb_register(struct vlib_main_t *vm, vlib_thread_callbacks_t *cb)
Definition: threads.c:1799
struct _unformat_input_t unformat_input_t
const char * barrier_context
Definition: threads.h:107
char * name
Track name vector.
Definition: elog.h:115
#define clib_error_return_unix(e, args...)
Definition: error.h:102
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:198
static void * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:64
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:218
#define ELOG_DATA(em, f)
Definition: elog.h:481
#define PREDICT_FALSE(x)
Definition: clib.h:111
static clib_error_t * vlib_launch_thread_int(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:627
void vlib_worker_thread_node_refork(void)
Definition: threads.c:1021
clib_error_t *(* vlib_thread_set_lcore_cb)(u32 thread, u16 cpu)
Definition: threads.h:277
u32 node_index
Node index.
Definition: node.h:495
u32 elog_global_id_for_msg_name(const char *msg_name)
Definition: threads.c:47
uword * init_functions_called
Definition: main.h:194
void clib_time_init(clib_time_t *c)
Definition: time.c:178
uword * frame_size_hash
Definition: node.h:754
u8 name[64]
Definition: memclnt.api:152
vlib_thread_main_t vlib_thread_main
Definition: threads.c:37
clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
Definition: sysfs.c:50
word fformat(FILE *f, char *fmt,...)
Definition: format.c:462
void(* thread_function)(void *)
Definition: threads.h:97
static clib_error_t * cpu_config(vlib_main_t *vm, unformat_input_t *input)
Definition: threads.c:1227
u8 len
Definition: ip_types.api:49
i32 n_vectors[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:778
u64 * counters
Definition: error.h:78
u32 owner_node_index
Definition: node.h:353
vlib_frame_queue_t * vlib_frame_queue_alloc(int nelts)
Definition: threads.c:391
volatile u64 tail
Definition: threads.h:123
#define clib_mem_alloc_no_fail(size)
Definition: mem.h:176
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:216
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
svmdb_client_t * c
u16 n_vectors
Definition: node.h:395
vlib_frame_queue_t ** vlib_frame_queues
Definition: threads.h:161
DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked)
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:212
vlib_main_t * vm
Definition: buffer.c:312
u32 node_runtime_index
Definition: node.h:451
vlib_pending_frame_t * pending_frames
Definition: node.h:724
vlib_thread_function_t * function
Definition: threads.h:36
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1598
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
void * heap_base
Definition: main.h:120
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:261
#define clib_warning(format, args...)
Definition: error.h:59
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
elog_main_t elog_main
Definition: main.h:172
frame_queue_nelt_counter_t * frame_queue_histogram
Definition: threads.h:166
#define VLIB_FRAME_PENDING
Definition: node.h:427
static uword clib_bitmap_get(uword *ai, uword i)
Gets the ith bit value from a bitmap.
Definition: bitmap.h:197
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:439
void vlib_worker_thread_init(vlib_worker_thread_t *w)
Definition: threads.c:552
uword os_get_nthreads(void)
Definition: threads.c:170
static void * clib_mem_get_heap(void)
Definition: mem.h:255
volatile u32 * wait_at_barrier
Definition: threads.h:90
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:155
#define FRAME_QUEUE_NELTS
Definition: threads.c:28
vlib_frame_queue_per_thread_data_t * per_thread_data
Definition: threads.h:162
void vlib_stat_segment_unlock(void)
Definition: threads.c:1482
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
Definition: threads.h:152
#define never_inline
Definition: clib.h:95
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define ASSERT(truth)
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:551
static clib_error_t * show_clock_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: threads.c:1844
static void barrier_trace_sync_rec(f64 t_entry)
Definition: threads.c:96
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:326
u16 flags
Definition: node.h:386
static void clib_mem_free(void *p)
Definition: mem.h:205
#define clib_error_report(e)
Definition: error.h:113
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
size_t count
Definition: vapi.c:47
int need_vlib_worker_thread_node_runtime_update
Definition: main.h:228
uword * thread_registrations_by_name
Definition: threads.h:288
#define BARRIER_MINIMUM_OPEN_LIMIT
Definition: threads.c:1377
clib_error_t *(* vlib_launch_thread_cb)(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.h:275
volatile u32 * node_reforks_required
Definition: threads.h:108
const char * barrier_caller
Definition: threads.h:106
#define clib_max(x, y)
Definition: clib.h:288
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
#define clib_atomic_fetch_add(a, b)
Definition: atomics.h:23
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:593
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:462
static void barrier_trace_release(f64 t_entry, f64 t_closed_total, f64 t_update_main)
Definition: threads.c:142
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1926
#define vec_dup_aligned(V, A)
Return copy of vector (no header, alignment specified).
Definition: vec.h:384
u32 elog_string(elog_main_t *em, char *fmt,...)
add a string to the event-log string table
Definition: elog.c:561
f64 barrier_no_close_before
Definition: main.h:237
static clib_error_t * start_workers(vlib_main_t *vm)
Definition: threads.c:654
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void vlib_rpc_call_main_thread(void *callback, u8 *args, u32 arg_size)
Definition: threads.c:1823
DECLARE_CJ_GLOBAL_LOG
Definition: threads.c:26
vlib_node_main_t node_main
Definition: main.h:135
u64 uword
Definition: types.h:112
vlib_next_frame_t * next_frames
Definition: node.h:721
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
Definition: vec.h:980
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:274
volatile u64 head_hint
Definition: threads.h:140
#define VLIB_THREAD_STACK_SIZE
Definition: threads.h:66
f64 barrier_epoch
Definition: main.h:234
vlib_frame_size_t * frame_sizes
Definition: node.h:757
#define hash_get_mem(h, key)
Definition: hash.h:269
static void worker_thread_node_runtime_update_internal(void)
Definition: threads.c:969
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:140
volatile u32 * workers_at_barrier
Definition: threads.h:91
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
static uword * clib_sysfs_list_to_bitmap(char *filename)
Definition: threads.c:207
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1487
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static f64 vlib_time_now_ticks(vlib_main_t *vm, u64 n)
Definition: main.h:261
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
void vlib_stat_segment_lock(void)
Definition: threads.c:1476
#define vec_foreach(var, vec)
Vector iterator.
void * vlib_worker_thread_bootstrap_fn(void *arg)
Definition: threads.c:589
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:115
u32 node_runtime_index
Definition: node.h:407
uword * cpu_socket_bitmap
Definition: threads.h:323
#define foreach_sched_policy
Definition: threads.h:258
vlib_thread_registration_t ** registrations
Definition: threads.h:286
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:486
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u8 ** vlib_thread_stacks
Definition: main.c:607
pthread_t thread_id
Definition: threads.h:114
vlib_thread_registration_t * registration
Definition: threads.h:102
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:725
volatile u32 worker_thread_release
Definition: threads.h:329
void vlib_worker_thread_fork_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1347
clib_random_buffer_t random_buffer
Definition: main.h:191
u8 runtime_data_bytes
Definition: node.h:310
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:430
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:237