FD.io VPP  v19.04.4-rc0-5-ge88582fac
Vector Packet Processing
threads.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #define _GNU_SOURCE
16 
17 #include <signal.h>
18 #include <math.h>
19 #include <vppinfra/format.h>
20 #include <vppinfra/linux/sysfs.h>
21 #include <vlib/vlib.h>
22 
23 #include <vlib/threads.h>
24 #include <vlib/unix/cj.h>
25 
27 
28 #define FRAME_QUEUE_NELTS 64
29 
30 u32
31 vl (void *p)
32 {
33  return vec_len (p);
34 }
35 
38 
39 /*
40  * Barrier tracing can be enabled on a normal build to collect information
41  * on barrier use, including timings and call stacks. Deliberately not
42  * keyed off CLIB_DEBUG, because that can add significant overhead which
43  * imapacts observed timings.
44  */
45 
46 u32
47 elog_global_id_for_msg_name (const char *msg_name)
48 {
49  uword *p, r;
50  static uword *h;
51  u8 *name_copy;
52 
53  if (!h)
54  h = hash_create_string (0, sizeof (uword));
55 
56  p = hash_get_mem (h, msg_name);
57  if (p)
58  return p[0];
59  r = elog_string (&vlib_global_main.elog_main, "%s", msg_name);
60 
61  name_copy = format (0, "%s%c", msg_name, 0);
62 
63  hash_set_mem (h, name_copy, r);
64 
65  return r;
66 }
67 
68 static inline void
69 barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed)
70 {
71  if (!vlib_worker_threads->barrier_elog_enabled)
72  return;
73 
74  /* *INDENT-OFF* */
75  ELOG_TYPE_DECLARE (e) =
76  {
77  .format = "bar-trace-%s-#%d",
78  .format_args = "T4i4",
79  };
80  /* *INDENT-ON* */
81  struct
82  {
83  u32 caller, count, t_entry, t_open, t_closed;
84  } *ed = 0;
85 
87  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
88  ed->caller = elog_global_id_for_msg_name
89  (vlib_worker_threads[0].barrier_caller);
90  ed->t_entry = (int) (1000000.0 * t_entry);
91  ed->t_open = (int) (1000000.0 * t_open);
92  ed->t_closed = (int) (1000000.0 * t_closed);
93 }
94 
95 static inline void
97 {
98  if (!vlib_worker_threads->barrier_elog_enabled)
99  return;
100 
101  /* *INDENT-OFF* */
102  ELOG_TYPE_DECLARE (e) =
103  {
104  .format = "bar-syncrec-%s-#%d",
105  .format_args = "T4i4",
106  };
107  /* *INDENT-ON* */
108  struct
109  {
110  u32 caller, depth;
111  } *ed = 0;
112 
114  ed->depth = (int) vlib_worker_threads[0].recursion_level - 1;
115  ed->caller = elog_global_id_for_msg_name
116  (vlib_worker_threads[0].barrier_caller);
117 }
118 
119 static inline void
121 {
122  if (!vlib_worker_threads->barrier_elog_enabled)
123  return;
124 
125  /* *INDENT-OFF* */
126  ELOG_TYPE_DECLARE (e) =
127  {
128  .format = "bar-relrrec-#%d",
129  .format_args = "i4",
130  };
131  /* *INDENT-ON* */
132  struct
133  {
134  u32 depth;
135  } *ed = 0;
136 
138  ed->depth = (int) vlib_worker_threads[0].recursion_level;
139 }
140 
141 static inline void
142 barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main)
143 {
144  if (!vlib_worker_threads->barrier_elog_enabled)
145  return;
146 
147  /* *INDENT-OFF* */
148  ELOG_TYPE_DECLARE (e) =
149  {
150  .format = "bar-rel-#%d-e%d-u%d-t%d",
151  .format_args = "i4i4i4i4",
152  };
153  /* *INDENT-ON* */
154  struct
155  {
156  u32 count, t_entry, t_update_main, t_closed_total;
157  } *ed = 0;
158 
160  ed->t_entry = (int) (1000000.0 * t_entry);
161  ed->t_update_main = (int) (1000000.0 * t_update_main);
162  ed->t_closed_total = (int) (1000000.0 * t_closed_total);
163  ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
164 
165  /* Reset context for next trace */
166  vlib_worker_threads[0].barrier_context = NULL;
167 }
168 
169 uword
171 {
172  u32 len;
173 
174  len = vec_len (vlib_thread_stacks);
175  if (len == 0)
176  return 1;
177  else
178  return len;
179 }
180 
181 void
183 {
184  int pthread_setname_np (pthread_t __target_thread, const char *__name);
185  int rv;
186  pthread_t thread = pthread_self ();
187 
188  if (thread)
189  {
190  rv = pthread_setname_np (thread, name);
191  if (rv)
192  clib_warning ("pthread_setname_np returned %d", rv);
193  }
194 }
195 
196 static int
197 sort_registrations_by_no_clone (void *a0, void *a1)
198 {
199  vlib_thread_registration_t **tr0 = a0;
200  vlib_thread_registration_t **tr1 = a1;
201 
202  return ((i32) ((*tr0)->no_data_structure_clone)
203  - ((i32) ((*tr1)->no_data_structure_clone)));
204 }
205 
206 static uword *
208 {
209  FILE *fp;
210  uword *r = 0;
211 
212  fp = fopen (filename, "r");
213 
214  if (fp != NULL)
215  {
216  u8 *buffer = 0;
217  vec_validate (buffer, 256 - 1);
218  if (fgets ((char *) buffer, 256, fp))
219  {
220  unformat_input_t in;
221  unformat_init_string (&in, (char *) buffer,
222  strlen ((char *) buffer));
223  if (unformat (&in, "%U", unformat_bitmap_list, &r) != 1)
224  clib_warning ("unformat_bitmap_list failed");
225  unformat_free (&in);
226  }
227  vec_free (buffer);
228  fclose (fp);
229  }
230  return r;
231 }
232 
233 
234 /* Called early in the init sequence */
235 
236 clib_error_t *
238 {
242  u32 n_vlib_mains = 1;
243  u32 first_index = 1;
244  u32 i;
245  uword *avail_cpu;
246 
247  /* get bitmaps of active cpu cores and sockets */
248  tm->cpu_core_bitmap =
249  clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online");
250  tm->cpu_socket_bitmap =
251  clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online");
252 
253  avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
254 
255  /* skip cores */
256  for (i = 0; i < tm->skip_cores; i++)
257  {
258  uword c = clib_bitmap_first_set (avail_cpu);
259  if (c == ~0)
260  return clib_error_return (0, "no available cpus to skip");
261 
262  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
263  }
264 
265  /* grab cpu for main thread */
266  if (tm->main_lcore == ~0)
267  {
268  /* if main-lcore is not set, we try to use lcore 1 */
269  if (clib_bitmap_get (avail_cpu, 1))
270  tm->main_lcore = 1;
271  else
272  tm->main_lcore = clib_bitmap_first_set (avail_cpu);
273  if (tm->main_lcore == (u8) ~ 0)
274  return clib_error_return (0, "no available cpus to be used for the"
275  " main thread");
276  }
277  else
278  {
279  if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0)
280  return clib_error_return (0, "cpu %u is not available to be used"
281  " for the main thread", tm->main_lcore);
282  }
283  avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
284 
285  /* assume that there is socket 0 only if there is no data from sysfs */
286  if (!tm->cpu_socket_bitmap)
287  tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1);
288 
289  /* pin main thread to main_lcore */
291  {
293  }
294  else
295  {
296  cpu_set_t cpuset;
297  CPU_ZERO (&cpuset);
298  CPU_SET (tm->main_lcore, &cpuset);
299  pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset);
300  }
301 
302  /* as many threads as stacks... */
303  vec_validate_aligned (vlib_worker_threads, vec_len (vlib_thread_stacks) - 1,
305 
306  /* Preallocate thread 0 */
307  _vec_len (vlib_worker_threads) = 1;
311  w->cpu_id = tm->main_lcore;
312  w->lwp = syscall (SYS_gettid);
313  w->thread_id = pthread_self ();
314  tm->n_vlib_mains = 1;
315 
316  if (tm->sched_policy != ~0)
317  {
318  struct sched_param sched_param;
319  if (!sched_getparam (w->lwp, &sched_param))
320  {
321  if (tm->sched_priority != ~0)
322  sched_param.sched_priority = tm->sched_priority;
323  sched_setscheduler (w->lwp, tm->sched_policy, &sched_param);
324  }
325  }
326 
327  /* assign threads to cores and set n_vlib_mains */
328  tr = tm->next;
329 
330  while (tr)
331  {
332  vec_add1 (tm->registrations, tr);
333  tr = tr->next;
334  }
335 
337 
338  for (i = 0; i < vec_len (tm->registrations); i++)
339  {
340  int j;
341  tr = tm->registrations[i];
342  tr->first_index = first_index;
343  first_index += tr->count;
344  n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0;
345 
346  /* construct coremask */
347  if (tr->use_pthreads || !tr->count)
348  continue;
349 
350  if (tr->coremask)
351  {
352  uword c;
353  /* *INDENT-OFF* */
354  clib_bitmap_foreach (c, tr->coremask, ({
355  if (clib_bitmap_get(avail_cpu, c) == 0)
356  return clib_error_return (0, "cpu %u is not available to be used"
357  " for the '%s' thread",c, tr->name);
358 
359  avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
360  }));
361 /* *INDENT-ON* */
362 
363  }
364  else
365  {
366  for (j = 0; j < tr->count; j++)
367  {
368  uword c = clib_bitmap_first_set (avail_cpu);
369  if (c == ~0)
370  return clib_error_return (0,
371  "no available cpus to be used for"
372  " the '%s' thread", tr->name);
373 
374  avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
375  tr->coremask = clib_bitmap_set (tr->coremask, c, 1);
376  }
377  }
378  }
379 
380  clib_bitmap_free (avail_cpu);
381 
382  tm->n_vlib_mains = n_vlib_mains;
383 
384  vec_validate_aligned (vlib_worker_threads, first_index - 1,
386 
387  return 0;
388 }
389 
392 {
393  vlib_frame_queue_t *fq;
394 
395  fq = clib_mem_alloc_aligned (sizeof (*fq), CLIB_CACHE_LINE_BYTES);
396  clib_memset (fq, 0, sizeof (*fq));
397  fq->nelts = nelts;
398  fq->vector_threshold = 128; // packets
400 
401  if (1)
402  {
403  if (((uword) & fq->tail) & (CLIB_CACHE_LINE_BYTES - 1))
404  fformat (stderr, "WARNING: fq->tail unaligned\n");
405  if (((uword) & fq->head) & (CLIB_CACHE_LINE_BYTES - 1))
406  fformat (stderr, "WARNING: fq->head unaligned\n");
407  if (((uword) fq->elts) & (CLIB_CACHE_LINE_BYTES - 1))
408  fformat (stderr, "WARNING: fq->elts unaligned\n");
409 
410  if (sizeof (fq->elts[0]) % CLIB_CACHE_LINE_BYTES)
411  fformat (stderr, "WARNING: fq->elts[0] size %d\n",
412  sizeof (fq->elts[0]));
413  if (nelts & (nelts - 1))
414  {
415  fformat (stderr, "FATAL: nelts MUST be a power of 2\n");
416  abort ();
417  }
418  }
419 
420  return (fq);
421 }
422 
423 void vl_msg_api_handler_no_free (void *) __attribute__ ((weak));
424 void
426 {
427 }
428 
429 /* Turned off, save as reference material... */
430 #if 0
431 static inline int
432 vlib_frame_queue_dequeue_internal (int thread_id,
433  vlib_main_t * vm, vlib_node_main_t * nm)
434 {
435  vlib_frame_queue_t *fq = vlib_frame_queues[thread_id];
437  vlib_frame_t *f;
440  u32 node_runtime_index;
441  int msg_type;
442  u64 before;
443  int processed = 0;
444 
445  ASSERT (vm == vlib_mains[thread_id]);
446 
447  while (1)
448  {
449  if (fq->head == fq->tail)
450  return processed;
451 
452  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
453 
454  if (!elt->valid)
455  return processed;
456 
457  before = clib_cpu_time_now ();
458 
459  f = elt->frame;
460  node_runtime_index = elt->node_runtime_index;
461  msg_type = elt->msg_type;
462 
463  switch (msg_type)
464  {
465  case VLIB_FRAME_QUEUE_ELT_FREE_BUFFERS:
467  /* note fallthrough... */
468  case VLIB_FRAME_QUEUE_ELT_FREE_FRAME:
470  node_runtime_index);
471  vlib_frame_free (vm, r, f);
472  break;
474  vec_add2 (vm->node_main.pending_frames, p, 1);
476  p->node_runtime_index = elt->node_runtime_index;
477  p->frame_index = vlib_frame_index (vm, f);
479  fq->dequeue_vectors += (u64) f->n_vectors;
480  break;
481  case VLIB_FRAME_QUEUE_ELT_API_MSG:
483  break;
484  default:
485  clib_warning ("bogus frame queue message, type %d", msg_type);
486  break;
487  }
488  elt->valid = 0;
489  fq->dequeues++;
490  fq->dequeue_ticks += clib_cpu_time_now () - before;
492  fq->head++;
493  processed++;
494  }
495  ASSERT (0);
496  return processed;
497 }
498 
499 int
500 vlib_frame_queue_dequeue (int thread_id,
501  vlib_main_t * vm, vlib_node_main_t * nm)
502 {
503  return vlib_frame_queue_dequeue_internal (thread_id, vm, nm);
504 }
505 
506 int
507 vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index,
508  u32 frame_queue_index, vlib_frame_t * frame,
510 {
511  vlib_frame_queue_t *fq = vlib_frame_queues[frame_queue_index];
513  u32 save_count;
514  u64 new_tail;
515  u64 before = clib_cpu_time_now ();
516 
517  ASSERT (fq);
518 
519  new_tail = clib_atomic_add_fetch (&fq->tail, 1);
520 
521  /* Wait until a ring slot is available */
522  while (new_tail >= fq->head + fq->nelts)
523  {
524  f64 b4 = vlib_time_now_ticks (vm, before);
526  /* Bad idea. Dequeue -> enqueue -> dequeue -> trouble */
527  // vlib_frame_queue_dequeue (vm->thread_index, vm, nm);
528  }
529 
530  elt = fq->elts + (new_tail & (fq->nelts - 1));
531 
532  /* this would be very bad... */
533  while (elt->valid)
534  {
535  }
536 
537  /* Once we enqueue the frame, frame->n_vectors is owned elsewhere... */
538  save_count = frame->n_vectors;
539 
540  elt->frame = frame;
541  elt->node_runtime_index = node_runtime_index;
542  elt->msg_type = type;
544  elt->valid = 1;
545 
546  return save_count;
547 }
548 #endif /* 0 */
549 
550 /* To be called by vlib worker threads upon startup */
551 void
553 {
555 
556  /*
557  * Note: disabling signals in worker threads as follows
558  * prevents the api post-mortem dump scheme from working
559  * {
560  * sigset_t s;
561  * sigfillset (&s);
562  * pthread_sigmask (SIG_SETMASK, &s, 0);
563  * }
564  */
565 
567 
568  if (vec_len (tm->thread_prefix) && w->registration->short_name)
569  {
570  w->name = format (0, "%v_%s_%d%c", tm->thread_prefix,
571  w->registration->short_name, w->instance_id, '\0');
572  vlib_set_thread_name ((char *) w->name);
573  }
574 
575  if (!w->registration->use_pthreads)
576  {
577 
578  /* Initial barrier sync, for both worker and i/o threads */
579  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
580 
581  while (*vlib_worker_threads->wait_at_barrier)
582  ;
583 
584  clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
585  }
586 }
587 
588 void *
590 {
591  void *rv;
592  vlib_worker_thread_t *w = arg;
593 
594  w->lwp = syscall (SYS_gettid);
595  w->thread_id = pthread_self ();
596 
597  __os_thread_index = w - vlib_worker_threads;
598 
599  rv = (void *) clib_calljmp
600  ((uword (*)(uword)) w->thread_function,
602  /* NOTREACHED, we hope */
603  return rv;
604 }
605 
606 static void
608 {
609  const char *sys_cpu_path = "/sys/devices/system/cpu/cpu";
610  u8 *p = 0;
611  int core_id = -1, socket_id = -1;
612 
613  p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, cpu_id, 0);
614  clib_sysfs_read ((char *) p, "%d", &core_id);
615  vec_reset_length (p);
616  p =
617  format (p, "%s%u/topology/physical_package_id%c", sys_cpu_path, cpu_id,
618  0);
619  clib_sysfs_read ((char *) p, "%d", &socket_id);
620  vec_free (p);
621 
622  w->core_id = core_id;
623  w->socket_id = socket_id;
624 }
625 
626 static clib_error_t *
627 vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
628 {
630  void *(*fp_arg) (void *) = fp;
631 
632  w->cpu_id = cpu_id;
633  vlib_get_thread_core_socket (w, cpu_id);
635  return tm->cb.vlib_launch_thread_cb (fp, (void *) w, cpu_id);
636  else
637  {
638  pthread_t worker;
639  cpu_set_t cpuset;
640  CPU_ZERO (&cpuset);
641  CPU_SET (cpu_id, &cpuset);
642 
643  if (pthread_create (&worker, NULL /* attr */ , fp_arg, (void *) w))
644  return clib_error_return_unix (0, "pthread_create");
645 
646  if (pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset))
647  return clib_error_return_unix (0, "pthread_setaffinity_np");
648 
649  return 0;
650  }
651 }
652 
653 static clib_error_t *
655 {
656  int i, j;
658  vlib_main_t *vm_clone;
659  void *oldheap;
663  u32 n_vlib_mains = tm->n_vlib_mains;
664  u32 worker_thread_index;
665  u8 *main_heap = clib_mem_get_per_cpu_heap ();
666 
667  vec_reset_length (vlib_worker_threads);
668 
669  /* Set up the main thread */
670  vec_add2_aligned (vlib_worker_threads, w, 1, CLIB_CACHE_LINE_BYTES);
671  w->elog_track.name = "main thread";
673 
674  if (vec_len (tm->thread_prefix))
675  {
676  w->name = format (0, "%v_main%c", tm->thread_prefix, '\0');
677  vlib_set_thread_name ((char *) w->name);
678  }
679 
680  vm->elog_main.lock =
682  vm->elog_main.lock[0] = 0;
683 
684  if (n_vlib_mains > 1)
685  {
686  /* Replace hand-crafted length-1 vector with a real vector */
687  vlib_mains = 0;
688 
691  _vec_len (vlib_mains) = 0;
693 
694  vlib_worker_threads->wait_at_barrier =
696  vlib_worker_threads->workers_at_barrier =
698 
699  vlib_worker_threads->node_reforks_required =
701 
702  /* We'll need the rpc vector lock... */
704 
705  /* Ask for an initial barrier sync */
706  *vlib_worker_threads->workers_at_barrier = 0;
707  *vlib_worker_threads->wait_at_barrier = 1;
708 
709  /* Without update or refork */
710  *vlib_worker_threads->node_reforks_required = 0;
712 
713  /* init timing */
714  vm->barrier_epoch = 0;
715  vm->barrier_no_close_before = 0;
716 
717  worker_thread_index = 1;
718 
719  for (i = 0; i < vec_len (tm->registrations); i++)
720  {
721  vlib_node_main_t *nm, *nm_clone;
722  int k;
723 
724  tr = tm->registrations[i];
725 
726  if (tr->count == 0)
727  continue;
728 
729  for (k = 0; k < tr->count; k++)
730  {
731  vlib_node_t *n;
732 
733  vec_add2 (vlib_worker_threads, w, 1);
734  /* Currently unused, may not really work */
735  if (tr->mheap_size)
736  {
737 #if USE_DLMALLOC == 0
738  w->thread_mheap =
739  mheap_alloc (0 /* use VM */ , tr->mheap_size);
740 #else
742  0 /* unlocked */ );
743 #endif
744  }
745  else
746  w->thread_mheap = main_heap;
747 
748  w->thread_stack =
749  vlib_thread_stack_init (w - vlib_worker_threads);
750  w->thread_function = tr->function;
751  w->thread_function_arg = w;
752  w->instance_id = k;
753  w->registration = tr;
754 
755  w->elog_track.name =
756  (char *) format (0, "%s %d", tr->name, k + 1);
757  vec_add1 (w->elog_track.name, 0);
759 
760  if (tr->no_data_structure_clone)
761  continue;
762 
763  /* Fork vlib_global_main et al. Look for bugs here */
764  oldheap = clib_mem_set_heap (w->thread_mheap);
765 
766  vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone),
768  clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone));
769 
770  vm_clone->thread_index = worker_thread_index;
771  vm_clone->heap_base = w->thread_mheap;
772  vm_clone->heap_aligned_base = (void *)
773  (((uword) w->thread_mheap) & ~(VLIB_FRAME_ALIGN - 1));
774  vm_clone->init_functions_called =
775  hash_create (0, /* value bytes */ 0);
776  vm_clone->pending_rpc_requests = 0;
777  vec_validate (vm_clone->pending_rpc_requests, 0);
778  _vec_len (vm_clone->pending_rpc_requests) = 0;
779  clib_memset (&vm_clone->random_buffer, 0,
780  sizeof (vm_clone->random_buffer));
781 
782  nm = &vlib_mains[0]->node_main;
783  nm_clone = &vm_clone->node_main;
784  /* fork next frames array, preserving node runtime indices */
785  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
787  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
788  {
789  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
790  u32 save_node_runtime_index;
791  u32 save_flags;
792 
793  save_node_runtime_index = nf->node_runtime_index;
794  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
796  nf->node_runtime_index = save_node_runtime_index;
797  nf->flags = save_flags;
798  }
799 
800  /* fork the frame dispatch queue */
801  nm_clone->pending_frames = 0;
802  vec_validate (nm_clone->pending_frames, 10);
803  _vec_len (nm_clone->pending_frames) = 0;
804 
805  /* fork nodes */
806  nm_clone->nodes = 0;
807 
808  /* Allocate all nodes in single block for speed */
809  n = clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*n));
810 
811  for (j = 0; j < vec_len (nm->nodes); j++)
812  {
813  clib_memcpy (n, nm->nodes[j], sizeof (*n));
814  /* none of the copied nodes have enqueue rights given out */
816  clib_memset (&n->stats_total, 0, sizeof (n->stats_total));
818  sizeof (n->stats_last_clear));
819  vec_add1 (nm_clone->nodes, n);
820  n++;
821  }
825  vec_foreach (rt,
827  {
828  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
829  rt->thread_index = vm_clone->thread_index;
830  /* copy initial runtime_data from node */
831  if (n->runtime_data && n->runtime_data_bytes > 0)
834  n->runtime_data_bytes));
835  }
836 
841  {
842  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
843  rt->thread_index = vm_clone->thread_index;
844  /* copy initial runtime_data from node */
845  if (n->runtime_data && n->runtime_data_bytes > 0)
848  n->runtime_data_bytes));
849  }
850 
854  vec_foreach (rt,
856  {
857  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
858  rt->thread_index = vm_clone->thread_index;
859  /* copy initial runtime_data from node */
860  if (n->runtime_data && n->runtime_data_bytes > 0)
863  n->runtime_data_bytes));
864  }
865 
866  nm_clone->processes = vec_dup_aligned (nm->processes,
868 
869  /* Create per-thread frame freelist */
870  nm_clone->frame_sizes = vec_new (vlib_frame_size_t, 1);
871 #ifdef VLIB_SUPPORTS_ARBITRARY_SCALAR_SIZES
872  nm_clone->frame_size_hash = hash_create (0, sizeof (uword));
873 #endif
874  nm_clone->node_by_error = nm->node_by_error;
875 
876  /* Packet trace buffers are guaranteed to be empty, nothing to do here */
877 
878  clib_mem_set_heap (oldheap);
880 
882  (vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES);
884  (vlib_mains[0]->error_main.counters_last_clear,
886 
887  worker_thread_index++;
888  }
889  }
890  }
891  else
892  {
893  /* only have non-data-structure copy threads to create... */
894  for (i = 0; i < vec_len (tm->registrations); i++)
895  {
896  tr = tm->registrations[i];
897 
898  for (j = 0; j < tr->count; j++)
899  {
900  vec_add2 (vlib_worker_threads, w, 1);
901  if (tr->mheap_size)
902  {
903 #if USE_DLMALLOC == 0
904  w->thread_mheap =
905  mheap_alloc (0 /* use VM */ , tr->mheap_size);
906 #else
907  w->thread_mheap =
908  create_mspace (tr->mheap_size, 0 /* locked */ );
909 #endif
910  }
911  else
912  w->thread_mheap = main_heap;
913  w->thread_stack =
914  vlib_thread_stack_init (w - vlib_worker_threads);
915  w->thread_function = tr->function;
916  w->thread_function_arg = w;
917  w->instance_id = j;
918  w->elog_track.name =
919  (char *) format (0, "%s %d", tr->name, j + 1);
920  w->registration = tr;
921  vec_add1 (w->elog_track.name, 0);
923  }
924  }
925  }
926 
927  worker_thread_index = 1;
928 
929  for (i = 0; i < vec_len (tm->registrations); i++)
930  {
931  clib_error_t *err;
932  int j;
933 
934  tr = tm->registrations[i];
935 
936  if (tr->use_pthreads || tm->use_pthreads)
937  {
938  for (j = 0; j < tr->count; j++)
939  {
940  w = vlib_worker_threads + worker_thread_index++;
942  w, 0);
943  if (err)
944  clib_error_report (err);
945  }
946  }
947  else
948  {
949  uword c;
950  /* *INDENT-OFF* */
951  clib_bitmap_foreach (c, tr->coremask, ({
952  w = vlib_worker_threads + worker_thread_index++;
953  err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
954  w, c);
955  if (err)
956  clib_error_report (err);
957  }));
958  /* *INDENT-ON* */
959  }
960  }
963  return 0;
964 }
965 
967 
968 
969 static inline void
971 {
972  int i, j;
973  vlib_main_t *vm;
974  vlib_node_main_t *nm, *nm_clone;
975  vlib_main_t *vm_clone;
977  never_inline void
980  uword n_calls,
981  uword n_vectors, uword n_clocks);
982 
983  ASSERT (vlib_get_thread_index () == 0);
984 
985  vm = vlib_mains[0];
986  nm = &vm->node_main;
987 
988  ASSERT (*vlib_worker_threads->wait_at_barrier == 1);
989 
990  /*
991  * Scrape all runtime stats, so we don't lose node runtime(s) with
992  * pending counts, or throw away worker / io thread counts.
993  */
994  for (j = 0; j < vec_len (nm->nodes); j++)
995  {
996  vlib_node_t *n;
997  n = nm->nodes[j];
998  vlib_node_sync_stats (vm, n);
999  }
1000 
1001  for (i = 1; i < vec_len (vlib_mains); i++)
1002  {
1003  vlib_node_t *n;
1004 
1005  vm_clone = vlib_mains[i];
1006  nm_clone = &vm_clone->node_main;
1007 
1008  for (j = 0; j < vec_len (nm_clone->nodes); j++)
1009  {
1010  n = nm_clone->nodes[j];
1011 
1012  rt = vlib_node_get_runtime (vm_clone, n->index);
1013  vlib_node_runtime_sync_stats (vm_clone, rt, 0, 0, 0);
1014  }
1015  }
1016 
1017  /* Per-worker clone rebuilds are now done on each thread */
1018 }
1019 
1020 
1021 void
1023 {
1024  vlib_main_t *vm, *vm_clone;
1025  vlib_node_main_t *nm, *nm_clone;
1026  vlib_node_t **old_nodes_clone;
1027  vlib_node_runtime_t *rt, *old_rt;
1028 
1029  vlib_node_t *new_n_clone;
1030 
1031  int j;
1032 
1033  vm = vlib_mains[0];
1034  nm = &vm->node_main;
1035  vm_clone = vlib_get_main ();
1036  nm_clone = &vm_clone->node_main;
1037 
1038  /* Re-clone error heap */
1039  u64 *old_counters = vm_clone->error_main.counters;
1040  u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear;
1041 
1042  clib_memcpy_fast (&vm_clone->error_main, &vm->error_main,
1043  sizeof (vm->error_main));
1044  j = vec_len (vm->error_main.counters) - 1;
1045  vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES);
1046  vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
1047  vm_clone->error_main.counters = old_counters;
1048  vm_clone->error_main.counters_last_clear = old_counters_all_clear;
1049 
1050  nm_clone = &vm_clone->node_main;
1051  vec_free (nm_clone->next_frames);
1052  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
1054 
1055  for (j = 0; j < vec_len (nm_clone->next_frames); j++)
1056  {
1057  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
1058  u32 save_node_runtime_index;
1059  u32 save_flags;
1060 
1061  save_node_runtime_index = nf->node_runtime_index;
1062  save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
1063  vlib_next_frame_init (nf);
1064  nf->node_runtime_index = save_node_runtime_index;
1065  nf->flags = save_flags;
1066  }
1067 
1068  old_nodes_clone = nm_clone->nodes;
1069  nm_clone->nodes = 0;
1070 
1071  /* re-fork nodes */
1072 
1073  /* Allocate all nodes in single block for speed */
1074  new_n_clone =
1075  clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*new_n_clone));
1076  for (j = 0; j < vec_len (nm->nodes); j++)
1077  {
1078  vlib_node_t *old_n_clone;
1079  vlib_node_t *new_n;
1080 
1081  new_n = nm->nodes[j];
1082  old_n_clone = old_nodes_clone[j];
1083 
1084  clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n));
1085  /* none of the copied nodes have enqueue rights given out */
1087 
1088  if (j >= vec_len (old_nodes_clone))
1089  {
1090  /* new node, set to zero */
1091  clib_memset (&new_n_clone->stats_total, 0,
1092  sizeof (new_n_clone->stats_total));
1093  clib_memset (&new_n_clone->stats_last_clear, 0,
1094  sizeof (new_n_clone->stats_last_clear));
1095  }
1096  else
1097  {
1098  /* Copy stats if the old data is valid */
1099  clib_memcpy_fast (&new_n_clone->stats_total,
1100  &old_n_clone->stats_total,
1101  sizeof (new_n_clone->stats_total));
1102  clib_memcpy_fast (&new_n_clone->stats_last_clear,
1103  &old_n_clone->stats_last_clear,
1104  sizeof (new_n_clone->stats_last_clear));
1105 
1106  /* keep previous node state */
1107  new_n_clone->state = old_n_clone->state;
1108  }
1109  vec_add1 (nm_clone->nodes, new_n_clone);
1110  new_n_clone++;
1111  }
1112  /* Free the old node clones */
1113  clib_mem_free (old_nodes_clone[0]);
1114 
1115  vec_free (old_nodes_clone);
1116 
1117 
1118  /* re-clone internal nodes */
1119  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
1123 
1125  {
1126  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1127  rt->thread_index = vm_clone->thread_index;
1128  /* copy runtime_data, will be overwritten later for existing rt */
1129  if (n->runtime_data && n->runtime_data_bytes > 0)
1132  n->runtime_data_bytes));
1133  }
1134 
1135  for (j = 0; j < vec_len (old_rt); j++)
1136  {
1137  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1138  rt->state = old_rt[j].state;
1139  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1141  }
1142 
1143  vec_free (old_rt);
1144 
1145  /* re-clone input nodes */
1146  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
1147  nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
1150 
1152  {
1153  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1154  rt->thread_index = vm_clone->thread_index;
1155  /* copy runtime_data, will be overwritten later for existing rt */
1156  if (n->runtime_data && n->runtime_data_bytes > 0)
1159  n->runtime_data_bytes));
1160  }
1161 
1162  for (j = 0; j < vec_len (old_rt); j++)
1163  {
1164  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1165  rt->state = old_rt[j].state;
1166  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1168  }
1169 
1170  vec_free (old_rt);
1171 
1172  /* re-clone pre-input nodes */
1173  old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT];
1177 
1179  {
1180  vlib_node_t *n = vlib_get_node (vm, rt->node_index);
1181  rt->thread_index = vm_clone->thread_index;
1182  /* copy runtime_data, will be overwritten later for existing rt */
1183  if (n->runtime_data && n->runtime_data_bytes > 0)
1186  n->runtime_data_bytes));
1187  }
1188 
1189  for (j = 0; j < vec_len (old_rt); j++)
1190  {
1191  rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1192  rt->state = old_rt[j].state;
1193  clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1195  }
1196 
1197  vec_free (old_rt);
1198 
1199  nm_clone->processes = vec_dup_aligned (nm->processes,
1201  nm_clone->node_by_error = nm->node_by_error;
1202 }
1203 
1204 void
1206 {
1207  /*
1208  * Make a note that we need to do a node runtime update
1209  * prior to releasing the barrier.
1210  */
1212 }
1213 
1214 u32
1215 unformat_sched_policy (unformat_input_t * input, va_list * args)
1216 {
1217  u32 *r = va_arg (*args, u32 *);
1218 
1219  if (0);
1220 #define _(v,f,s) else if (unformat (input, s)) *r = SCHED_POLICY_##f;
1222 #undef _
1223  else
1224  return 0;
1225  return 1;
1226 }
1227 
1228 static clib_error_t *
1230 {
1232  uword *p;
1234  u8 *name;
1235  uword *bitmap;
1236  u32 count;
1237 
1239 
1240  tm->n_thread_stacks = 1; /* account for main thread */
1241  tm->sched_policy = ~0;
1242  tm->sched_priority = ~0;
1243  tm->main_lcore = ~0;
1244 
1245  tr = tm->next;
1246 
1247  while (tr)
1248  {
1250  tr = tr->next;
1251  }
1252 
1253  while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
1254  {
1255  if (unformat (input, "use-pthreads"))
1256  tm->use_pthreads = 1;
1257  else if (unformat (input, "thread-prefix %v", &tm->thread_prefix))
1258  ;
1259  else if (unformat (input, "main-core %u", &tm->main_lcore))
1260  ;
1261  else if (unformat (input, "skip-cores %u", &tm->skip_cores))
1262  ;
1263  else if (unformat (input, "coremask-%s %U", &name,
1264  unformat_bitmap_mask, &bitmap) ||
1265  unformat (input, "corelist-%s %U", &name,
1266  unformat_bitmap_list, &bitmap))
1267  {
1269  if (p == 0)
1270  return clib_error_return (0, "no such thread type '%s'", name);
1271 
1272  tr = (vlib_thread_registration_t *) p[0];
1273 
1274  if (tr->use_pthreads)
1275  return clib_error_return (0,
1276  "corelist cannot be set for '%s' threads",
1277  name);
1278 
1279  tr->coremask = bitmap;
1281  }
1282  else
1283  if (unformat
1284  (input, "scheduler-policy %U", unformat_sched_policy,
1285  &tm->sched_policy))
1286  ;
1287  else if (unformat (input, "scheduler-priority %u", &tm->sched_priority))
1288  ;
1289  else if (unformat (input, "%s %u", &name, &count))
1290  {
1292  if (p == 0)
1293  return clib_error_return (0, "no such thread type 3 '%s'", name);
1294 
1295  tr = (vlib_thread_registration_t *) p[0];
1296  if (tr->fixed_count)
1297  return clib_error_return
1298  (0, "number of %s threads not configurable", tr->name);
1299  tr->count = count;
1300  }
1301  else
1302  break;
1303  }
1304 
1305  if (tm->sched_priority != ~0)
1306  {
1307  if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR)
1308  {
1309  u32 prio_max = sched_get_priority_max (tm->sched_policy);
1310  u32 prio_min = sched_get_priority_min (tm->sched_policy);
1311  if (tm->sched_priority > prio_max)
1312  tm->sched_priority = prio_max;
1313  if (tm->sched_priority < prio_min)
1314  tm->sched_priority = prio_min;
1315  }
1316  else
1317  {
1318  return clib_error_return
1319  (0,
1320  "scheduling priority (%d) is not allowed for `normal` scheduling policy",
1321  tm->sched_priority);
1322  }
1323  }
1324  tr = tm->next;
1325 
1326  if (!tm->thread_prefix)
1327  tm->thread_prefix = format (0, "vpp");
1328 
1329  while (tr)
1330  {
1331  tm->n_thread_stacks += tr->count;
1332  tm->n_pthreads += tr->count * tr->use_pthreads;
1333  tm->n_threads += tr->count * (tr->use_pthreads == 0);
1334  tr = tr->next;
1335  }
1336 
1337  return 0;
1338 }
1339 
1341 
1342 void vnet_main_fixup (vlib_fork_fixup_t which) __attribute__ ((weak));
1343 void
1345 {
1346 }
1347 
1348 void
1350 {
1351  vlib_main_t *vm = vlib_get_main ();
1352 
1353  if (vlib_mains == 0)
1354  return;
1355 
1356  ASSERT (vlib_get_thread_index () == 0);
1358 
1359  switch (which)
1360  {
1363  break;
1364 
1365  default:
1366  ASSERT (0);
1367  }
1369 }
1370 
1371  /*
1372  * Enforce minimum open time to minimize packet loss due to Rx overflow,
1373  * based on a test based heuristic that barrier should be open for at least
1374  * 3 time as long as it is closed (with an upper bound of 1ms because by that
1375  * point it is probably too late to make a difference)
1376  */
1377 
1378 #ifndef BARRIER_MINIMUM_OPEN_LIMIT
1379 #define BARRIER_MINIMUM_OPEN_LIMIT 0.001
1380 #endif
1381 
1382 #ifndef BARRIER_MINIMUM_OPEN_FACTOR
1383 #define BARRIER_MINIMUM_OPEN_FACTOR 3
1384 #endif
1385 
1386 void
1388 {
1389  f64 deadline;
1390  f64 now;
1391  f64 t_entry;
1392  f64 t_open;
1393  f64 t_closed;
1394  f64 max_vector_rate;
1395  u32 count;
1396  int i;
1397 
1398  if (vec_len (vlib_mains) < 2)
1399  return;
1400 
1401  ASSERT (vlib_get_thread_index () == 0);
1402 
1403  vlib_worker_threads[0].barrier_caller = func_name;
1404  count = vec_len (vlib_mains) - 1;
1405 
1406  /* Record entry relative to last close */
1407  now = vlib_time_now (vm);
1408  t_entry = now - vm->barrier_epoch;
1409 
1410  /* Tolerate recursive calls */
1411  if (++vlib_worker_threads[0].recursion_level > 1)
1412  {
1413  barrier_trace_sync_rec (t_entry);
1414  return;
1415  }
1416 
1417  /*
1418  * Need data to decide if we're working hard enough to honor
1419  * the barrier hold-down timer.
1420  */
1421  max_vector_rate = 0.0;
1422  for (i = 1; i < vec_len (vlib_mains); i++)
1423  max_vector_rate =
1424  clib_max (max_vector_rate,
1426 
1427  vlib_worker_threads[0].barrier_sync_count++;
1428 
1429  /* Enforce minimum barrier open time to minimize packet loss */
1431 
1432  /*
1433  * If any worker thread seems busy, which we define
1434  * as a vector rate above 10, we enforce the barrier hold-down timer
1435  */
1436  if (max_vector_rate > 10.0)
1437  {
1438  while (1)
1439  {
1440  now = vlib_time_now (vm);
1441  /* Barrier hold-down timer expired? */
1442  if (now >= vm->barrier_no_close_before)
1443  break;
1444  if ((vm->barrier_no_close_before - now)
1445  > (2.0 * BARRIER_MINIMUM_OPEN_LIMIT))
1446  {
1447  clib_warning
1448  ("clock change: would have waited for %.4f seconds",
1449  (vm->barrier_no_close_before - now));
1450  break;
1451  }
1452  }
1453  }
1454  /* Record time of closure */
1455  t_open = now - vm->barrier_epoch;
1456  vm->barrier_epoch = now;
1457 
1458  deadline = now + BARRIER_SYNC_TIMEOUT;
1459 
1460  *vlib_worker_threads->wait_at_barrier = 1;
1461  while (*vlib_worker_threads->workers_at_barrier != count)
1462  {
1463  if ((now = vlib_time_now (vm)) > deadline)
1464  {
1465  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1466  os_panic ();
1467  }
1468  }
1469 
1470  t_closed = now - vm->barrier_epoch;
1471 
1472  barrier_trace_sync (t_entry, t_open, t_closed);
1473 
1474 }
1475 
1476 void vlib_stat_segment_lock (void) __attribute__ ((weak));
1477 void
1479 {
1480 }
1481 
1482 void vlib_stat_segment_unlock (void) __attribute__ ((weak));
1483 void
1485 {
1486 }
1487 
1488 void
1490 {
1491  f64 deadline;
1492  f64 now;
1493  f64 minimum_open;
1494  f64 t_entry;
1495  f64 t_closed_total;
1496  f64 t_update_main = 0.0;
1497  int refork_needed = 0;
1498 
1499  if (vec_len (vlib_mains) < 2)
1500  return;
1501 
1502  ASSERT (vlib_get_thread_index () == 0);
1503 
1504 
1505  now = vlib_time_now (vm);
1506  t_entry = now - vm->barrier_epoch;
1507 
1508  if (--vlib_worker_threads[0].recursion_level > 0)
1509  {
1510  barrier_trace_release_rec (t_entry);
1511  return;
1512  }
1513 
1514  /* Update (all) node runtimes before releasing the barrier, if needed */
1516  {
1517  /*
1518  * Lock stat segment here, so we's safe when
1519  * rebuilding the stat segment node clones from the
1520  * stat thread...
1521  */
1523 
1524  /* Do stats elements on main thread */
1527 
1528  /* Do per thread rebuilds in parallel */
1529  refork_needed = 1;
1530  clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
1531  (vec_len (vlib_mains) - 1));
1532  now = vlib_time_now (vm);
1533  t_update_main = now - vm->barrier_epoch;
1534  }
1535 
1536  deadline = now + BARRIER_SYNC_TIMEOUT;
1537 
1538  /*
1539  * Note when we let go of the barrier.
1540  * Workers can use this to derive a reasonably accurate
1541  * time offset. See vlib_time_now(...)
1542  */
1545 
1546  *vlib_worker_threads->wait_at_barrier = 0;
1547 
1548  while (*vlib_worker_threads->workers_at_barrier > 0)
1549  {
1550  if ((now = vlib_time_now (vm)) > deadline)
1551  {
1552  fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1553  os_panic ();
1554  }
1555  }
1556 
1557  /* Wait for reforks before continuing */
1558  if (refork_needed)
1559  {
1560  now = vlib_time_now (vm);
1561 
1562  deadline = now + BARRIER_SYNC_TIMEOUT;
1563 
1564  while (*vlib_worker_threads->node_reforks_required > 0)
1565  {
1566  if ((now = vlib_time_now (vm)) > deadline)
1567  {
1568  fformat (stderr, "%s: worker thread refork deadlock\n",
1569  __FUNCTION__);
1570  os_panic ();
1571  }
1572  }
1574  }
1575 
1576  t_closed_total = now - vm->barrier_epoch;
1577 
1578  minimum_open = t_closed_total * BARRIER_MINIMUM_OPEN_FACTOR;
1579 
1580  if (minimum_open > BARRIER_MINIMUM_OPEN_LIMIT)
1581  {
1582  minimum_open = BARRIER_MINIMUM_OPEN_LIMIT;
1583  }
1584 
1585  vm->barrier_no_close_before = now + minimum_open;
1586 
1587  /* Record barrier epoch (used to enforce minimum open time) */
1588  vm->barrier_epoch = now;
1589 
1590  barrier_trace_release (t_entry, t_closed_total, t_update_main);
1591 
1592 }
1593 
1594 /*
1595  * Check the frame queue to see if any frames are available.
1596  * If so, pull the packets off the frames and put them to
1597  * the handoff node.
1598  */
1599 int
1601 {
1602  u32 thread_id = vm->thread_index;
1603  vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id];
1605  u32 *from, *to;
1606  vlib_frame_t *f;
1607  int msg_type;
1608  int processed = 0;
1609  u32 n_left_to_node;
1610  u32 vectors = 0;
1611 
1612  ASSERT (fq);
1613  ASSERT (vm == vlib_mains[thread_id]);
1614 
1615  if (PREDICT_FALSE (fqm->node_index == ~0))
1616  return 0;
1617  /*
1618  * Gather trace data for frame queues
1619  */
1620  if (PREDICT_FALSE (fq->trace))
1621  {
1622  frame_queue_trace_t *fqt;
1624  u32 elix;
1625 
1626  fqt = &fqm->frame_queue_traces[thread_id];
1627 
1628  fqt->nelts = fq->nelts;
1629  fqt->head = fq->head;
1630  fqt->head_hint = fq->head_hint;
1631  fqt->tail = fq->tail;
1632  fqt->threshold = fq->vector_threshold;
1633  fqt->n_in_use = fqt->tail - fqt->head;
1634  if (fqt->n_in_use >= fqt->nelts)
1635  {
1636  // if beyond max then use max
1637  fqt->n_in_use = fqt->nelts - 1;
1638  }
1639 
1640  /* Record the number of elements in use in the histogram */
1641  fqh = &fqm->frame_queue_histogram[thread_id];
1642  fqh->count[fqt->n_in_use]++;
1643 
1644  /* Record a snapshot of the elements in use */
1645  for (elix = 0; elix < fqt->nelts; elix++)
1646  {
1647  elt = fq->elts + ((fq->head + 1 + elix) & (fq->nelts - 1));
1648  if (1 || elt->valid)
1649  {
1650  fqt->n_vectors[elix] = elt->n_vectors;
1651  }
1652  }
1653  fqt->written = 1;
1654  }
1655 
1656  while (1)
1657  {
1658  if (fq->head == fq->tail)
1659  {
1660  fq->head_hint = fq->head;
1661  return processed;
1662  }
1663 
1664  elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1));
1665 
1666  if (!elt->valid)
1667  {
1668  fq->head_hint = fq->head;
1669  return processed;
1670  }
1671 
1672  from = elt->buffer_index;
1673  msg_type = elt->msg_type;
1674 
1676  ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE);
1677 
1678  f = vlib_get_frame_to_node (vm, fqm->node_index);
1679 
1680  to = vlib_frame_vector_args (f);
1681 
1682  n_left_to_node = elt->n_vectors;
1683 
1684  while (n_left_to_node >= 4)
1685  {
1686  to[0] = from[0];
1687  to[1] = from[1];
1688  to[2] = from[2];
1689  to[3] = from[3];
1690  to += 4;
1691  from += 4;
1692  n_left_to_node -= 4;
1693  }
1694 
1695  while (n_left_to_node > 0)
1696  {
1697  to[0] = from[0];
1698  to++;
1699  from++;
1700  n_left_to_node--;
1701  }
1702 
1703  vectors += elt->n_vectors;
1704  f->n_vectors = elt->n_vectors;
1705  vlib_put_frame_to_node (vm, fqm->node_index, f);
1706 
1707  elt->valid = 0;
1708  elt->n_vectors = 0;
1709  elt->msg_type = 0xfefefefe;
1711  fq->head++;
1712  processed++;
1713 
1714  /*
1715  * Limit the number of packets pushed into the graph
1716  */
1717  if (vectors >= fq->vector_threshold)
1718  {
1719  fq->head_hint = fq->head;
1720  return processed;
1721  }
1722  }
1723  ASSERT (0);
1724  return processed;
1725 }
1726 
1727 void
1729 {
1732  vlib_main_t *vm = vlib_get_main ();
1733  clib_error_t *e;
1734 
1736 
1738  clib_time_init (&vm->clib_time);
1740 
1741  /* Wait until the dpdk init sequence is complete */
1742  while (tm->extern_thread_mgmt && tm->worker_thread_release == 0)
1744 
1746  (vm, vm->worker_init_function_registrations, 1 /* call_once */ );
1747  if (e)
1748  clib_error_report (e);
1749 
1750  vlib_worker_loop (vm);
1751 }
1752 
1753 /* *INDENT-OFF* */
1754 VLIB_REGISTER_THREAD (worker_thread_reg, static) = {
1755  .name = "workers",
1756  .short_name = "wk",
1757  .function = vlib_worker_thread_fn,
1758 };
1759 /* *INDENT-ON* */
1760 
1761 u32
1762 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts)
1763 {
1766  vlib_frame_queue_t *fq;
1767  int i;
1768 
1769  if (frame_queue_nelts == 0)
1770  frame_queue_nelts = FRAME_QUEUE_NELTS;
1771 
1772  ASSERT (frame_queue_nelts >= 8);
1773 
1774  vec_add2 (tm->frame_queue_mains, fqm, 1);
1775 
1776  fqm->node_index = node_index;
1777  fqm->frame_queue_nelts = frame_queue_nelts;
1778  fqm->queue_hi_thresh = frame_queue_nelts - 2;
1779 
1781  vec_validate (fqm->per_thread_data, tm->n_vlib_mains - 1);
1782  _vec_len (fqm->vlib_frame_queues) = 0;
1783  for (i = 0; i < tm->n_vlib_mains; i++)
1784  {
1786  fq = vlib_frame_queue_alloc (frame_queue_nelts);
1787  vec_add1 (fqm->vlib_frame_queues, fq);
1788 
1789  ptd = vec_elt_at_index (fqm->per_thread_data, i);
1791  tm->n_vlib_mains - 1);
1793  tm->n_vlib_mains - 1,
1794  (vlib_frame_queue_t *) (~0));
1795  }
1796 
1797  return (fqm - tm->frame_queue_mains);
1798 }
1799 
1800 int
1802 {
1804 
1805  if (tm->extern_thread_mgmt)
1806  return -1;
1807 
1809  tm->extern_thread_mgmt = 1;
1810  return 0;
1811 }
1812 
1813 void
1815  args)
1816 {
1817  ASSERT (vlib_get_thread_index () == 0);
1819  args->type_opaque, args->data);
1820 }
1821 
1823 
1824 void
1825 vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size)
1826 {
1828  {
1829  void (*fp) (void *, u8 *, u32) = rpc_call_main_thread_cb_fn;
1830  (*fp) (callback, args, arg_size);
1831  }
1832  else
1833  clib_warning ("BUG: rpc_call_main_thread_cb_fn NULL!");
1834 }
1835 
1836 clib_error_t *
1838 {
1839  return 0;
1840 }
1841 
1843 
1844 
1845 static clib_error_t *
1847  unformat_input_t * input, vlib_cli_command_t * cmd)
1848 {
1849  int i;
1850  f64 now;
1851 
1852  now = vlib_time_now (vm);
1853 
1854  vlib_cli_output (vm, "Time now %.9f", now);
1855 
1856  if (vec_len (vlib_mains) == 1)
1857  return 0;
1858 
1859  vlib_cli_output (vm, "Time last barrier release %.9f",
1861 
1862  for (i = 1; i < vec_len (vlib_mains); i++)
1863  {
1864  if (vlib_mains[i] == 0)
1865  continue;
1866  vlib_cli_output (vm, "Thread %d offset %.9f error %.9f", i,
1867  vlib_mains[i]->time_offset,
1870  }
1871  return 0;
1872 }
1873 
1874 /* *INDENT-OFF* */
1875 VLIB_CLI_COMMAND (f_command, static) =
1876 {
1877  .path = "show clock",
1878  .short_help = "show clock",
1879  .function = show_clock_command_fn,
1880 };
1881 /* *INDENT-ON* */
1882 
1883 /*
1884  * fd.io coding-style-patch-verification: ON
1885  *
1886  * Local Variables:
1887  * eval: (c-set-style "gnu")
1888  * End:
1889  */
_vlib_init_function_list_elt_t * worker_init_function_registrations
Definition: main.h:203
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
static void barrier_trace_release_rec(f64 t_entry)
Definition: threads.c:120
static void barrier_trace_sync(f64 t_entry, f64 t_open, f64 t_closed)
Definition: threads.c:69
u32 vl(void *p)
Definition: threads.c:31
static f64 vlib_last_vectors_per_main_loop_as_f64(vlib_main_t *vm)
Definition: main.h:337
uword * pending_rpc_requests
Definition: main.h:243
vlib_main_t vlib_global_main
Definition: main.c:1924
#define clib_min(x, y)
Definition: clib.h:295
clib_spinlock_t pending_rpc_lock
Definition: main.h:245
vlib_process_t ** processes
Definition: node.h:738
#define VLIB_PENDING_FRAME_NO_NEXT_FRAME
Definition: node.h:459
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:865
u8 runtime_data[0]
Function dependent node-runtime data.
Definition: node.h:521
int vlib_frame_queue_enqueue(vlib_main_t *vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t *frame, vlib_frame_queue_msg_type_t type)
#define clib_atomic_add_fetch(a, b)
Definition: atomics.h:30
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x)
Definition: init.h:166
word elog_track_register(elog_main_t *em, elog_track_t *t)
register an event track
Definition: elog.c:198
unsigned long u64
Definition: types.h:89
void * mheap_alloc(void *memory, uword size)
Definition: mheap.c:963
#define CLIB_MEMORY_STORE_BARRIER()
Definition: clib.h:118
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
#define NULL
Definition: clib.h:58
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 index
Definition: node.h:279
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:255
#define vec_add2_aligned(V, P, N, A)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:572
clib_error_t * threads_init(vlib_main_t *vm)
Definition: threads.c:1837
void os_panic(void)
Definition: unix-misc.c:174
u32 vlib_frame_queue_main_init(u32 node_index, u32 frame_queue_nelts)
Definition: threads.c:1762
u32 thread_index
Definition: main.h:197
void * thread_function_arg
Definition: threads.h:98
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static int sort_registrations_by_no_clone(void *a0, void *a1)
Definition: threads.c:197
static u64 clib_cpu_time_now(void)
Definition: time.h:75
frame_queue_trace_t * frame_queue_traces
Definition: threads.h:165
void vlib_process_signal_event_mt_helper(vlib_process_signal_event_mt_args_t *args)
Definition: threads.c:1814
elog_track_t elog_track
Definition: threads.h:100
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:560
int i
void vnet_main_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1344
static uword * clib_bitmap_set(uword *ai, uword i, uword value)
Sets the ith bit of a bitmap to new_value Removes trailing zeros from the bitmap. ...
Definition: bitmap.h:167
#define hash_set_mem(h, key, value)
Definition: hash.h:275
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
clib_time_t clib_time
Definition: main.h:72
void vlib_worker_thread_fn(void *arg)
Definition: threads.c:1728
u32 unformat_sched_policy(unformat_input_t *input, va_list *args)
Definition: threads.c:1215
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
struct vlib_thread_registration_ * next
Definition: threads.h:31
u32 buffer_index[VLIB_FRAME_SIZE]
Definition: threads.h:82
void * runtime_data
Definition: node.h:285
volatile u32 valid
Definition: threads.h:76
void vlib_worker_thread_barrier_sync_int(vlib_main_t *vm, const char *func_name)
Definition: threads.c:1387
vlib_main_t ** vlib_mains
Definition: buffer.c:321
static void vlib_get_thread_core_socket(vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:607
unsigned char u8
Definition: types.h:56
#define clib_bitmap_dup(v)
Duplicate a bitmap.
Definition: bitmap.h:87
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
u8 state
Definition: node.h:307
u16 thread_index
thread this node runs on
Definition: node.h:519
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:204
#define clib_memcpy(d, s, n)
Definition: string.h:180
u64 * counters_last_clear
Definition: error.h:51
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:390
vlib_thread_registration_t * next
Definition: threads.h:283
#define vec_add1_aligned(V, E, A)
Add 1 element to end of vector (alignment specified).
Definition: vec.h:531
#define VLIB_NODE_RUNTIME_DATA_SIZE
Definition: node.h:532
vlib_node_stats_t stats_last_clear
Definition: node.h:273
void vlib_worker_thread_node_runtime_update(void)
Definition: threads.c:1205
u64 count[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:801
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
#define VLIB_INVALID_NODE_INDEX
Definition: node.h:373
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:311
void * heap_aligned_base
Definition: main.h:123
vlib_frame_queue_msg_type_t
Definition: threads.h:68
vlib_node_t ** nodes
Definition: node.h:697
vlib_frame_queue_elt_t ** handoff_queue_elt_by_thread_index
Definition: threads.h:151
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
Definition: main.c:187
#define clib_error_return(e, args...)
Definition: error.h:99
#define VLIB_FRAME_ALIGN
Definition: node.h:377
uword * lock
SMP lock, non-zero means locking required.
Definition: elog.h:175
uword * cpu_core_bitmap
Definition: threads.h:320
#define BARRIER_MINIMUM_OPEN_FACTOR
Definition: threads.c:1383
clib_error_t * vlib_call_init_exit_functions(vlib_main_t *vm, _vlib_init_function_list_elt_t *head, int call_once)
Definition: init.c:43
vlib_frame_queue_elt_t * elts
Definition: threads.h:144
pthread_t thread[MAX_CONNS]
Definition: main.c:142
f64 time_last_barrier_release
Definition: main.h:75
unsigned int u32
Definition: types.h:88
vlib_node_runtime_t * nodes_by_type[VLIB_N_NODE_TYPE]
Definition: node.h:707
#define VLIB_FRAME_SIZE
Definition: node.h:376
void vlib_set_thread_name(char *name)
Definition: threads.c:182
void vl_msg_api_handler_no_free(void *)
Definition: threads.c:425
#define hash_create_string(elts, value_bytes)
Definition: hash.h:690
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:1023
vlib_fork_fixup_t
Definition: threads.h:227
#define BARRIER_SYNC_TIMEOUT
Definition: threads.h:199
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
void * rpc_call_main_thread_cb_fn
Definition: threads.c:1822
VLIB_REGISTER_THREAD(worker_thread_reg, static)
int extern_thread_mgmt
Definition: threads.h:339
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:36
#define clib_bitmap_foreach(i, ai, body)
Macro to iterate across set bits in a bitmap.
Definition: bitmap.h:361
void * thread_mheap
Definition: threads.h:95
u32 next_frame_index
Definition: node.h:456
vlib_node_stats_t stats_total
Definition: node.h:269
volatile u64 head
Definition: threads.h:131
u16 state
Input node state.
Definition: node.h:509
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:934
static uword clib_bitmap_first_set(uword *ai)
Return the lowest numbered set bit in a bitmap.
Definition: bitmap.h:385
u8 * vlib_thread_stack_init(uword thread_index)
Definition: main.c:624
static void vlib_next_frame_init(vlib_next_frame_t *nf)
Definition: node.h:440
vlib_error_main_t error_main
Definition: main.h:158
vlib_thread_callbacks_t cb
Definition: threads.h:338
#define VLIB_FRAME_NO_FREE_AFTER_DISPATCH
Definition: node.h:413
int vlib_thread_cb_register(struct vlib_main_t *vm, vlib_thread_callbacks_t *cb)
Definition: threads.c:1801
struct _unformat_input_t unformat_input_t
const char * barrier_context
Definition: threads.h:107
char * name
Track name vector.
Definition: elog.h:116
#define clib_error_return_unix(e, args...)
Definition: error.h:102
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
Definition: main.c:196
static void * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:64
u32 * node_by_error
Definition: node.h:765
void vlib_frame_free(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_frame_t *f)
Definition: main.c:216
#define ELOG_DATA(em, f)
Definition: elog.h:484
#define PREDICT_FALSE(x)
Definition: clib.h:111
static clib_error_t * vlib_launch_thread_int(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.c:627
void vlib_worker_thread_node_refork(void)
Definition: threads.c:1022
clib_error_t *(* vlib_thread_set_lcore_cb)(u32 thread, u16 cpu)
Definition: threads.h:277
u32 node_index
Node index.
Definition: node.h:494
u32 elog_global_id_for_msg_name(const char *msg_name)
Definition: threads.c:47
uword * init_functions_called
Definition: main.h:194
void clib_time_init(clib_time_t *c)
Definition: time.c:178
uword * frame_size_hash
Definition: node.h:753
u8 name[64]
Definition: memclnt.api:152
vlib_thread_main_t vlib_thread_main
Definition: threads.c:37
clib_error_t * clib_sysfs_read(char *file_name, char *fmt,...)
Definition: sysfs.c:50
word fformat(FILE *f, char *fmt,...)
Definition: format.c:462
void(* thread_function)(void *)
Definition: threads.h:97
static clib_error_t * cpu_config(vlib_main_t *vm, unformat_input_t *input)
Definition: threads.c:1229
u8 len
Definition: ip_types.api:49
i32 n_vectors[FRAME_QUEUE_MAX_NELTS]
Definition: node.h:796
u64 * counters
Definition: error.h:48
u32 owner_node_index
Definition: node.h:353
vlib_frame_queue_t * vlib_frame_queue_alloc(int nelts)
Definition: threads.c:391
volatile u64 tail
Definition: threads.h:123
#define clib_mem_alloc_no_fail(size)
Definition: mem.h:176
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:216
#define UNFORMAT_END_OF_INPUT
Definition: format.h:144
svmdb_client_t * c
u16 n_vectors
Definition: node.h:395
vlib_frame_queue_t ** vlib_frame_queues
Definition: threads.h:161
DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked)
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:212
vlib_main_t * vm
Definition: buffer.c:312
u32 node_runtime_index
Definition: node.h:450
vlib_pending_frame_t * pending_frames
Definition: node.h:723
vlib_thread_function_t * function
Definition: threads.h:36
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1600
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
void * heap_base
Definition: main.h:120
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:261
#define clib_warning(format, args...)
Definition: error.h:59
static vlib_node_runtime_t * vlib_node_get_runtime(vlib_main_t *vm, u32 node_index)
Get node runtime by node index.
Definition: node_funcs.h:89
elog_main_t elog_main
Definition: main.h:172
frame_queue_nelt_counter_t * frame_queue_histogram
Definition: threads.h:166
#define VLIB_FRAME_PENDING
Definition: node.h:427
static uword clib_bitmap_get(uword *ai, uword i)
Gets the ith bit value from a bitmap.
Definition: bitmap.h:197
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:442
void vlib_worker_thread_init(vlib_worker_thread_t *w)
Definition: threads.c:552
uword os_get_nthreads(void)
Definition: threads.c:170
static void * clib_mem_get_heap(void)
Definition: mem.h:255
volatile u32 * wait_at_barrier
Definition: threads.h:90
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:155
#define FRAME_QUEUE_NELTS
Definition: threads.c:28
vlib_frame_queue_per_thread_data_t * per_thread_data
Definition: threads.h:162
void vlib_stat_segment_unlock(void)
Definition: threads.c:1484
vlib_frame_queue_t ** congested_handoff_queue_by_thread_index
Definition: threads.h:152
#define never_inline
Definition: clib.h:95
signed int i32
Definition: types.h:77
#define hash_create(elts, value_bytes)
Definition: hash.h:696
#define ASSERT(truth)
never_inline void vlib_node_runtime_sync_stats(vlib_main_t *vm, vlib_node_runtime_t *r, uword n_calls, uword n_vectors, uword n_clocks, uword n_ticks0, uword n_ticks1)
Definition: main.c:544
static clib_error_t * show_clock_command_fn(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: threads.c:1846
static void barrier_trace_sync_rec(f64 t_entry)
Definition: threads.c:96
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:326
u16 flags
Definition: node.h:386
static void clib_mem_free(void *p)
Definition: mem.h:205
#define clib_error_report(e)
Definition: error.h:113
#define clib_bitmap_free(v)
Free a bitmap.
Definition: bitmap.h:92
size_t count
Definition: vapi.c:47
int need_vlib_worker_thread_node_runtime_update
Definition: main.h:228
uword * thread_registrations_by_name
Definition: threads.h:288
#define BARRIER_MINIMUM_OPEN_LIMIT
Definition: threads.c:1379
clib_error_t *(* vlib_launch_thread_cb)(void *fp, vlib_worker_thread_t *w, unsigned cpu_id)
Definition: threads.h:275
volatile u32 * node_reforks_required
Definition: threads.h:108
const char * barrier_caller
Definition: threads.h:106
#define clib_max(x, y)
Definition: clib.h:288
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
#define clib_atomic_fetch_add(a, b)
Definition: atomics.h:23
void vlib_node_sync_stats(vlib_main_t *vm, vlib_node_t *n)
Definition: main.c:586
static uword clib_bitmap_count_set_bits(uword *ai)
Return the number of set bits in a bitmap.
Definition: bitmap.h:462
static void barrier_trace_release(f64 t_entry, f64 t_closed_total, f64 t_update_main)
Definition: threads.c:142
void vlib_worker_loop(vlib_main_t *vm)
Definition: main.c:1919
#define vec_dup_aligned(V, A)
Return copy of vector (no header, alignment specified).
Definition: vec.h:384
u32 elog_string(elog_main_t *em, char *fmt,...)
add a string to the event-log string table
Definition: elog.c:562
f64 barrier_no_close_before
Definition: main.h:237
static clib_error_t * start_workers(vlib_main_t *vm)
Definition: threads.c:654
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void vlib_rpc_call_main_thread(void *callback, u8 *args, u32 arg_size)
Definition: threads.c:1825
DECLARE_CJ_GLOBAL_LOG
Definition: threads.c:26
vlib_node_main_t node_main
Definition: main.h:135
u64 uword
Definition: types.h:112
vlib_next_frame_t * next_frames
Definition: node.h:720
#define vec_sort_with_function(vec, f)
Sort a vector using the supplied element comparison function.
Definition: vec.h:980
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
Definition: node_funcs.h:244
volatile u64 head_hint
Definition: threads.h:140
#define VLIB_THREAD_STACK_SIZE
Definition: threads.h:66
f64 barrier_epoch
Definition: main.h:234
vlib_frame_size_t * frame_sizes
Definition: node.h:756
#define hash_get_mem(h, key)
Definition: hash.h:269
static void worker_thread_node_runtime_update_internal(void)
Definition: threads.c:970
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:140
volatile u32 * workers_at_barrier
Definition: threads.h:91
uword clib_calljmp(uword(*func)(uword func_arg), uword func_arg, void *stack)
static uword * clib_sysfs_list_to_bitmap(char *filename)
Definition: threads.c:207
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1489
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static f64 vlib_time_now_ticks(vlib_main_t *vm, u64 n)
Definition: main.h:261
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
Definition: node_funcs.h:59
void vlib_stat_segment_lock(void)
Definition: threads.c:1478
#define vec_foreach(var, vec)
Vector iterator.
void * vlib_worker_thread_bootstrap_fn(void *arg)
Definition: threads.c:589
#define SYS_gettid
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:115
u32 node_runtime_index
Definition: node.h:407
uword * cpu_socket_bitmap
Definition: threads.h:323
#define foreach_sched_policy
Definition: threads.h:258
vlib_thread_registration_t ** registrations
Definition: threads.h:286
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:486
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
u8 ** vlib_thread_stacks
Definition: main.c:607
pthread_t thread_id
Definition: threads.h:114
vlib_thread_registration_t * registration
Definition: threads.h:102
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:762
volatile u32 worker_thread_release
Definition: threads.h:329
void vlib_worker_thread_fork_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1349
clib_random_buffer_t random_buffer
Definition: main.h:191
u8 runtime_data_bytes
Definition: node.h:310
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:170
#define VLIB_FRAME_FREE_AFTER_DISPATCH
Definition: node.h:430
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:237