FD.io VPP  v17.04-9-g99c0734
Vector Packet Processing
threads.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef included_vlib_threads_h
16 #define included_vlib_threads_h
17 
18 #include <vlib/main.h>
19 #include <linux/sched.h>
20 
21 extern vlib_main_t **vlib_mains;
22 
23 void vlib_set_thread_name (char *name);
24 
25 /* arg is actually a vlib__thread_t * */
26 typedef void (vlib_thread_function_t) (void *arg);
27 
29 {
30  /* constructor generated list of thread registrations */
32 
33  /* config parameters */
34  char *name;
35  char *short_name;
42 
43  /* All threads of this type run on pthreads */
48 
49 /*
50  * Frames have their cpu / vlib_main_t index in the low-order N bits
51  * Make VLIB_MAX_CPUS a power-of-two, please...
52  */
53 
54 #ifndef VLIB_MAX_CPUS
55 #define VLIB_MAX_CPUS 256
56 #endif
57 
58 #if VLIB_MAX_CPUS > CLIB_MAX_MHEAPS
59 #error Please increase number of per-cpu mheaps
60 #endif
61 
62 #define VLIB_CPU_MASK (VLIB_MAX_CPUS - 1) /* 0x3f, max */
63 #define VLIB_OFFSET_MASK (~VLIB_CPU_MASK)
64 
65 #define VLIB_LOG2_THREAD_STACK_SIZE (20)
66 #define VLIB_THREAD_STACK_SIZE (1<<VLIB_LOG2_THREAD_STACK_SIZE)
67 
68 typedef enum
69 {
72 
73 typedef struct
74 {
75  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
76  volatile u32 valid;
80 
81  /* 256 * 4 = 1024 bytes, even mult of cache line size */
82  u32 buffer_index[VLIB_FRAME_SIZE];
83 }
85 
86 typedef struct
87 {
88  /* First cache line */
89  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
90  volatile u32 *wait_at_barrier;
92 
93  /* Second Cache Line */
94  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
95  void *thread_mheap;
97  void (*thread_function) (void *);
105 
106  long lwp;
107  int lcore_id;
108  pthread_t thread_id;
110 
112 
113 typedef struct
114 {
115  /* enqueue side */
116  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
117  volatile u64 tail;
122 
123  /* dequeue side */
124  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
125  volatile u64 head;
131 
132  /* dequeue hint to enqueue side */
133  CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
134  volatile u64 head_hint;
135 
136  /* read-only, constant, shared */
137  CLIB_CACHE_LINE_ALIGN_MARK (cacheline3);
140 }
142 
143 typedef struct
144 {
147 
148  /* for frame queue tracing */
152 
153 /* Called early, in thread 0's context */
155 
157 
158 int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index,
159  u32 frame_queue_index, vlib_frame_t * frame,
160  vlib_frame_queue_msg_type_t type);
161 
162 int
164 
166  vlib_node_runtime_t * node,
167  vlib_node_type_t type,
168  vlib_node_state_t dispatch_state,
169  vlib_frame_t * frame, u64 last_time_stamp);
170 
172  vlib_pending_frame_t * p, u64 last_time_stamp);
173 
175 
176 void vlib_create_worker_threads (vlib_main_t * vm, int n,
177  void (*thread_function) (void *));
178 
181 
182 /* Check for a barrier sync request every 30ms */
183 #define BARRIER_SYNC_DELAY (0.030000)
184 
185 #if CLIB_DEBUG > 0
186 /* long barrier timeout, for gdb... */
187 #define BARRIER_SYNC_TIMEOUT (600.1)
188 #else
189 #define BARRIER_SYNC_TIMEOUT (1.0)
190 #endif
191 
194 
195 always_inline void
197 {
198  if (CLIB_DEBUG > 0)
199  {
200  if (os_get_cpu_number ())
201  fformat (stderr, "%s: SMP unsafe warning...\n", __FUNCTION__);
202  }
203 }
204 
205 typedef enum
206 {
210 
211 void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which);
212 
213 static inline void
215 {
216  if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier))
217  {
218  clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1);
219  while (*vlib_worker_threads->wait_at_barrier)
220  ;
221  clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1);
222  }
223 }
224 
225 #define foreach_vlib_main(body) \
226 do { \
227  vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \
228  int ii; \
229  \
230  for (ii = 0; ii < vec_len (vlib_mains); ii++) \
231  { \
232  this_vlib_main = vlib_mains[ii]; \
233  if (this_vlib_main) \
234  vec_add1 (__vlib_mains, this_vlib_main); \
235  } \
236  \
237  for (ii = 0; ii < vec_len (__vlib_mains); ii++) \
238  { \
239  this_vlib_main = __vlib_mains[ii]; \
240  /* body uses this_vlib_main... */ \
241  (body); \
242  } \
243  vec_free (__vlib_mains); \
244 } while (0);
245 
246 #define foreach_sched_policy \
247  _(SCHED_OTHER, OTHER, "other") \
248  _(SCHED_BATCH, BATCH, "batch") \
249  _(SCHED_IDLE, IDLE, "idle") \
250  _(SCHED_FIFO, FIFO, "fifo") \
251  _(SCHED_RR, RR, "rr")
252 
253 typedef enum
254 {
255 #define _(v,f,s) SCHED_POLICY_##f = v,
257 #undef _
260 
261 typedef struct
262 {
263  clib_error_t *(*vlib_launch_thread_cb) (void *fp, vlib_worker_thread_t * w,
264  unsigned lcore_id);
265  clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 lcore);
266 } vlib_thread_callbacks_t;
267 
268 typedef struct
269 {
270  /* Link list of registrations, built by constructors */
272 
273  /* Vector of registrations, w/ non-data-structure clones at the top */
275 
277 
279 
280  /*
281  * Launch all threads as pthreads,
282  * not eal_rte_launch (strict affinity) threads
283  */
285 
286  /* Number of vlib_main / vnet_main clones */
288 
289  /* Number of thread stacks to create */
291 
292  /* Number of pthreads */
294 
295  /* Number of threads */
297 
298  /* Number of cores to skip, must match the core mask */
300 
301  /* Thread prefix name */
303 
304  /* main thread lcore */
306 
307  /* Bitmap of available CPU cores */
309 
310  /* Bitmap of available CPU sockets (NUMA nodes) */
312 
313  /* Worker handoff queues */
315 
316  /* worker thread initialization barrier */
318 
319  /* scheduling policy */
321 
322  /* scheduling policy priority */
324 
325  /* callbacks */
326  vlib_thread_callbacks_t cb;
329 
331 
332 #define VLIB_REGISTER_THREAD(x,...) \
333  __VA_ARGS__ vlib_thread_registration_t x; \
334 static void __vlib_add_thread_registration_##x (void) \
335  __attribute__((__constructor__)) ; \
336 static void __vlib_add_thread_registration_##x (void) \
337 { \
338  vlib_thread_main_t * tm = &vlib_thread_main; \
339  x.next = tm->next; \
340  tm->next = &x; \
341 } \
342 __VA_ARGS__ vlib_thread_registration_t x
343 
346 {
347  return vlib_thread_main.n_vlib_mains - 1;
348 }
349 
352 {
353  return worker_index + 1;
354 }
355 
358 {
359  return cpu_index - 1;
360 }
361 
364 {
365  return os_get_cpu_number () - 1;
366 }
367 
370 {
371  vlib_main_t *vm;
373  ASSERT (worker_index < tm->n_vlib_mains - 1);
374  vm = vlib_mains[worker_index + 1];
375  ASSERT (vm);
376  return vm;
377 }
378 
379 static inline void
381 {
383  hf->valid = 1;
384 }
385 
386 static inline vlib_frame_queue_elt_t *
387 vlib_get_frame_queue_elt (u32 frame_queue_index, u32 index)
388 {
389  vlib_frame_queue_t *fq;
393  vec_elt_at_index (tm->frame_queue_mains, frame_queue_index);
394  u64 new_tail;
395 
396  fq = fqm->vlib_frame_queues[index];
397  ASSERT (fq);
398 
399  new_tail = __sync_add_and_fetch (&fq->tail, 1);
400 
401  /* Wait until a ring slot is available */
402  while (new_tail >= fq->head_hint + fq->nelts)
404 
405  elt = fq->elts + (new_tail & (fq->nelts - 1));
406 
407  /* this would be very bad... */
408  while (elt->valid)
409  ;
410 
412  elt->last_n_vectors = elt->n_vectors = 0;
413 
414  return elt;
415 }
416 
417 static inline vlib_frame_queue_t *
419  u32 index,
420  u32 queue_hi_thresh,
422  handoff_queue_by_worker_index)
423 {
424  vlib_frame_queue_t *fq;
427  vec_elt_at_index (tm->frame_queue_mains, frame_queue_index);
428 
429  fq = handoff_queue_by_worker_index[index];
430  if (fq != (vlib_frame_queue_t *) (~0))
431  return fq;
432 
433  fq = fqm->vlib_frame_queues[index];
434  ASSERT (fq);
435 
436  if (PREDICT_FALSE (fq->tail >= (fq->head_hint + queue_hi_thresh)))
437  {
438  /* a valid entry in the array will indicate the queue has reached
439  * the specified threshold and is congested
440  */
441  handoff_queue_by_worker_index[index] = fq;
442  fq->enqueue_full_events++;
443  return fq;
444  }
445 
446  return NULL;
447 }
448 
449 static inline vlib_frame_queue_elt_t *
451  u32 vlib_worker_index,
453  handoff_queue_elt_by_worker_index)
454 {
456 
457  if (handoff_queue_elt_by_worker_index[vlib_worker_index])
458  return handoff_queue_elt_by_worker_index[vlib_worker_index];
459 
460  elt = vlib_get_frame_queue_elt (frame_queue_index, vlib_worker_index);
461 
462  handoff_queue_elt_by_worker_index[vlib_worker_index] = elt;
463 
464  return elt;
465 }
466 
467 int vlib_thread_cb_register (struct vlib_main_t *vm,
468  vlib_thread_callbacks_t * cb);
469 
470 #endif /* included_vlib_threads_h */
471 
472 /*
473  * fd.io coding-style-patch-verification: ON
474  *
475  * Local Variables:
476  * eval: (c-set-style "gnu")
477  * End:
478  */
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:68
void vlib_worker_thread_init(vlib_worker_thread_t *w)
Definition: threads.c:455
int vlib_frame_queue_enqueue(vlib_main_t *vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t *frame, vlib_frame_queue_msg_type_t type)
vlib_worker_thread_t * vlib_alloc_thread(vlib_main_t *vm)
Definition: threads.c:279
#define NULL
Definition: clib.h:55
void vlib_set_thread_name(char *name)
Definition: threads.c:74
void * thread_function_arg
Definition: threads.h:98
frame_queue_trace_t * frame_queue_traces
Definition: threads.h:149
elog_track_t elog_track
Definition: threads.h:100
vlib_worker_thread_t * vlib_worker_threads
Definition: threads.c:35
struct vlib_thread_registration_ * next
Definition: threads.h:31
int vlib_thread_cb_register(struct vlib_main_t *vm, vlib_thread_callbacks_t *cb)
Definition: threads.c:1450
volatile u32 valid
Definition: threads.h:76
vlib_node_state_t
Definition: node.h:207
static vlib_frame_queue_t * is_vlib_frame_queue_congested(u32 frame_queue_index, u32 index, u32 queue_hi_thresh, vlib_frame_queue_t **handoff_queue_by_worker_index)
Definition: threads.h:418
static void vlib_smp_unsafe_warning(void)
Definition: threads.h:196
static void vlib_worker_thread_barrier_check(void)
Definition: threads.h:214
vlib_thread_registration_t * next
Definition: threads.h:271
u64 dispatch_node(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_node_type_t type, vlib_node_state_t dispatch_state, vlib_frame_t *frame, u64 last_time_stamp)
Definition: main.c:921
#define clib_smp_atomic_add(addr, increment)
Definition: smp.h:46
#define always_inline
Definition: clib.h:84
vlib_frame_queue_msg_type_t
Definition: threads.h:68
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
uword * cpu_core_bitmap
Definition: threads.h:308
unsigned long u64
Definition: types.h:89
int vlib_frame_queue_dequeue(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
Definition: threads.c:1261
vlib_frame_queue_elt_t * elts
Definition: threads.h:138
static u32 vlib_get_current_worker_index()
Definition: threads.h:363
vlib_fork_fixup_t
Definition: threads.h:205
int extern_thread_mgmt
Definition: threads.h:327
sched_policy_t
Definition: threads.h:253
void * thread_mheap
Definition: threads.h:95
volatile u64 head
Definition: threads.h:125
vlib_node_type_t
Definition: node.h:58
vlib_thread_callbacks_t cb
Definition: threads.h:326
uword os_get_cpu_number(void)
Definition: unix-misc.c:224
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1231
#define PREDICT_FALSE(x)
Definition: clib.h:97
static vlib_frame_queue_elt_t * vlib_get_worker_handoff_queue_elt(u32 frame_queue_index, u32 vlib_worker_index, vlib_frame_queue_elt_t **handoff_queue_elt_by_worker_index)
Definition: threads.h:450
#define VLIB_FRAME_SIZE
Definition: node.h:328
long i64
Definition: types.h:82
word fformat(FILE *f, char *fmt,...)
Definition: format.c:452
void vlib_worker_thread_node_runtime_update(void)
Definition: threads.c:814
void vlib_worker_thread_fork_fixup(vlib_fork_fixup_t which)
Definition: threads.c:1176
volatile u64 tail
Definition: threads.h:117
vlib_frame_queue_t ** vlib_frame_queues
Definition: threads.h:146
vlib_main_t * vm
Definition: buffer.c:276
static u32 vlib_get_worker_index(u32 cpu_index)
Definition: threads.h:357
frame_queue_nelt_counter_t * frame_queue_histogram
Definition: threads.h:150
vlib_worker_thread_t * worker_threads
Definition: threads.h:278
static vlib_frame_queue_elt_t * vlib_get_frame_queue_elt(u32 frame_queue_index, u32 index)
Definition: threads.h:387
volatile u32 * wait_at_barrier
Definition: threads.h:90
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
vlib_frame_queue_main_t * frame_queue_mains
Definition: threads.h:314
struct vlib_thread_registration_ vlib_thread_registration_t
uword * thread_registrations_by_name
Definition: threads.h:276
u32 vlib_frame_queue_main_init(u32 node_index, u32 frame_queue_nelts)
Definition: threads.c:1423
clib_error_t * vlib_thread_init(vlib_main_t *vm)
Definition: threads.c:129
u64 uword
Definition: types.h:112
u32 enqueue_full_events
Definition: threads.h:121
unsigned short u16
Definition: types.h:57
unsigned char u8
Definition: types.h:56
vlib_main_t ** vlib_mains
Definition: buffer.c:285
void vlib_create_worker_threads(vlib_main_t *vm, int n, void(*thread_function)(void *))
volatile u64 head_hint
Definition: threads.h:134
static void vlib_put_frame_queue_elt(vlib_frame_queue_elt_t *hf)
Definition: threads.h:380
static u32 vlib_get_worker_cpu_index(u32 worker_index)
Definition: threads.h:351
void vlib_worker_thread_barrier_sync(vlib_main_t *vm)
Definition: threads.c:1199
volatile u32 * workers_at_barrier
Definition: threads.h:91
static u32 vlib_num_workers()
Definition: threads.h:345
u64 dispatch_pending_node(vlib_main_t *vm, vlib_pending_frame_t *p, u64 last_time_stamp)
Definition: main.c:1097
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:101
uword * cpu_socket_bitmap
Definition: threads.h:311
#define foreach_sched_policy
Definition: threads.h:246
void( vlib_thread_function_t)(void *arg)
Definition: threads.h:26
vlib_thread_registration_t ** registrations
Definition: threads.h:274
vlib_thread_main_t vlib_thread_main
Definition: threads.c:36
pthread_t thread_id
Definition: threads.h:108
vlib_thread_registration_t * registration
Definition: threads.h:102
volatile u32 worker_thread_release
Definition: threads.h:317
static vlib_main_t * vlib_get_worker_vlib_main(u32 worker_index)
Definition: threads.h:369