FD.io VPP  v20.01-rc0-788-g61e3ade84
Vector Packet Processing
memory_api.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 #include <signal.h>
18 
19 #include <vlib/vlib.h>
20 #include <vlibapi/api.h>
21 #include <vlibmemory/api.h>
22 #include <vlibmemory/memory_api.h>
23 
24 #include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */
25 
26 #define vl_typedefs /* define message structures */
28 #undef vl_typedefs
29 
30 /* instantiate all the print functions we know about */
31 #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__)
32 #define vl_printfun
34 #undef vl_printfun
35 
36 /* instantiate all the endian swap functions we know about */
37 #define vl_endianfun
39 #undef vl_endianfun
40 
41 static inline void *
43 {
44  vl_print (handle, "vl_api_memclnt_create_t:\n");
45  vl_print (handle, "name: %s\n", a->name);
46  vl_print (handle, "input_queue: 0x%wx\n", a->input_queue);
47  vl_print (handle, "context: %u\n", (unsigned) a->context);
48  vl_print (handle, "ctx_quota: %ld\n", (long) a->ctx_quota);
49  return handle;
50 }
51 
52 static inline void *
54 {
55  vl_print (handle, "vl_api_memclnt_delete_t:\n");
56  vl_print (handle, "index: %u\n", (unsigned) a->index);
57  vl_print (handle, "handle: 0x%wx\n", a->handle);
58  return handle;
59 }
60 
61 volatile int **vl_api_queue_cursizes;
62 
63 static void
65 {
66  int i;
67  api_main_t *am = &api_main;
68 
70  1 + vec_len (am->vlib_private_rps)))
71  {
72  vl_shmem_hdr_t *shmem_hdr = am->shmem_hdr;
73  svm_queue_t *q;
74 
75  if (shmem_hdr == 0)
76  return;
77 
78  q = shmem_hdr->vl_input_queue;
79  if (q == 0)
80  return;
81 
82  vec_add1 (vl_api_queue_cursizes, &q->cursize);
83 
84  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
85  {
86  svm_region_t *vlib_rp = am->vlib_private_rps[i];
87 
88  shmem_hdr = (void *) vlib_rp->user_ctx;
89  q = shmem_hdr->vl_input_queue;
90  vec_add1 (vl_api_queue_cursizes, &q->cursize);
91  }
92  }
93 
94  for (i = 0; i < vec_len (vl_api_queue_cursizes); i++)
95  {
96  if (*vl_api_queue_cursizes[i])
97  {
98  vm->queue_signal_pending = 1;
99  vm->api_queue_nonempty = 1;
101  /* event_type */ QUEUE_SIGNAL_EVENT,
102  /* event_data */ 0);
103  break;
104  }
105  }
106  if (vec_len (vm->pending_rpc_requests))
107  {
108  vm->queue_signal_pending = 1;
109  vm->api_queue_nonempty = 1;
111  /* event_type */ QUEUE_SIGNAL_EVENT,
112  /* event_data */ 0);
113  }
114 }
115 
116 /*
117  * vl_api_memclnt_create_internal
118  */
119 u32
121 {
122  vl_api_registration_t **regpp;
123  vl_api_registration_t *regp;
124  svm_region_t *svm;
125  void *oldheap;
126  api_main_t *am = &api_main;
127 
128  ASSERT (vlib_get_thread_index () == 0);
129  pool_get (am->vl_clients, regpp);
130 
131  svm = am->vlib_rp;
132 
133  pthread_mutex_lock (&svm->mutex);
134  oldheap = svm_push_data_heap (svm);
135  *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
136 
137  regp = *regpp;
138  clib_memset (regp, 0, sizeof (*regp));
140  regp->vl_api_registration_pool_index = regpp - am->vl_clients;
141  regp->vlib_rp = svm;
142  regp->shmem_hdr = am->shmem_hdr;
143 
144  regp->vl_input_queue = q;
145  regp->name = format (0, "%s%c", name, 0);
146 
147  pthread_mutex_unlock (&svm->mutex);
148  svm_pop_heap (oldheap);
152 }
153 
154 /*
155  * vl_api_memclnt_create_t_handler
156  */
157 void
159 {
160  vl_api_registration_t **regpp;
161  vl_api_registration_t *regp;
163  svm_region_t *svm;
164  svm_queue_t *q;
165  int rv = 0;
166  void *oldheap;
167  api_main_t *am = &api_main;
168  u8 *msg_table;
169 
170  /*
171  * This is tortured. Maintain a vlib-address-space private
172  * pool of client registrations. We use the shared-memory virtual
173  * address of client structure as a handle, to allow direct
174  * manipulation of context quota vbls from the client library.
175  *
176  * This scheme causes trouble w/ API message trace replay, since
177  * some random VA from clib_mem_alloc() certainly won't
178  * occur in the Linux sim. The (very) few places
179  * that care need to use the pool index.
180  *
181  * Putting the registration object(s) into a pool in shared memory and
182  * using the pool index as a handle seems like a great idea.
183  * Unfortunately, each and every reference to that pool would need
184  * to be protected by a mutex:
185  *
186  * Client VLIB
187  * ------ ----
188  * convert pool index to
189  * pointer.
190  * <deschedule>
191  * expand pool
192  * <deschedule>
193  * kaboom!
194  */
195 
196  pool_get (am->vl_clients, regpp);
197 
198  svm = am->vlib_rp;
199 
200  pthread_mutex_lock (&svm->mutex);
201  oldheap = svm_push_data_heap (svm);
202  *regpp = clib_mem_alloc (sizeof (vl_api_registration_t));
203 
204  regp = *regpp;
205  clib_memset (regp, 0, sizeof (*regp));
207  regp->vl_api_registration_pool_index = regpp - am->vl_clients;
208  regp->vlib_rp = svm;
209  regp->shmem_hdr = am->shmem_hdr;
211 
212  q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue;
214 
215  regp->name = format (0, "%s", mp->name);
216  vec_add1 (regp->name, 0);
217 
221 
222  if (am->vlib_rp != am->vlib_primary_rp)
223  msg_table = vl_api_serialize_message_table (am, 0);
224  else
225  msg_table = am->serialized_message_table_in_shmem;
226 
227  pthread_mutex_unlock (&svm->mutex);
228  svm_pop_heap (oldheap);
229 
230  rp = vl_msg_api_alloc (sizeof (*rp));
231  rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_REPLY);
232  rp->handle = (uword) regp;
236  rp->context = mp->context;
237  rp->response = ntohl (rv);
238  rp->message_table = pointer_to_uword (msg_table);
239 
240  vl_msg_api_send_shmem (q, (u8 *) & rp);
241 }
242 
243 int
245 {
246  clib_error_t *error = 0;
247  _vl_msg_api_function_list_elt_t *i;
248 
250  while (i)
251  {
252  error = i->f (client_index);
253  if (error)
254  clib_error_report (error);
255  i = i->next_init_function;
256  }
257  return 0;
258 }
259 
260 /*
261  * vl_api_memclnt_delete_t_handler
262  */
263 void
265 {
266  vl_api_registration_t **regpp;
267  vl_api_registration_t *regp;
269  svm_region_t *svm;
270  void *oldheap;
271  api_main_t *am = &api_main;
272  u32 handle, client_index, epoch;
273 
274  handle = mp->index;
275 
276  if (vl_api_call_reaper_functions (handle))
277  return;
278 
279  epoch = vl_msg_api_handle_get_epoch (handle);
280  client_index = vl_msg_api_handle_get_index (handle);
281 
282  if (epoch != (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK))
283  {
285  ("Stale clnt delete index %d old epoch %d cur epoch %d",
286  client_index, epoch,
288  return;
289  }
290 
291  regpp = pool_elt_at_index (am->vl_clients, client_index);
292 
293  if (!pool_is_free (am->vl_clients, regpp))
294  {
295  int i;
296  regp = *regpp;
297  svm = am->vlib_rp;
298  int private_registration = 0;
299 
300  /* Send reply unless client asked us to do the cleanup */
301  if (!mp->do_cleanup)
302  {
303  /*
304  * Note: the API message handling path will set am->vlib_rp
305  * as appropriate for pairwise / private memory segments
306  */
307  rp = vl_msg_api_alloc (sizeof (*rp));
308  rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_DELETE_REPLY);
309  rp->handle = mp->handle;
310  rp->response = 1;
311 
312  vl_msg_api_send_shmem (regp->vl_input_queue, (u8 *) & rp);
313  if (client_index != regp->vl_api_registration_pool_index)
314  {
315  clib_warning ("mismatch client_index %d pool_index %d",
316  client_index,
318  vl_msg_api_free (rp);
319  return;
320  }
321  }
322 
323  /* No dangling references, please */
324  *regpp = 0;
325 
326  /* For horizontal scaling, add a hash table... */
327  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
328  {
329  /* Is this a pairwise / private API segment? */
330  if (am->vlib_private_rps[i] == svm)
331  {
332  /* Note: account for the memfd header page */
333  uword virtual_base = svm->virtual_base - MMAP_PAGESIZE;
334  uword virtual_size = svm->virtual_size + MMAP_PAGESIZE;
335 
336  /*
337  * Kill the registration pool element before we make
338  * the index vanish forever
339  */
342 
343  vec_delete (am->vlib_private_rps, 1, i);
344  /* Kill it, accounting for the memfd header page */
345  if (munmap ((void *) virtual_base, virtual_size) < 0)
346  clib_unix_warning ("munmap");
347  /* Reset the queue-length-address cache */
349  private_registration = 1;
350  break;
351  }
352  }
353 
354  if (private_registration == 0)
355  {
358  pthread_mutex_lock (&svm->mutex);
359  oldheap = svm_push_data_heap (svm);
360  if (mp->do_cleanup)
362  vec_free (regp->name);
363  /* Poison the old registration */
364  clib_memset (regp, 0xF1, sizeof (*regp));
365  clib_mem_free (regp);
366  pthread_mutex_unlock (&svm->mutex);
367  svm_pop_heap (oldheap);
368  /*
369  * These messages must be freed manually, since they're set up
370  * as "bounce" messages. In the private_registration == 1 case,
371  * we kill the shared-memory segment which contains the message
372  * with munmap.
373  */
374  vl_msg_api_free (mp);
375  }
376  }
377  else
378  {
379  clib_warning ("unknown client ID %d", mp->index);
380  }
381 }
382 
383 /**
384  * client answered a ping, stave off the grim reaper...
385  */
386 void
388  (vl_api_memclnt_keepalive_reply_t * mp)
389 {
390  vl_api_registration_t *regp;
392 
393  regp = vl_api_client_index_to_registration (mp->context);
394  if (regp)
395  {
396  regp->last_heard = vlib_time_now (vm);
397  regp->unanswered_pings = 0;
398  }
399  else
400  clib_warning ("BUG: anonymous memclnt_keepalive_reply");
401 }
402 
403 /**
404  * We can send ourselves these messages if someone uses the
405  * builtin binary api test tool...
406  */
407 static void
409 {
410  vl_api_memclnt_keepalive_reply_t *rmp;
411  api_main_t *am;
412  vl_shmem_hdr_t *shmem_hdr;
413 
414  am = &api_main;
415  shmem_hdr = am->shmem_hdr;
416 
417  rmp = vl_msg_api_alloc_as_if_client (sizeof (*rmp));
418  clib_memset (rmp, 0, sizeof (*rmp));
419  rmp->_vl_msg_id = ntohs (VL_API_MEMCLNT_KEEPALIVE_REPLY);
420  rmp->context = mp->context;
421  vl_msg_api_send_shmem (shmem_hdr->vl_input_queue, (u8 *) & rmp);
422 }
423 
424 /*
425  * To avoid filling the API trace buffer with boring messages,
426  * don't trace memclnt_keepalive[_reply] msgs
427  */
428 
429 #define foreach_vlib_api_msg \
430 _(MEMCLNT_CREATE, memclnt_create, 1) \
431 _(MEMCLNT_DELETE, memclnt_delete, 1) \
432 _(MEMCLNT_KEEPALIVE, memclnt_keepalive, 0) \
433 _(MEMCLNT_KEEPALIVE_REPLY, memclnt_keepalive_reply, 0)
434 
435 /*
436  * memory_api_init
437  */
438 int
439 vl_mem_api_init (const char *region_name)
440 {
441  int rv;
442  api_main_t *am = &api_main;
444  vl_msg_api_msg_config_t *c = &cfg;
445  vl_shmem_hdr_t *shm;
447 
448  clib_memset (c, 0, sizeof (*c));
449 
450  if ((rv = vl_map_shmem (region_name, 1 /* is_vlib */ )) < 0)
451  return rv;
452 
453 #define _(N,n,t) do { \
454  c->id = VL_API_##N; \
455  c->name = #n; \
456  c->handler = vl_api_##n##_t_handler; \
457  c->cleanup = vl_noop_handler; \
458  c->endian = vl_api_##n##_t_endian; \
459  c->print = vl_api_##n##_t_print; \
460  c->size = sizeof(vl_api_##n##_t); \
461  c->traced = t; /* trace, so these msgs print */ \
462  c->replay = 0; /* don't replay client create/delete msgs */ \
463  c->message_bounce = 0; /* don't bounce this message */ \
464  vl_msg_api_config(c);} while (0);
465 
467 #undef _
468 
469  /*
470  * special-case freeing of memclnt_delete messages, so we can
471  * simply munmap pairwise / private API segments...
472  */
473  am->message_bounce[VL_API_MEMCLNT_DELETE] = 1;
474  am->is_mp_safe[VL_API_MEMCLNT_KEEPALIVE_REPLY] = 1;
475  am->is_mp_safe[VL_API_MEMCLNT_KEEPALIVE] = 1;
476 
478 
479  shm = am->shmem_hdr;
480  ASSERT (shm && shm->vl_input_queue);
481 
482  /* Make a note so we can always find the primary region easily */
483  am->vlib_primary_rp = am->vlib_rp;
484 
485  return 0;
486 }
487 
488 clib_error_t *
490 {
491  api_main_t *am = &api_main;
492  int rv;
493 
494  if ((rv = vl_mem_api_init (am->region_name)) < 0)
495  {
496  return clib_error_return (0, "vl_mem_api_init (%s) failed",
497  am->region_name);
498  }
499  return 0;
500 }
501 
502 static void
504 {
506  svm_queue_t *q;
507  api_main_t *am = &api_main;
508  svm_region_t *save_vlib_rp = am->vlib_rp;
509  vl_shmem_hdr_t *save_shmem_hdr = am->shmem_hdr;
510 
511  q = regp->vl_input_queue;
512 
513  /*
514  * If the queue head is moving, assume that the client is processing
515  * messages and skip the ping. This heuristic may fail if the queue
516  * is in the same position as last time, net of wrapping; in which
517  * case, the client will receive a keepalive.
518  */
519  if (regp->last_queue_head != q->head)
520  {
521  regp->last_heard = now;
522  regp->unanswered_pings = 0;
523  regp->last_queue_head = q->head;
524  return;
525  }
526 
527  /*
528  * push/pop shared memory segment, so this routine
529  * will work with "normal" as well as "private segment"
530  * memory clients..
531  */
532 
533  am->vlib_rp = regp->vlib_rp;
534  am->shmem_hdr = regp->shmem_hdr;
535 
536  mp = vl_msg_api_alloc (sizeof (*mp));
537  clib_memset (mp, 0, sizeof (*mp));
538  mp->_vl_msg_id = clib_host_to_net_u16 (VL_API_MEMCLNT_KEEPALIVE);
539  mp->context = mp->client_index =
543 
544  regp->unanswered_pings++;
545 
546  /* Failure-to-send due to a stuffed queue is absolutely expected */
547  if (svm_queue_add (q, (u8 *) & mp, 1 /* nowait */ ))
548  vl_msg_api_free (mp);
549 
550  am->vlib_rp = save_vlib_rp;
551  am->shmem_hdr = save_shmem_hdr;
552 }
553 
554 static void
556  vl_api_registration_t ** regpp,
557  u32 ** dead_indices,
558  u32 ** confused_indices)
559 {
560  vl_api_registration_t *regp = *regpp;
561  if (regp)
562  {
563  /* If we haven't heard from this client recently... */
564  if (regp->last_heard < (now - 10.0))
565  {
566  if (regp->unanswered_pings == 2)
567  {
568  svm_queue_t *q;
569  q = regp->vl_input_queue;
570  if (kill (q->consumer_pid, 0) >= 0)
571  {
572  clib_warning ("REAPER: lazy binary API client '%s'",
573  regp->name);
574  regp->unanswered_pings = 0;
575  regp->last_heard = now;
576  }
577  else
578  {
579  clib_warning ("REAPER: binary API client '%s' died",
580  regp->name);
581  vec_add1 (*dead_indices, regpp - am->vl_clients);
582  }
583  }
584  else
585  send_memclnt_keepalive (regp, now);
586  }
587  else
588  regp->unanswered_pings = 0;
589  }
590  else
591  {
592  clib_warning ("NULL client registration index %d",
593  regpp - am->vl_clients);
594  vec_add1 (*confused_indices, regpp - am->vl_clients);
595  }
596 }
597 
598 void
600 {
601  vl_api_registration_t **regpp;
602  static u32 *dead_indices;
603  static u32 *confused_indices;
604 
605  vec_reset_length (dead_indices);
606  vec_reset_length (confused_indices);
607 
608  /* *INDENT-OFF* */
609  pool_foreach (regpp, am->vl_clients, ({
610  vl_mem_send_client_keepalive_w_reg (am, now, regpp, &dead_indices,
611  &confused_indices);
612  }));
613  /* *INDENT-ON* */
614 
615  /* This should "never happen," but if it does, fix it... */
616  if (PREDICT_FALSE (vec_len (confused_indices) > 0))
617  {
618  int i;
619  for (i = 0; i < vec_len (confused_indices); i++)
620  {
621  pool_put_index (am->vl_clients, confused_indices[i]);
622  }
623  }
624 
625  if (PREDICT_FALSE (vec_len (dead_indices) > 0))
626  {
627  int i;
628  svm_region_t *svm;
629  void *oldheap;
630 
631  /* Allow the application to clean up its registrations */
632  for (i = 0; i < vec_len (dead_indices); i++)
633  {
634  regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
635  if (regpp)
636  {
637  u32 handle;
638 
640  (dead_indices[i], shm->application_restarts);
641  (void) vl_api_call_reaper_functions (handle);
642  }
643  }
644 
645  svm = am->vlib_rp;
646  pthread_mutex_lock (&svm->mutex);
647  oldheap = svm_push_data_heap (svm);
648 
649  for (i = 0; i < vec_len (dead_indices); i++)
650  {
651  regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]);
652  if (regpp)
653  {
654  /* Is this a pairwise SVM segment? */
655  if ((*regpp)->vlib_rp != svm)
656  {
657  int i;
658  svm_region_t *dead_rp = (*regpp)->vlib_rp;
659  /* Note: account for the memfd header page */
660  uword virtual_base = dead_rp->virtual_base - MMAP_PAGESIZE;
661  uword virtual_size = dead_rp->virtual_size + MMAP_PAGESIZE;
662 
663  /* For horizontal scaling, add a hash table... */
664  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
665  if (am->vlib_private_rps[i] == dead_rp)
666  {
667  vec_delete (am->vlib_private_rps, 1, i);
668  goto found;
669  }
670  svm_pop_heap (oldheap);
671  clib_warning ("private rp %llx AWOL", dead_rp);
672  oldheap = svm_push_data_heap (svm);
673 
674  found:
675  /* Kill it, accounting for the memfd header page */
676  svm_pop_heap (oldheap);
677  if (munmap ((void *) virtual_base, virtual_size) < 0)
678  clib_unix_warning ("munmap");
679  /* Reset the queue-length-address cache */
681  oldheap = svm_push_data_heap (svm);
682  }
683  else
684  {
685  /* Poison the old registration */
686  clib_memset (*regpp, 0xF3, sizeof (**regpp));
687  clib_mem_free (*regpp);
688  }
689  /* no dangling references, please */
690  *regpp = 0;
691  }
692  else
693  {
694  svm_pop_heap (oldheap);
695  clib_warning ("Duplicate free, client index %d",
696  regpp - am->vl_clients);
697  oldheap = svm_push_data_heap (svm);
698  }
699  }
700 
702 
703  pthread_mutex_unlock (&svm->mutex);
704  svm_pop_heap (oldheap);
705  for (i = 0; i < vec_len (dead_indices); i++)
706  pool_put_index (am->vl_clients, dead_indices[i]);
707  }
708 }
709 
710 static inline int
712  vlib_node_runtime_t * node, svm_queue_t * q)
713 {
714  uword mp;
715  if (!svm_queue_sub2 (q, (u8 *) & mp))
716  {
717  VL_MSG_API_UNPOISON ((void *) mp);
718  vl_msg_api_handler_with_vm_node (am, (void *) mp, vm, node);
719  return 0;
720  }
721  return -1;
722 }
723 
724 int
726 {
727  api_main_t *am = &api_main;
728  return void_mem_api_handle_msg_i (am, vm, node,
730 }
731 
732 int
734 {
735  api_main_t *am = &api_main;
736  int i;
737  uword *tmp, mp;
738 
739  /*
740  * Swap pending and processing vectors, then process the RPCs
741  * Avoid deadlock conditions by construction.
742  */
744  tmp = vm->processing_rpc_requests;
745  vec_reset_length (tmp);
747  vm->pending_rpc_requests = tmp;
749 
750  /*
751  * RPCs are used to reflect function calls to thread 0
752  * when the underlying code is not thread-safe.
753  *
754  * Grabbing the thread barrier across a set of RPCs
755  * greatly increases efficiency, and avoids
756  * running afoul of the barrier sync holddown timer.
757  * The barrier sync code supports recursive locking.
758  *
759  * We really need to rewrite RPC-based code...
760  */
762  {
764  for (i = 0; i < vec_len (vm->processing_rpc_requests); i++)
765  {
766  mp = vm->processing_rpc_requests[i];
767  vl_msg_api_handler_with_vm_node (am, (void *) mp, vm, node);
768  }
770  }
771 
772  return 0;
773 }
774 
775 int
777  u32 reg_index)
778 {
779  api_main_t *am = &api_main;
780  vl_shmem_hdr_t *save_shmem_hdr = am->shmem_hdr;
781  svm_region_t *vlib_rp, *save_vlib_rp = am->vlib_rp;
782  svm_queue_t *q;
783  int rv;
784 
785  vlib_rp = am->vlib_rp = am->vlib_private_rps[reg_index];
786 
787  am->shmem_hdr = (void *) vlib_rp->user_ctx;
788  q = am->shmem_hdr->vl_input_queue;
789 
790  rv = void_mem_api_handle_msg_i (am, vm, node, q);
791 
792  am->shmem_hdr = save_shmem_hdr;
793  am->vlib_rp = save_vlib_rp;
794 
795  return rv;
796 }
797 
800 {
801  vl_api_registration_t **regpp;
802  vl_api_registration_t *regp;
803  api_main_t *am = &api_main;
804  vl_shmem_hdr_t *shmem_hdr;
805  u32 index;
806 
807  index = vl_msg_api_handle_get_index (handle);
808  regpp = am->vl_clients + index;
809 
810  if (pool_is_free (am->vl_clients, regpp))
811  {
813  return 0;
814  }
815  regp = *regpp;
816 
817  shmem_hdr = (vl_shmem_hdr_t *) regp->shmem_hdr;
818  if (!vl_msg_api_handle_is_valid (handle, shmem_hdr->application_restarts))
819  {
821  return 0;
822  }
823 
824  return (regp);
825 }
826 
827 svm_queue_t *
829 {
830  vl_api_registration_t *regp;
831  api_main_t *am = &api_main;
832 
833  /* Special case: vlib trying to send itself a message */
834  if (index == (u32) ~ 0)
835  return (am->shmem_hdr->vl_input_queue);
836 
838  if (!regp)
839  return 0;
840  return (regp->vl_input_queue);
841 }
842 
843 static clib_error_t *
845 {
846  atexit (vl_unmap_shmem);
847  return 0;
848 }
849 
851 
852 u8 *
853 format_api_message_rings (u8 * s, va_list * args)
854 {
855  api_main_t *am = va_arg (*args, api_main_t *);
856  vl_shmem_hdr_t *shmem_hdr = va_arg (*args, vl_shmem_hdr_t *);
857  int main_segment = va_arg (*args, int);
858  ring_alloc_t *ap;
859  int i;
860 
861  if (shmem_hdr == 0)
862  return format (s, "%8s %8s %8s %8s %8s\n",
863  "Owner", "Size", "Nitems", "Hits", "Misses");
864 
865  ap = shmem_hdr->vl_rings;
866 
867  for (i = 0; i < vec_len (shmem_hdr->vl_rings); i++)
868  {
869  s = format (s, "%8s %8d %8d %8d %8d\n",
870  "vlib", ap->size, ap->nitems, ap->hits, ap->misses);
871  ap++;
872  }
873 
874  ap = shmem_hdr->client_rings;
875 
876  for (i = 0; i < vec_len (shmem_hdr->client_rings); i++)
877  {
878  s = format (s, "%8s %8d %8d %8d %8d\n",
879  "clnt", ap->size, ap->nitems, ap->hits, ap->misses);
880  ap++;
881  }
882 
883  if (main_segment)
884  {
885  s = format (s, "%d ring miss fallback allocations\n", am->ring_misses);
886  s = format
887  (s,
888  "%d application restarts, %d reclaimed msgs, %d garbage collects\n",
889  shmem_hdr->application_restarts, shmem_hdr->restart_reclaims,
890  shmem_hdr->garbage_collects);
891  }
892  return s;
893 }
894 
895 static clib_error_t *
897  unformat_input_t * input, vlib_cli_command_t * cli_cmd)
898 {
899  int i;
900  vl_shmem_hdr_t *shmem_hdr;
901  api_main_t *am = &api_main;
902 
903  /* First, dump the primary region rings.. */
904 
905  if (am->vlib_primary_rp == 0 || am->vlib_primary_rp->user_ctx == 0)
906  {
907  vlib_cli_output (vm, "Shared memory segment not initialized...\n");
908  return 0;
909  }
910 
911  shmem_hdr = (void *) am->vlib_primary_rp->user_ctx;
912 
913  vlib_cli_output (vm, "Main API segment rings:");
914 
916  0 /* print header */ , 0 /* notused */ );
917 
919  shmem_hdr, 1 /* main segment */ );
920 
921  for (i = 0; i < vec_len (am->vlib_private_rps); i++)
922  {
923  svm_region_t *vlib_rp = am->vlib_private_rps[i];
924  shmem_hdr = (void *) vlib_rp->user_ctx;
925  vl_api_registration_t **regpp;
926  vl_api_registration_t *regp = 0;
927 
928  /* For horizontal scaling, add a hash table... */
929  /* *INDENT-OFF* */
930  pool_foreach (regpp, am->vl_clients,
931  ({
932  regp = *regpp;
933  if (regp && regp->vlib_rp == vlib_rp)
934  {
935  vlib_cli_output (vm, "%s segment rings:", regp->name);
936  goto found;
937  }
938  }));
939  vlib_cli_output (vm, "regp %llx not found?", regp);
940  continue;
941  /* *INDENT-ON* */
942  found:
944  0 /* print header */ , 0 /* notused */ );
946  shmem_hdr, 0 /* main segment */ );
947  }
948 
949  return 0;
950 }
951 
952 /*?
953  * Display binary api message allocation ring statistics
954 ?*/
955 /* *INDENT-OFF* */
956 VLIB_CLI_COMMAND (cli_show_api_ring_command, static) =
957 {
958  .path = "show api ring-stats",
959  .short_help = "Message ring statistics",
960  .function = vl_api_ring_command,
961 };
962 /* *INDENT-ON* */
963 
964 clib_error_t *
966 {
967  api_main_t *am = &api_main;
968  svm_map_region_args_t _a, *a = &_a;
969  u8 *remove_path1, *remove_path2;
970  void vlibsocket_reference (void);
971 
973 
974  /*
975  * By popular request / to avoid support fires, remove any old api segment
976  * files Right Here.
977  */
978  if (am->root_path == 0)
979  {
980  remove_path1 = format (0, "/dev/shm/global_vm%c", 0);
981  remove_path2 = format (0, "/dev/shm/vpe-api%c", 0);
982  }
983  else
984  {
985  remove_path1 = format (0, "/dev/shm/%s-global_vm%c", am->root_path, 0);
986  remove_path2 = format (0, "/dev/shm/%s-vpe-api%c", am->root_path, 0);
987  }
988 
989  (void) unlink ((char *) remove_path1);
990  (void) unlink ((char *) remove_path2);
991 
992  vec_free (remove_path1);
993  vec_free (remove_path2);
994 
995  clib_memset (a, 0, sizeof (*a));
996  a->root_path = am->root_path;
998  a->baseva = (am->global_baseva != 0) ?
1000  a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE;
1001  a->flags = SVM_FLAGS_NODATA;
1002  a->uid = am->api_uid;
1003  a->gid = am->api_gid;
1004  a->pvt_heap_size =
1005  (am->global_pvt_heap_size !=
1007 
1009 
1010  return 0;
1011 }
1012 
1013 void
1015 {
1016  api_main_t *am = &api_main;
1017  am->region_name = name;
1018 }
1019 
1020 /*
1021  * fd.io coding-style-patch-verification: ON
1022  *
1023  * Local Variables:
1024  * eval: (c-set-style "gnu")
1025  * End:
1026  */
int vl_mem_api_handle_msg_private(vlib_main_t *vm, vlib_node_runtime_t *node, u32 reg_index)
Definition: memory_api.c:776
int vl_mem_api_handle_msg_main(vlib_main_t *vm, vlib_node_runtime_t *node)
Definition: memory_api.c:725
void vlibsocket_reference()
Definition: socket_api.c:831
uword * pending_rpc_requests
Definition: main.h:267
static u32 vl_msg_api_handle_get_index(u32 index)
Definition: memory_api.h:48
void vl_api_memclnt_create_t_handler(vl_api_memclnt_create_t *mp)
Definition: memory_api.c:158
u8 * name
Client name.
Definition: api_common.h:53
#define SVM_GLOBAL_REGION_NAME
Definition: svm_common.h:101
const char * root_path
Definition: svm_common.h:67
static void * vl_api_memclnt_create_t_print(vl_api_memclnt_create_t *a, void *handle)
Definition: memory_api.c:42
static void vl_api_memclnt_keepalive_t_handler(vl_api_memclnt_keepalive_t *mp)
We can send ourselves these messages if someone uses the builtin binary api test tool...
Definition: memory_api.c:408
static void svm_pop_heap(void *oldheap)
Definition: svm.h:94
clib_spinlock_t pending_rpc_lock
Definition: main.h:269
#define vl_print(handle,...)
Definition: memory_api.c:31
int svm_queue_add(svm_queue_t *q, u8 *elem, int nowait)
Definition: queue.c:239
u32 vl_api_memclnt_create_internal(char *name, svm_queue_t *q)
Definition: memory_api.c:120
a
Definition: bitmap.h:538
#define SVM_FLAGS_NODATA
Definition: svm_common.h:29
static void send_memclnt_keepalive(vl_api_registration_t *regp, f64 now)
Definition: memory_api.c:503
int vl_map_shmem(const char *region_name, int is_vlib)
uword * processing_rpc_requests
Definition: main.h:268
volatile int ** vl_api_queue_cursizes
Definition: memory_api.c:61
u32 application_restarts
Definition: memory_shared.h:95
#define PREDICT_TRUE(x)
Definition: clib.h:112
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:279
void vl_unmap_shmem(void)
int svm_queue_sub2(svm_queue_t *q, u8 *elem)
Definition: queue.c:413
u8 * message_bounce
Don&#39;t automatically free message buffer vetor.
Definition: api_common.h:243
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:110
Message configuration definition.
Definition: api_common.h:120
svm_region_t * vlib_primary_rp
Primary api segment descriptor.
Definition: api_common.h:279
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
uword virtual_base
Definition: svm_common.h:42
#define SVM_PVT_MHEAP_SIZE
Definition: svm_common.h:32
int i
int api_uid
uid for the api shared memory region
Definition: api_common.h:304
ring_alloc_t * client_rings
Definition: memory_shared.h:92
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
#define pool_is_free(P, E)
Use free bitmap to query whether given element is free.
Definition: pool.h:276
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:237
void * vl_msg_api_alloc(int nbytes)
int api_gid
gid for the api shared memory region
Definition: api_common.h:307
unsigned char u8
Definition: types.h:56
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
_vl_msg_api_function_list_elt_t * reaper_function_registrations
List of API client reaper functions.
Definition: api_common.h:361
u8 * format_api_message_rings(u8 *s, va_list *args)
Definition: memory_api.c:853
u32 clib_file_index
Socket only: file index.
Definition: api_common.h:66
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:498
void vl_msg_api_barrier_sync(void)
Definition: api_shared.c:416
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
u32 ring_misses
Number of times that the ring allocator failed.
Definition: api_common.h:252
static void * svm_push_data_heap(svm_region_t *rp)
Definition: svm.h:86
vl_api_registration_t ** vl_clients
vlib/vpp only: vector of client registrations
Definition: api_common.h:289
vl_api_registration_t * vl_mem_api_client_index_to_registration(u32 handle)
Definition: memory_api.c:799
volatile void * user_ctx
Definition: svm_common.h:47
static clib_error_t * vl_api_ring_command(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cli_cmd)
Definition: memory_api.c:896
static u32 vl_msg_api_handle_from_index_and_epoch(u32 index, u32 epoch)
Definition: memory_api.h:54
void vl_api_memclnt_keepalive_reply_t_handler(vl_api_memclnt_keepalive_reply_t *mp)
client answered a ping, stave off the grim reaper...
Definition: memory_api.c:388
void vl_mem_api_dead_client_scan(api_main_t *am, vl_shmem_hdr_t *shm, f64 now)
Definition: memory_api.c:599
vlib_node_registration_t vl_api_clnt_node
(constructor) VLIB_REGISTER_NODE (vl_api_clnt_node)
Definition: vlib_api.c:415
const char * root_path
Chroot path to the shared memory API files.
Definition: api_common.h:352
#define clib_error_return(e, args...)
Definition: error.h:99
svm_region_t * vlib_rp
Current binary api segment descriptor.
Definition: api_common.h:276
struct vl_shmem_hdr_ * shmem_hdr
Binary API shared-memory segment header pointer.
Definition: api_common.h:286
unsigned int u32
Definition: types.h:88
void vl_msg_api_send_shmem(svm_queue_t *q, u8 *elem)
static void vlib_set_queue_signal_callback(vlib_main_t *vm, void(*fp)(vlib_main_t *))
Definition: main.h:380
vl_registration_type_t registration_type
type
Definition: api_common.h:48
char * name
Definition: main.h:140
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:519
#define foreach_vlib_api_msg
Definition: memory_api.c:429
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:934
svm_queue_t * vl_input_queue
shared memory only: pointer to client input queue
Definition: api_common.h:61
struct _unformat_input_t unformat_input_t
svm_region_t ** vlib_private_rps
Vector of all mapped shared-VM segments.
Definition: api_common.h:282
#define PREDICT_FALSE(x)
Definition: clib.h:111
uword virtual_size
Definition: svm_common.h:43
void svm_region_init_args(svm_map_region_args_t *a)
Definition: svm.c:894
#define SVM_GLOBAL_REGION_SIZE
Definition: svm_common.h:100
void vl_msg_api_handler_with_vm_node(api_main_t *am, void *the_msg, vlib_main_t *vm, vlib_node_runtime_t *node)
Definition: api_shared.c:528
clib_error_t * vlibmemory_init(vlib_main_t *vm)
Definition: memory_api.c:965
volatile u32 queue_signal_pending
Definition: main.h:231
int vl_mem_api_handle_rpc(vlib_main_t *vm, vlib_node_runtime_t *node)
Definition: memory_api.c:733
clib_error_t * map_api_segment_init(vlib_main_t *vm)
Definition: memory_api.c:489
API main structure, used by both vpp and binary API clients.
Definition: api_common.h:223
u64 global_size
size of the global VM region
Definition: api_common.h:313
An API client registration, only in vpp/vlib.
Definition: api_common.h:46
Shared memory connection.
Definition: api_common.h:38
ring_alloc_t * vl_rings
Definition: memory_shared.h:89
svmdb_client_t * c
static_always_inline uword vlib_get_thread_index(void)
Definition: threads.h:213
vlib_main_t * vm
Definition: buffer.c:323
svm_queue_t * vl_api_client_index_to_input_queue(u32 index)
Definition: memory_api.c:828
u64 global_baseva
base virtual address for global VM region
Definition: api_common.h:310
static clib_error_t * setup_memclnt_exit(vlib_main_t *vm)
Definition: memory_api.c:844
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:342
#define clib_warning(format, args...)
Definition: error.h:59
static void * vl_api_memclnt_delete_t_print(vl_api_memclnt_delete_t *a, void *handle)
Definition: memory_api.c:53
static int void_mem_api_handle_msg_i(api_main_t *am, vlib_main_t *vm, vlib_node_runtime_t *node, svm_queue_t *q)
Definition: memory_api.c:711
static vl_api_registration_t * vl_api_client_index_to_registration(u32 index)
Definition: api.h:57
svm_queue_t * vl_input_queue
Definition: memory_shared.h:84
u8 name[64]
Definition: ip.api:42
static CLIB_NOSANITIZE_ADDR void VL_MSG_API_SVM_QUEUE_UNPOISON(const svm_queue_t *q)
Definition: api_common.h:153
void svm_queue_free(svm_queue_t *q)
Definition: queue.c:89
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:161
static u32 vl_msg_api_handle_get_epoch(u32 index)
Definition: memory_api.h:42
#define pool_put_index(p, i)
Free pool element with given index.
Definition: pool.h:316
#define ASSERT(truth)
int vl_mem_api_init(const char *region_name)
Definition: memory_api.c:439
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:785
static u8 vl_msg_api_handle_is_valid(u32 handle, u32 restarts)
Definition: memory_api.h:64
static void vl_mem_send_client_keepalive_w_reg(api_main_t *am, f64 now, vl_api_registration_t **regpp, u32 **dead_indices, u32 **confused_indices)
Definition: memory_api.c:555
static void clib_mem_free(void *p)
Definition: mem.h:226
void vl_set_memory_region_name(const char *name)
Definition: memory_api.c:1014
u64 svm_get_global_region_base_va()
Definition: svm.c:62
u8 * serialized_message_table_in_shmem
vlib/vpp only: serialized (message, name, crc) table
Definition: api_common.h:292
u8 * vl_api_serialize_message_table(api_main_t *am, u8 *vector)
Definition: api_shared.c:193
#define clib_error_report(e)
Definition: error.h:113
u64 global_pvt_heap_size
size of the global VM private mheap
Definition: api_common.h:319
void vl_api_memclnt_delete_t_handler(vl_api_memclnt_delete_t *mp)
Definition: memory_api.c:264
static void * clib_mem_alloc(uword size)
Definition: mem.h:153
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
void vl_msg_api_free(void *)
#define MMAP_PAGESIZE
Definition: ssvm.h:43
#define VL_API_EPOCH_MASK
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
const char * name
Definition: svm_common.h:68
static void memclnt_queue_callback(vlib_main_t *vm)
Definition: memory_api.c:64
u64 uword
Definition: types.h:112
#define clib_unix_warning(format, args...)
Definition: error.h:68
u32 vl_api_registration_pool_index
Index in VLIB&#39;s brain (not shared memory).
Definition: api_common.h:51
void svm_client_scan_this_region_nolock(svm_region_t *rp)
Definition: svm.c:1240
volatile u32 api_queue_nonempty
Definition: main.h:232
struct _svm_queue svm_queue_t
u8 * is_mp_safe
Message is mp safe vector.
Definition: api_common.h:246
void vl_msg_api_increment_missing_client_counter(void)
Definition: api_shared.c:44
int vl_api_call_reaper_functions(u32 client_index)
Definition: memory_api.c:244
void * vl_msg_api_alloc_as_if_client(int nbytes)
const char * region_name
Shared VM binary API region name.
Definition: api_common.h:349
void vl_msg_api_barrier_release(void)
Definition: api_shared.c:421
api_main_t api_main
Definition: api_shared.c:35
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:772
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:95
svm_region_t * vlib_rp
Definition: api_common.h:62
pthread_mutex_t mutex
Definition: svm_common.h:37
static CLIB_NOSANITIZE_ADDR void VL_MSG_API_UNPOISON(const void *a)
Definition: api_common.h:146