FD.io VPP  v19.08.1-401-g8e4ed521a
Vector Packet Processing
fifo_segment.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016-2019 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <svm/fifo_segment.h>
17 
18 /**
19  * Fifo segment free space
20  *
21  * Queries the underlying memory manager, dlmalloc, for free space. Since this
22  * ends up walking the internal data structures, it should not be called
23  * indiscriminately.
24  *
25  * @param fs fifo segment
26  * @return number of free bytes
27  */
28 static u32
30 {
31  struct dlmallinfo dlminfo;
32 
33  dlminfo = mspace_mallinfo (fs->ssvm.sh->heap);
34  return dlminfo.fordblks;
35 }
36 
37 /**
38  * Initialize fifo segment shared header
39  */
40 int
42 {
45  void *oldheap;
46 
47  sh = fs->ssvm.sh;
48  oldheap = ssvm_push_heap (sh);
49 
50  fsh = clib_mem_alloc (sizeof (*fsh));
51  clib_memset (fsh, 0, sizeof (*fsh));
52  fs->h = sh->opaque[0] = fsh;
53 
54  ssvm_pop_heap (oldheap);
55 
56  fsh->n_free_bytes = fs_free_space (fs);
57  sh->ready = 1;
58  return (0);
59 }
60 
61 /**
62  * Create a fifo segment and initialize as master
63  */
64 int
66 {
67  fifo_segment_t *fs;
68  uword baseva;
69  int rv;
70 
71  /* Allocate a fresh segment */
72  pool_get_zero (sm->segments, fs);
73 
74  baseva = a->segment_type == SSVM_SEGMENT_PRIVATE ? ~0ULL : sm->next_baseva;
75  fs->ssvm.ssvm_size = a->segment_size;
76  fs->ssvm.i_am_master = 1;
77  fs->ssvm.my_pid = getpid ();
78  fs->ssvm.name = format (0, "%s%c", a->segment_name, 0);
79  fs->ssvm.requested_va = baseva;
80 
81  if ((rv = ssvm_master_init (&fs->ssvm, a->segment_type)))
82  {
83  pool_put (sm->segments, fs);
84  return (rv);
85  }
86 
87  /* Note: requested_va updated due to seg base addr randomization */
88  sm->next_baseva = fs->ssvm.sh->ssvm_va + fs->ssvm.ssvm_size;
89 
90  fifo_segment_init (fs);
91  vec_add1 (a->new_segment_indices, fs - sm->segments);
92  return (0);
93 }
94 
95 /**
96  * Attach as slave to a fifo segment
97  */
98 int
100 {
101  fifo_segment_t *s;
102  int rv;
103 
104  pool_get_zero (sm->segments, s);
105 
106  s->ssvm.ssvm_size = a->segment_size;
107  s->ssvm.my_pid = getpid ();
108  s->ssvm.name = format (0, "%s%c", a->segment_name, 0);
109  s->ssvm.requested_va = sm->next_baseva;
111  s->ssvm.fd = a->memfd_fd;
112  else
114 
115  if ((rv = ssvm_slave_init (&s->ssvm, a->segment_type)))
116  {
117  _vec_len (s) = vec_len (s) - 1;
118  return (rv);
119  }
120 
121  /* Fish the segment header */
122  s->h = s->ssvm.sh->opaque[0];
123 
124  vec_add1 (a->new_segment_indices, s - sm->segments);
125  return (0);
126 }
127 
128 void
130 {
131  ssvm_delete (&s->ssvm);
132  clib_memset (s, 0xfe, sizeof (*s));
133  pool_put (sm->segments, s);
134 }
135 
136 u32
138 {
139  return s - sm->segments;
140 }
141 
142 void *
144 {
145  return seg->ssvm.sh->heap;
146 }
147 
150 {
151  return pool_elt_at_index (sm->segments, segment_index);
152 }
153 
154 void
155 fifo_segment_info (fifo_segment_t * seg, char **address, size_t * size)
156 {
157  *address = (char *) seg->ssvm.sh->ssvm_va;
158  *size = seg->ssvm.ssvm_size;
159 }
160 
161 void
163  u32 timeout_in_seconds)
164 {
165  sm->next_baseva = baseva;
166  sm->timeout_in_seconds = timeout_in_seconds;
167 }
168 
169 static inline u32
171 {
172  return max_log2 (size) - max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE);
173 }
174 
175 static inline u32
177 {
178  return 1 << (fl_index + max_log2 (FIFO_SEGMENT_MIN_FIFO_SIZE));
179 }
180 
181 static inline int
183 {
184  /*
185  * 4K minimum. It's not likely that anything good will happen
186  * with a smaller FIFO.
187  */
188  return size >= FIFO_SEGMENT_MIN_FIFO_SIZE
189  && size <= FIFO_SEGMENT_MAX_FIFO_SIZE;
190 }
191 
192 static svm_fifo_t *
193 fs_try_alloc_fifo_freelist (fifo_segment_t * fs, u32 fl_index, u32 data_bytes)
194 {
195  fifo_segment_header_t *fsh = fs->h;
197  svm_fifo_t *f;
198 
199  f = fsh->free_fifos;
200  c = fsh->free_chunks[fl_index];
201 
202  if (!f || !c)
203  return 0;
204 
205  fsh->free_fifos = f->next;
206  fsh->free_chunks[fl_index] = c->next;
207  c->next = c;
208  c->start_byte = 0;
209  c->length = data_bytes;
210  memset (f, 0, sizeof (*f));
211  f->start_chunk = c;
212  f->end_chunk = c;
213 
214  fsh->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index);
215  return f;
216 }
217 
218 static svm_fifo_t *
220 {
221  svm_fifo_chunk_t *c, *first = 0, *last = 0;
222  fifo_segment_header_t *fsh = fs->h;
223  u32 fl_index, fl_size, n_alloc = 0;
224  svm_fifo_t *f;
225 
226  f = fsh->free_fifos;
227  if (!f)
228  {
229  void *oldheap = ssvm_push_heap (fs->ssvm.sh);
231  ssvm_pop_heap (oldheap);
232  if (!f)
233  return 0;
234  memset (f, 0, sizeof (*f));
235  fsh->n_free_bytes -= sizeof (*f);
236  }
237  else
238  {
239  fsh->free_fifos = f->next;
240  }
241 
242  fl_index = fs_freelist_for_size (data_bytes) - 1;
243  vec_validate_init_empty (fsh->free_chunks, fl_index, 0);
244  fl_size = fs_freelist_index_to_size (fl_index);
245 
246  while (data_bytes)
247  {
248  c = fsh->free_chunks[fl_index];
249  if (c)
250  {
251  fsh->free_chunks[fl_index] = c->next;
252  if (!last)
253  last = c;
254  c->next = first;
255  first = c;
256  n_alloc += fl_size;
257  c->length = clib_min (fl_size, data_bytes);
258  data_bytes -= c->length;
259  }
260  else
261  {
262  ASSERT (fl_index > 0);
263  fl_index -= 1;
264  fl_size = fl_size >> 1;
265  }
266  }
267  f->start_chunk = first;
268  f->end_chunk = last;
269  last->next = first;
270  fsh->n_fl_chunk_bytes -= n_alloc;
271  return f;
272 }
273 
274 static int
275 fs_try_alloc_fifo_batch (fifo_segment_t * fs, u32 fl_index, u32 batch_size)
276 {
277  fifo_segment_header_t *fsh = fs->h;
278  u32 size, hdrs, rounded_data_size;
280  svm_fifo_t *f;
281  void *oldheap;
282  u8 *fmem;
283  int i;
284 
285  vec_validate_init_empty (fsh->free_chunks, fl_index, 0);
286  rounded_data_size = fs_freelist_index_to_size (fl_index);
287  hdrs = sizeof (*f) + sizeof (*c);
288  size = (hdrs + rounded_data_size) * batch_size;
289 
290  oldheap = ssvm_push_heap (fs->ssvm.sh);
292  0 /* align_offset */ ,
293  0 /* os_out_of_memory */ );
294  ssvm_pop_heap (oldheap);
295 
296  /* Out of space.. */
297  if (fmem == 0)
298  return -1;
299 
300  /* Carve fifo + chunk space */
301  for (i = 0; i < batch_size; i++)
302  {
303  f = (svm_fifo_t *) fmem;
304  memset (f, 0, sizeof (*f));
305  f->next = fsh->free_fifos;
306  fsh->free_fifos = f;
307  c = (svm_fifo_chunk_t *) (fmem + sizeof (*f));
308  c->start_byte = 0;
309  c->length = rounded_data_size;
310  c->next = fsh->free_chunks[fl_index];
311  fsh->free_chunks[fl_index] = c;
312  fmem += hdrs + rounded_data_size;
313  }
314 
315  fsh->n_fl_chunk_bytes += batch_size * rounded_data_size;
316  fsh->n_free_bytes -= size;
317 
318  return 0;
319 }
320 
321 /**
322  * Try to allocate new fifo
323  *
324  * Tries the following steps in order:
325  * - grab fifo and chunk from freelists
326  * - batch fifo and chunk allocation
327  * - single fifo allocation
328  * - grab multiple fifo chunks from freelists
329  */
330 static svm_fifo_t *
332 {
333  fifo_segment_header_t *fsh = fs->h;
334  u32 fifo_sz, fl_index;
335  svm_fifo_t *f = 0;
336 
337  fl_index = fs_freelist_for_size (data_bytes);
338  vec_validate_init_empty (fsh->free_chunks, fl_index, 0);
339  fifo_sz = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t);
340  fifo_sz += 1 << max_log2 (data_bytes);
341 
342  if (fsh->free_fifos && fsh->free_chunks[fl_index])
343  {
344  f = fs_try_alloc_fifo_freelist (fs, fl_index, data_bytes);
345  if (f)
346  goto done;
347  }
348  if (fifo_sz * FIFO_SEGMENT_ALLOC_BATCH_SIZE < fsh->n_free_bytes)
349  {
350  if (fs_try_alloc_fifo_batch (fs, fl_index,
352  goto done;
353 
354  f = fs_try_alloc_fifo_freelist (fs, fl_index, data_bytes);
355  goto done;
356  }
357  if (fifo_sz <= fsh->n_free_bytes)
358  {
359  void *oldheap = ssvm_push_heap (fs->ssvm.sh);
360  f = svm_fifo_create (data_bytes);
361  ssvm_pop_heap (oldheap);
362  if (f)
363  {
364  fsh->n_free_bytes -= fifo_sz;
365  goto done;
366  }
367  }
368  if (data_bytes <= fsh->n_fl_chunk_bytes)
369  f = fs_try_alloc_fifo_freelist_multi_chunk (fs, data_bytes);
370 
371 done:
372 
373  return f;
374 }
375 
376 /**
377  * Allocate fifo in fifo segment
378  */
379 svm_fifo_t *
381  fifo_segment_ftype_t ftype)
382 {
384  svm_fifo_t *f = 0;
385 
386  if (!fs_chunk_size_is_valid (data_bytes))
387  {
388  clib_warning ("fifo size out of range %d", data_bytes);
389  return 0;
390  }
391 
392  fsh = fs->h;
394 
395  f = fs_try_alloc_fifo (fs, data_bytes);
396  if (!f)
397  goto done;
398 
399  /* (re)initialize the fifo, as in svm_fifo_create */
400  svm_fifo_init (f, data_bytes);
401 
402  /* Initialize chunks and rbtree for multi-chunk fifos */
403  if (f->start_chunk->next != f->start_chunk)
404  {
405  void *oldheap = ssvm_push_heap (fs->ssvm.sh);
407  ssvm_pop_heap (oldheap);
408  }
409 
410  /* If rx fifo type add to active fifos list. When cleaning up segment,
411  * we need a list of active sessions that should be disconnected. Since
412  * both rx and tx fifos keep pointers to the session, it's enough to track
413  * only one. */
414  if (ftype == FIFO_SEGMENT_RX_FIFO)
415  {
416  if (fsh->fifos)
417  {
418  fsh->fifos->prev = f;
419  f->next = fsh->fifos;
420  }
421  fsh->fifos = f;
422  f->flags |= SVM_FIFO_F_LL_TRACKED;
423  }
424  fsh->n_active_fifos++;
425 
426 done:
428  return (f);
429 }
430 
431 /**
432  * Free fifo allocated in fifo segment
433  */
434 void
436 {
437  svm_fifo_chunk_t *cur, *next;
440  void *oldheap;
441  int fl_index;
442 
443  ASSERT (f->refcnt > 0);
444 
445  if (--f->refcnt > 0)
446  return;
447 
448  sh = fs->ssvm.sh;
449  fsh = fs->h;
450 
451  ssvm_lock_non_recursive (sh, 2);
452 
453  /* Remove from active list. Only rx fifos are tracked */
454  if (f->flags & SVM_FIFO_F_LL_TRACKED)
455  {
456  if (f->prev)
457  f->prev->next = f->next;
458  else
459  fsh->fifos = f->next;
460  if (f->next)
461  f->next->prev = f->prev;
462  f->flags &= ~SVM_FIFO_F_LL_TRACKED;
463  }
464 
465  /* Add to free list */
466  f->next = fsh->free_fifos;
467  f->prev = 0;
468  fsh->free_fifos = f;
469 
470  /* Free fifo chunks */
471  cur = f->start_chunk;
472  do
473  {
474  next = cur->next;
475  fl_index = fs_freelist_for_size (cur->length);
476  ASSERT (fl_index < vec_len (fsh->free_chunks));
477  cur->next = fsh->free_chunks[fl_index];
478  fsh->free_chunks[fl_index] = cur;
479  fsh->n_fl_chunk_bytes += fs_freelist_index_to_size (fl_index);
480  cur = next;
481  }
482  while (cur != f->start_chunk);
483 
484  f->start_chunk = f->end_chunk = f->new_chunks = 0;
485  f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0;
486 
487  oldheap = ssvm_push_heap (sh);
489  ssvm_pop_heap (oldheap);
490 
491  /* not allocated on segment heap */
493 
494  if (CLIB_DEBUG)
495  {
496  f->master_session_index = ~0;
497  f->master_thread_index = ~0;
498  }
499 
500  fsh->n_active_fifos--;
502 }
503 
504 int
506 {
507  fifo_segment_header_t *fsh = fs->h;
508  svm_fifo_t *f;
509  void *oldheap;
510  u32 size;
511  u8 *fmem;
512  int i;
513 
514  size = (sizeof (*f)) * batch_size;
515 
516  oldheap = ssvm_push_heap (fs->ssvm.sh);
518  0 /* align_offset */ ,
519  0 /* os_out_of_memory */ );
520  ssvm_pop_heap (oldheap);
521 
522  /* Out of space.. */
523  if (fmem == 0)
524  return -1;
525 
526  /* Carve fifo + chunk space */
527  for (i = 0; i < batch_size; i++)
528  {
529  f = (svm_fifo_t *) fmem;
530  memset (f, 0, sizeof (*f));
531  f->next = fsh->free_fifos;
532  fsh->free_fifos = f;
533  fmem += sizeof (*f);
534  }
535 
536  fsh->n_free_bytes -= size;
537 
538  return 0;
539 }
540 
541 int
543  u32 batch_size)
544 {
545  fifo_segment_header_t *fsh = fs->h;
546  u32 size, rounded_data_size, fl_index;
548  void *oldheap;
549  u8 *cmem;
550  int i;
551 
552  if (!fs_chunk_size_is_valid (chunk_size))
553  {
554  clib_warning ("chunk size out of range %d", chunk_size);
555  return -1;
556  }
557 
558  fl_index = fs_freelist_for_size (chunk_size);
559  vec_validate_init_empty (fsh->free_chunks, fl_index, 0);
560  rounded_data_size = fs_freelist_index_to_size (fl_index);
561  size = (sizeof (*c) + rounded_data_size) * batch_size;
562 
563  oldheap = ssvm_push_heap (fs->ssvm.sh);
565  0 /* align_offset */ ,
566  0 /* os_out_of_memory */ );
567  ssvm_pop_heap (oldheap);
568 
569  /* Out of space.. */
570  if (cmem == 0)
571  return -1;
572 
573  /* Carve fifo + chunk space */
574  for (i = 0; i < batch_size; i++)
575  {
576  c = (svm_fifo_chunk_t *) cmem;
577  c->start_byte = 0;
578  c->length = rounded_data_size;
579  c->next = fsh->free_chunks[fl_index];
580  fsh->free_chunks[fl_index] = c;
581  cmem += sizeof (*c) + rounded_data_size;
582  }
583 
584  fsh->n_fl_chunk_bytes += batch_size * rounded_data_size;
585  fsh->n_free_bytes -= size;
586 
587  return 0;
588 }
589 
590 /**
591  * Pre-allocates fifo pairs in fifo segment
592  */
593 void
595  u32 rx_fifo_size, u32 tx_fifo_size,
596  u32 * n_fifo_pairs)
597 {
598  u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc;
599  int rx_fl_index, tx_fl_index;
600  uword space_available;
601  u32 hdrs;
602 
603  /* Parameter check */
604  if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
605  return;
606 
607  if (!fs_chunk_size_is_valid (rx_fifo_size))
608  {
609  clib_warning ("rx fifo_size out of range %d", rx_fifo_size);
610  return;
611  }
612 
613  if (!fs_chunk_size_is_valid (tx_fifo_size))
614  {
615  clib_warning ("tx fifo_size out of range %d", tx_fifo_size);
616  return;
617  }
618 
619  rx_rounded_data_size = (1 << (max_log2 (rx_fifo_size)));
620  rx_fl_index = fs_freelist_for_size (rx_fifo_size);
621  tx_rounded_data_size = (1 << (max_log2 (tx_fifo_size)));
622  tx_fl_index = fs_freelist_for_size (tx_fifo_size);
623 
624  hdrs = sizeof (svm_fifo_t) + sizeof (svm_fifo_chunk_t);
625 
626  /* Calculate space requirements */
627  pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size;
628  space_available = fs_free_space (fs);
629  pairs_to_alloc = space_available / pair_size;
630  pairs_to_alloc = clib_min (pairs_to_alloc, *n_fifo_pairs);
631 
632  if (!pairs_to_alloc)
633  return;
634 
635  if (fs_try_alloc_fifo_batch (fs, rx_fl_index, pairs_to_alloc))
636  clib_warning ("rx prealloc failed: pairs %u", pairs_to_alloc);
637  if (fs_try_alloc_fifo_batch (fs, tx_fl_index, pairs_to_alloc))
638  clib_warning ("tx prealloc failed: pairs %u", pairs_to_alloc);
639 
640  /* Account for the pairs allocated */
641  *n_fifo_pairs -= pairs_to_alloc;
642 }
643 
644 int
646 {
649  void *oldheap;
650  int fl_index;
651 
652  if (!fs_chunk_size_is_valid (chunk_size))
653  {
654  clib_warning ("chunk size out of range %d", chunk_size);
655  return -1;
656  }
657 
658  fl_index = fs_freelist_for_size (chunk_size);
659 
660  sh = fs->ssvm.sh;
661  ssvm_lock_non_recursive (sh, 1);
662 
663  vec_validate_init_empty (fs->h->free_chunks, fl_index, 0);
664  c = fs->h->free_chunks[fl_index];
665 
666  oldheap = ssvm_push_heap (sh);
667 
668  if (!c)
669  {
670  c = svm_fifo_chunk_alloc (chunk_size);
671  if (!c)
672  {
673  ssvm_pop_heap (oldheap);
675  return -1;
676  }
677  fs->h->n_free_bytes -= chunk_size + sizeof (*c);
678  }
679  else
680  {
681  fs->h->free_chunks[fl_index] = c->next;
682  c->next = 0;
683  fs->h->n_fl_chunk_bytes -= fs_freelist_index_to_size (fl_index);
684  }
685 
686  svm_fifo_add_chunk (f, c);
687 
688  ssvm_pop_heap (oldheap);
690  return 0;
691 }
692 
693 int
695 {
696  svm_fifo_chunk_t *cur, *next;
698  void *oldheap;
699  int fl_index;
700 
701  sh = fs->ssvm.sh;
702  ssvm_lock_non_recursive (sh, 1);
703 
704  oldheap = ssvm_push_heap (sh);
705  cur = svm_fifo_collect_chunks (f);
706 
707  while (cur)
708  {
709  next = cur->next;
710  fl_index = fs_freelist_for_size (cur->length);
711  cur->next = fs->h->free_chunks[fl_index];
712  fs->h->free_chunks[fl_index] = cur;
713  cur = next;
714  }
715 
716  ssvm_pop_heap (oldheap);
718 
719  return 0;
720 }
721 
722 /**
723  * Get number of active fifos
724  */
725 u32
727 {
728  return fs->h->n_active_fifos;
729 }
730 
731 u32
733 {
734  fifo_segment_header_t *fsh = fs->h;
735  svm_fifo_t *f;
736  u32 count = 0;
737 
738  f = fsh->free_fifos;
739  if (f == 0)
740  return 0;
741 
742  while (f)
743  {
744  f = f->next;
745  count++;
746  }
747  return count;
748 }
749 
750 u32
752 {
753  u32 count = 0, rounded_size, fl_index;
756  int i;
757 
758  fsh = fs->h;
759 
760  /* Count all free chunks? */
761  if (size == ~0)
762  {
763  for (i = 0; i < vec_len (fsh->free_chunks); i++)
764  {
765  c = fsh->free_chunks[i];
766  if (c == 0)
767  continue;
768 
769  while (c)
770  {
771  c = c->next;
772  count++;
773  }
774  }
775  return count;
776  }
777 
778  rounded_size = (1 << (max_log2 (size)));
779  fl_index = fs_freelist_for_size (rounded_size);
780 
781  if (fl_index >= vec_len (fsh->free_chunks))
782  return 0;
783 
784  c = fsh->free_chunks[fl_index];
785  if (c == 0)
786  return 0;
787 
788  while (c)
789  {
790  c = c->next;
791  count++;
792  }
793  return count;
794 }
795 
796 void
798 {
799  fs->h->n_free_bytes = fs_free_space (fs);
800 }
801 
802 u32
804 {
805  return fs->h->n_free_bytes;
806 }
807 
808 u32
810 {
811  return fs->h->n_fl_chunk_bytes;
812 }
813 
814 u8
816 {
817  return fs->h->fifos != 0;
818 }
819 
820 svm_fifo_t *
822 {
823  return fs->h->fifos;
824 }
825 
826 u8 *
827 format_fifo_segment_type (u8 * s, va_list * args)
828 {
829  fifo_segment_t *sp;
830  sp = va_arg (*args, fifo_segment_t *);
831  ssvm_segment_type_t st = ssvm_type (&sp->ssvm);
832 
833  if (st == SSVM_SEGMENT_PRIVATE)
834  s = format (s, "%s", "private-heap");
835  else if (st == SSVM_SEGMENT_MEMFD)
836  s = format (s, "%s", "memfd");
837  else if (st == SSVM_SEGMENT_SHM)
838  s = format (s, "%s", "shm");
839  else
840  s = format (s, "%s", "unknown");
841  return s;
842 }
843 
844 /**
845  * Segment format function
846  */
847 u8 *
848 format_fifo_segment (u8 * s, va_list * args)
849 {
850  u32 count, indent, active_fifos, free_fifos, fifo_hdr = 0, chunk_size;
851  fifo_segment_t *fs = va_arg (*args, fifo_segment_t *);
852  int verbose __attribute__ ((unused)) = va_arg (*args, int);
853  u32 est_chunk_bytes, est_free_seg_bytes;
856  u64 chunk_bytes = 0;
857  char *address;
858  size_t size;
859  int i;
860 
861  indent = format_get_indent (s) + 2;
862 #if USE_DLMALLOC == 0
863  s = format (s, "%U segment heap: %U\n", format_white_space, indent,
864  format_mheap, svm_fifo_segment_heap (fs), verbose);
865  s = format (s, "%U segment has %u active fifos\n",
867 #endif
868 
869  if (fs == 0)
870  {
871  s = format (s, "%-15s%15s%15s%15s%15s%15s", "Name", "Type",
872  "HeapSize (M)", "ActiveFifos", "FreeFifos", "Address");
873  return s;
874  }
875 
876  fsh = fs->h;
877  fifo_segment_info (fs, &address, &size);
878  active_fifos = fifo_segment_num_fifos (fs);
879  free_fifos = fifo_segment_num_free_fifos (fs);
880 
881  s = format (s, "%-15v%15U%15llu%15u%15u%15llx", ssvm_name (&fs->ssvm),
882  format_fifo_segment_type, fs, size >> 20ULL, active_fifos,
883  free_fifos, address);
884 
885  if (!verbose)
886  return s;
887 
888  if (fsh->free_chunks)
889  s = format (s, "\n\n%UFree chunks by size:\n", format_white_space,
890  indent + 2);
891  else
892  s = format (s, "\n");
893 
894  for (i = 0; i < vec_len (fsh->free_chunks); i++)
895  {
896  c = fsh->free_chunks[i];
897  if (c == 0)
898  continue;
899  count = 0;
900  while (c)
901  {
902  c = c->next;
903  count++;
904  }
905 
906  chunk_size = fs_freelist_index_to_size (i);
907  s = format (s, "%U%-5u kB: %u\n", format_white_space, indent + 2,
908  chunk_size >> 10, count);
909 
910  chunk_bytes += count * chunk_size;
911  }
912 
913  fifo_hdr = free_fifos * sizeof (svm_fifo_t);
914  est_chunk_bytes = fifo_segment_fl_chunk_bytes (fs);
915  est_free_seg_bytes = fsh->n_free_bytes;
917 
918  s = format (s, "\n%Useg free bytes: %U (%u) estimated: %U (%u)\n",
921  est_free_seg_bytes, est_free_seg_bytes);
922  s = format (s, "%Uchunk free bytes: %U (%lu) estimated: %U (%u)\n",
923  format_white_space, indent + 2, format_memory_size, chunk_bytes,
924  chunk_bytes, format_memory_size, est_chunk_bytes,
925  est_chunk_bytes);
926  s = format (s, "%Ufifo hdr free bytes: %U (%u)\n", format_white_space,
927  indent + 2, format_memory_size, fifo_hdr, fifo_hdr);
928  s = format (s, "\n");
929 
930  return s;
931 }
932 
933 /*
934  * fd.io coding-style-patch-verification: ON
935  *
936  * Local Variables:
937  * eval: (c-set-style "gnu")
938  * End:
939  */
u32 length
length of chunk in bytes
Definition: svm_fifo.h:61
void fifo_segment_info(fifo_segment_t *seg, char **address, size_t *size)
Definition: fifo_segment.c:155
u64 ssvm_size
Definition: ssvm.h:85
typedef address
Definition: ip_types.api:83
fifo_segment_header_t * h
fifo segment data
Definition: fifo_segment.h:53
#define clib_min(x, y)
Definition: clib.h:295
svm_fifo_chunk_t * svm_fifo_collect_chunks(svm_fifo_t *f)
Removes chunks that are after fifo end byte.
Definition: svm_fifo.c:715
static u32 fs_free_space(fifo_segment_t *fs)
Fifo segment free space.
Definition: fifo_segment.c:29
uword requested_va
Definition: ssvm.h:88
i32 next
Definition: heap.h:78
static void * clib_mem_alloc_aligned_at_offset(uword size, uword align, uword align_offset, int os_out_of_memory_on_failure)
Definition: mem.h:102
a
Definition: bitmap.h:538
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
Definition: pool.h:239
void svm_fifo_free_chunk_lookup(svm_fifo_t *f)
Cleanup fifo chunk lookup rb tree.
Definition: svm_fifo.c:847
volatile u32 ready
Definition: ssvm.h:77
unsigned long u64
Definition: types.h:89
void fifo_segment_delete(fifo_segment_main_t *sm, fifo_segment_t *s)
Definition: fifo_segment.c:129
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 n_active_fifos
Number of active fifos.
Definition: fifo_segment.h:44
void * opaque[SSVM_N_OPAQUE]
Definition: ssvm.h:74
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
void svm_fifo_init(svm_fifo_t *f, u32 size)
Initialize fifo.
Definition: svm_fifo.c:392
int i
static u32 format_get_indent(u8 *s)
Definition: format.h:72
ssvm_shared_header_t * sh
Definition: ssvm.h:84
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static svm_fifo_t * fs_try_alloc_fifo_freelist(fifo_segment_t *fs, u32 fl_index, u32 data_bytes)
Definition: fifo_segment.c:193
DLMALLOC_EXPORT struct dlmallinfo mspace_mallinfo(mspace msp)
static u32 fs_freelist_index_to_size(u32 fl_index)
Definition: fifo_segment.c:176
u32 n_free_bytes
Bytes usable for new allocs.
Definition: fifo_segment.h:46
unsigned char u8
Definition: types.h:56
u32 fifo_segment_num_free_fifos(fifo_segment_t *fs)
Definition: fifo_segment.c:732
void ssvm_delete(ssvm_private_t *ssvm)
Definition: ssvm.c:437
struct _svm_fifo svm_fifo_t
void svm_fifo_init_chunks(svm_fifo_t *f)
Initialize fifo chunks and rbtree.
Definition: svm_fifo.c:408
u32 fifo_segment_num_free_chunks(fifo_segment_t *fs, u32 size)
Find number of free chunks of given size.
Definition: fifo_segment.c:751
fifo_segment_t * fifo_segment_get_segment(fifo_segment_main_t *sm, u32 segment_index)
Definition: fifo_segment.c:149
u32 n_fl_chunk_bytes
Chunk bytes on freelist.
Definition: fifo_segment.h:47
u32 fifo_segment_free_bytes(fifo_segment_t *fs)
Fifo segment estimate of number of free bytes.
Definition: fifo_segment.c:803
u8 * ssvm_name(const ssvm_private_t *ssvm)
Definition: ssvm.c:449
int fifo_segment_prealloc_fifo_hdrs(fifo_segment_t *fs, u32 batch_size)
Try to preallocate fifo headers.
Definition: fifo_segment.c:505
enum ssvm_segment_type_ ssvm_segment_type_t
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
svm_fifo_chunk_t * svm_fifo_chunk_alloc(u32 size)
Creates a fifo chunk in the current heap.
Definition: svm_fifo.c:471
static u32 fs_freelist_for_size(u32 size)
Definition: fifo_segment.c:170
u8 * format_memory_size(u8 *s, va_list *va)
Definition: std-formats.c:209
void svm_fifo_free_ooo_data(svm_fifo_t *f)
Cleanup fifo ooo data.
Definition: svm_fifo.c:132
static void * ssvm_push_heap(ssvm_shared_header_t *sh)
Definition: ssvm.h:143
unsigned int u32
Definition: types.h:88
int attach_timeout
shm segments attach timeout (sec)
Definition: ssvm.h:94
u64 next_baseva
Where to put the next one.
Definition: fifo_segment.h:59
int ssvm_master_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
Definition: ssvm.c:425
static void ssvm_pop_heap(void *oldheap)
Definition: ssvm.h:151
char * segment_name
segment name
Definition: fifo_segment.h:68
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
int fifo_segment_attach(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Attach as slave to a fifo segment.
Definition: fifo_segment.c:99
void fifo_segment_preallocate_fifo_pairs(fifo_segment_t *fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 *n_fifo_pairs)
Pre-allocates fifo pairs in fifo segment.
Definition: fifo_segment.c:594
u8 * format_mheap(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:354
svm_fifo_chunk_t ** free_chunks
Freelists by chunk size.
Definition: fifo_segment.h:43
u8 * format_fifo_segment_type(u8 *s, va_list *args)
Definition: fifo_segment.c:827
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
uword size
struct svm_fifo_chunk_ * next
pointer to next chunk in linked-lists
Definition: svm_fifo.h:62
u8 * format_fifo_segment(u8 *s, va_list *args)
Segment format function.
Definition: fifo_segment.c:848
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
u32 fifo_segment_fl_chunk_bytes(fifo_segment_t *fs)
Number of bytes on chunk free lists.
Definition: fifo_segment.c:809
int fifo_segment_grow_fifo(fifo_segment_t *fs, svm_fifo_t *f, u32 chunk_size)
Grow fifo size by adding an additional chunk of memory.
Definition: fifo_segment.c:645
static int fs_chunk_size_is_valid(u32 size)
Definition: fifo_segment.c:182
svm_fifo_t * free_fifos
Freelists by fifo size.
Definition: fifo_segment.h:42
svm_fifo_t * fifo_segment_get_fifo_list(fifo_segment_t *fs)
Definition: fifo_segment.c:821
static int fs_try_alloc_fifo_batch(fifo_segment_t *fs, u32 fl_index, u32 batch_size)
Definition: fifo_segment.c:275
int fifo_segment_prealloc_fifo_chunks(fifo_segment_t *fs, u32 chunk_size, u32 batch_size)
Try to preallocate fifo chunks on segment.
Definition: fifo_segment.c:542
int fifo_segment_create(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Create a fifo segment and initialize as master.
Definition: fifo_segment.c:65
#define FIFO_SEGMENT_MIN_FIFO_SIZE
Definition: fifo_segment.h:29
u32 segment_size
size of the segment
Definition: fifo_segment.h:66
ssvm_private_t ssvm
ssvm segment data
Definition: fifo_segment.h:52
svmdb_client_t * c
void fifo_segment_main_init(fifo_segment_main_t *sm, u64 baseva, u32 timeout_in_seconds)
Definition: fifo_segment.c:162
#define clib_warning(format, args...)
Definition: error.h:59
void svm_fifo_add_chunk(svm_fifo_t *f, svm_fifo_chunk_t *c)
Grow fifo size by adding chunk to chunk list.
Definition: svm_fifo.c:579
int ssvm_slave_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
Definition: ssvm.c:431
u32 my_pid
Definition: ssvm.h:86
u32 start_byte
chunk start byte
Definition: svm_fifo.h:60
int memfd_fd
fd for memfd segments
Definition: fifo_segment.h:67
int fd
memfd segments
Definition: ssvm.h:93
u32 fifo_segment_index(fifo_segment_main_t *sm, fifo_segment_t *s)
Definition: fifo_segment.c:137
#define ASSERT(truth)
void * svm_fifo_segment_heap(fifo_segment_t *seg)
Definition: fifo_segment.c:143
void fifo_segment_free_fifo(fifo_segment_t *fs, svm_fifo_t *f)
Free fifo allocated in fifo segment.
Definition: fifo_segment.c:435
static svm_fifo_t * fs_try_alloc_fifo(fifo_segment_t *fs, u32 data_bytes)
Try to allocate new fifo.
Definition: fifo_segment.c:331
fifo_segment_t * segments
pool of fifo segments
Definition: fifo_segment.h:58
fifo_segment_ftype_t
Definition: fifo_segment.h:21
#define FIFO_SEGMENT_MAX_FIFO_SIZE
Definition: fifo_segment.h:30
size_t count
Definition: vapi.c:47
svm_fifo_t * fifo_segment_alloc_fifo(fifo_segment_t *fs, u32 data_bytes, fifo_segment_ftype_t ftype)
Allocate fifo in fifo segment.
Definition: fifo_segment.c:380
static void * clib_mem_alloc(uword size)
Definition: mem.h:153
u8 * name
Definition: ssvm.h:87
ssvm_segment_type_t segment_type
type of segment requested
Definition: fifo_segment.h:65
svm_fifo_t * fifos
Linked list of active RX fifos.
Definition: fifo_segment.h:41
u32 fifo_segment_num_fifos(fifo_segment_t *fs)
Get number of active fifos.
Definition: fifo_segment.c:726
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static uword max_log2(uword x)
Definition: clib.h:191
u64 uword
Definition: types.h:112
void fifo_segment_update_free_bytes(fifo_segment_t *fs)
Update fifo segment free bytes estimate.
Definition: fifo_segment.c:797
u8 fifo_segment_has_fifos(fifo_segment_t *fs)
Definition: fifo_segment.c:815
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:161
int fifo_segment_collect_fifo_chunks(fifo_segment_t *fs, svm_fifo_t *f)
Collect unused chunks for fifo.
Definition: fifo_segment.c:694
#define FIFO_SEGMENT_ALLOC_BATCH_SIZE
Definition: fifo_segment.h:31
static void ssvm_unlock_non_recursive(ssvm_shared_header_t *h)
Definition: ssvm.h:136
u32 timeout_in_seconds
Time to wait during attach.
Definition: fifo_segment.h:60
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:486
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
int fifo_segment_init(fifo_segment_t *fs)
Initialize fifo segment shared header.
Definition: fifo_segment.c:41
int i_am_master
Definition: ssvm.h:89
static void ssvm_lock_non_recursive(ssvm_shared_header_t *h, u32 tag)
Definition: ssvm.h:116
u32 * new_segment_indices
return vec of new seg indices
Definition: fifo_segment.h:69
static svm_fifo_t * fs_try_alloc_fifo_freelist_multi_chunk(fifo_segment_t *fs, u32 data_bytes)
Definition: fifo_segment.c:219
ssvm_segment_type_t ssvm_type(const ssvm_private_t *ssvm)
Definition: ssvm.c:443
svm_fifo_t * svm_fifo_create(u32 data_size_in_bytes)
Creates a fifo in the current heap.
Definition: svm_fifo.c:436