34 return dlminfo.fordblks;
117 _vec_len (s) =
vec_len (s) - 1;
163 u32 timeout_in_seconds)
210 memset (f, 0,
sizeof (*f));
223 u32 fl_index, fl_size, n_alloc = 0;
234 memset (f, 0,
sizeof (*f));
259 fl_size = fl_size >> 1;
262 f->start_chunk =
first;
273 u32 size, hdrs, rounded_data_size;
282 hdrs =
sizeof (*f) +
sizeof (*c);
283 size = (hdrs + rounded_data_size) * batch_size;
296 for (i = 0; i < batch_size; i++)
299 memset (f, 0,
sizeof (*f));
304 c->
length = rounded_data_size;
307 fmem += hdrs + rounded_data_size;
329 u32 fifo_sz, fl_index;
335 fifo_sz += 1 <<
max_log2 (data_bytes);
343 if (fifo_sz * FIFO_SEGMENT_ALLOC_BATCH_SIZE < fsh->n_free_bytes)
352 if (fifo_sz <= fsh->n_free_bytes)
363 if (data_bytes <= fsh->n_fl_chunk_bytes)
398 if (f->start_chunk->next != f->start_chunk)
413 fsh->
fifos->prev = f;
414 f->next = fsh->
fifos;
452 f->prev->next = f->next;
454 fsh->
fifos = f->next;
456 f->next->prev = f->prev;
466 cur = f->start_chunk;
477 while (cur != f->start_chunk);
479 f->start_chunk = f->end_chunk = f->new_chunks = 0;
480 f->head_chunk = f->tail_chunk = f->ooo_enq = f->ooo_deq = 0;
491 f->master_session_index = ~0;
492 f->master_thread_index = ~0;
509 size = (
sizeof (*f)) * batch_size;
522 for (i = 0; i < batch_size; i++)
525 memset (f, 0,
sizeof (*f));
541 u32 size, rounded_data_size, fl_index;
549 clib_warning (
"chunk size out of range %d", chunk_size);
556 size = (
sizeof (*c) + rounded_data_size) * batch_size;
569 for (i = 0; i < batch_size; i++)
573 c->
length = rounded_data_size;
576 cmem +=
sizeof (*c) + rounded_data_size;
590 u32 rx_fifo_size,
u32 tx_fifo_size,
593 u32 rx_rounded_data_size, tx_rounded_data_size, pair_size, pairs_to_alloc;
594 int rx_fl_index, tx_fl_index;
595 uword space_available;
599 if (rx_fifo_size == 0 || tx_fifo_size == 0 || *n_fifo_pairs == 0)
604 clib_warning (
"rx fifo_size out of range %d", rx_fifo_size);
610 clib_warning (
"tx fifo_size out of range %d", tx_fifo_size);
614 rx_rounded_data_size = (1 << (
max_log2 (rx_fifo_size)));
616 tx_rounded_data_size = (1 << (
max_log2 (tx_fifo_size)));
622 pair_size = 2 * hdrs + rx_rounded_data_size + tx_rounded_data_size;
624 pairs_to_alloc = space_available / pair_size;
625 pairs_to_alloc =
clib_min (pairs_to_alloc, *n_fifo_pairs);
631 clib_warning (
"rx prealloc failed: pairs %u", pairs_to_alloc);
633 clib_warning (
"tx prealloc failed: pairs %u", pairs_to_alloc);
636 *n_fifo_pairs -= pairs_to_alloc;
649 clib_warning (
"chunk size out of range %d", chunk_size);
747 u32 count = 0, rounded_size, fl_index;
772 rounded_size = (1 << (
max_log2 (size)));
828 s =
format (s,
"%s",
"private-heap");
830 s =
format (s,
"%s",
"memfd");
832 s =
format (s,
"%s",
"shm");
834 s =
format (s,
"%s",
"unknown");
845 int verbose __attribute__ ((unused)) = va_arg (*args,
int);
856 #if USE_DLMALLOC == 0 859 s =
format (s,
"%U segment has %u active fifos\n",
865 s =
format (s,
"%-15s%15s%15s%15s%15s%15s",
"Name",
"Type",
866 "HeapSize (M)",
"ActiveFifos",
"FreeFifos",
"Address");
877 free_fifos, address);
u32 length
length of chunk in bytes
void fifo_segment_info(fifo_segment_t *seg, char **address, size_t *size)
fifo_segment_header_t * h
fifo segment data
svm_fifo_chunk_t * svm_fifo_collect_chunks(svm_fifo_t *f)
Removes chunks that are after fifo end byte.
static u32 fs_free_space(fifo_segment_t *fs)
Fifo segment free space.
static void * clib_mem_alloc_aligned_at_offset(uword size, uword align, uword align_offset, int os_out_of_memory_on_failure)
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
void svm_fifo_free_chunk_lookup(svm_fifo_t *f)
Cleanup fifo chunk lookup rb tree.
void fifo_segment_delete(fifo_segment_main_t *sm, fifo_segment_t *s)
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
static heap_elt_t * last(heap_header_t *h)
void svm_fifo_init(svm_fifo_t *f, u32 size)
Initialize fifo.
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
ssvm_shared_header_t * sh
static svm_fifo_t * fs_try_alloc_fifo_freelist(fifo_segment_t *fs, u32 fl_index, u32 data_bytes)
DLMALLOC_EXPORT struct dlmallinfo mspace_mallinfo(mspace msp)
static u32 fs_freelist_index_to_size(u32 fl_index)
u32 fifo_segment_num_free_fifos(fifo_segment_t *fs)
void ssvm_delete(ssvm_private_t *ssvm)
struct _svm_fifo svm_fifo_t
void svm_fifo_init_chunks(svm_fifo_t *f)
Initialize fifo chunks and rbtree.
u32 fifo_segment_num_free_chunks(fifo_segment_t *fs, u32 size)
Find number of free chunks of given size.
fifo_segment_t * fifo_segment_get_segment(fifo_segment_main_t *sm, u32 segment_index)
u32 fifo_segment_free_bytes(fifo_segment_t *fs)
Fifo segment estimate of number of free bytes.
u8 * ssvm_name(const ssvm_private_t *ssvm)
int fifo_segment_prealloc_fifo_hdrs(fifo_segment_t *fs, u32 batch_size)
Try to preallocate fifo headers.
enum ssvm_segment_type_ ssvm_segment_type_t
svm_fifo_chunk_t * svm_fifo_chunk_alloc(u32 size)
Creates a fifo chunk in the current heap.
static u32 fs_freelist_for_size(u32 size)
void svm_fifo_free_ooo_data(svm_fifo_t *f)
Cleanup fifo ooo data.
static void * ssvm_push_heap(ssvm_shared_header_t *sh)
int attach_timeout
shm segments attach timeout (sec)
u64 next_baseva
Where to put the next one.
int ssvm_master_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
static void ssvm_pop_heap(void *oldheap)
char * segment_name
segment name
static heap_elt_t * first(heap_header_t *h)
int fifo_segment_attach(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Attach as slave to a fifo segment.
void fifo_segment_preallocate_fifo_pairs(fifo_segment_t *fs, u32 rx_fifo_size, u32 tx_fifo_size, u32 *n_fifo_pairs)
Pre-allocates fifo pairs in fifo segment.
u8 * format_mheap(u8 *s, va_list *va)
u8 * format_fifo_segment_type(u8 *s, va_list *args)
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
struct svm_fifo_chunk_ * next
pointer to next chunk in linked-lists
u8 * format_fifo_segment(u8 *s, va_list *args)
Segment format function.
#define pool_put(P, E)
Free an object E in pool P.
u32 fifo_segment_fl_chunk_bytes(fifo_segment_t *fs)
Number of bytes on chunk free lists.
int fifo_segment_grow_fifo(fifo_segment_t *fs, svm_fifo_t *f, u32 chunk_size)
Grow fifo size by adding an additional chunk of memory.
static int fs_chunk_size_is_valid(u32 size)
svm_fifo_t * fifo_segment_get_fifo_list(fifo_segment_t *fs)
static int fs_try_alloc_fifo_batch(fifo_segment_t *fs, u32 fl_index, u32 batch_size)
int fifo_segment_prealloc_fifo_chunks(fifo_segment_t *fs, u32 chunk_size, u32 batch_size)
Try to preallocate fifo chunks on segment.
int fifo_segment_create(fifo_segment_main_t *sm, fifo_segment_create_args_t *a)
Create a fifo segment and initialize as master.
#define FIFO_SEGMENT_MIN_FIFO_SIZE
u32 segment_size
size of the segment
ssvm_private_t ssvm
ssvm segment data
void fifo_segment_main_init(fifo_segment_main_t *sm, u64 baseva, u32 timeout_in_seconds)
#define clib_warning(format, args...)
void svm_fifo_add_chunk(svm_fifo_t *f, svm_fifo_chunk_t *c)
Grow fifo size by adding chunk to chunk list.
int ssvm_slave_init(ssvm_private_t *ssvm, ssvm_segment_type_t type)
u32 start_byte
chunk start byte
int memfd_fd
fd for memfd segments
u32 fifo_segment_index(fifo_segment_main_t *sm, fifo_segment_t *s)
void * svm_fifo_segment_heap(fifo_segment_t *seg)
void fifo_segment_free_fifo(fifo_segment_t *fs, svm_fifo_t *f)
Free fifo allocated in fifo segment.
static svm_fifo_t * fs_try_alloc_fifo(fifo_segment_t *fs, u32 data_bytes)
Try to allocate new fifo.
fifo_segment_t * segments
pool of fifo segments
#define FIFO_SEGMENT_MAX_FIFO_SIZE
svm_fifo_t * fifo_segment_alloc_fifo(fifo_segment_t *fs, u32 data_bytes, fifo_segment_ftype_t ftype)
Allocate fifo in fifo segment.
static void * clib_mem_alloc(uword size)
ssvm_segment_type_t segment_type
type of segment requested
u32 fifo_segment_num_fifos(fifo_segment_t *fs)
Get number of active fifos.
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static uword max_log2(uword x)
void fifo_segment_update_free_bytes(fifo_segment_t *fs)
Update fifo segment free bytes estimate.
u8 fifo_segment_has_fifos(fifo_segment_t *fs)
static void * clib_mem_alloc_aligned(uword size, uword align)
int fifo_segment_collect_fifo_chunks(fifo_segment_t *fs, svm_fifo_t *f)
Collect unused chunks for fifo.
#define FIFO_SEGMENT_ALLOC_BATCH_SIZE
static void ssvm_unlock_non_recursive(ssvm_shared_header_t *h)
u32 timeout_in_seconds
Time to wait during attach.
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
#define CLIB_CACHE_LINE_BYTES
int fifo_segment_init(fifo_segment_t *fs)
Initialize fifo segment shared header.
static void ssvm_lock_non_recursive(ssvm_shared_header_t *h, u32 tag)
u32 * new_segment_indices
return vec of new seg indices
static svm_fifo_t * fs_try_alloc_fifo_freelist_multi_chunk(fifo_segment_t *fs, u32 data_bytes)
ssvm_segment_type_t ssvm_type(const ssvm_private_t *ssvm)
svm_fifo_t * svm_fifo_create(u32 data_size_in_bytes)
Creates a fifo in the current heap.