FD.io VPP  v19.08-27-gf4dcae4
Vector Packet Processing
svm_fifo.c File Reference
+ Include dependency graph for svm_fifo.c:

Go to the source code of this file.

Functions

 CLIB_MARCH_FN (svm_fifo_copy_to_chunk, void, svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx, const u8 *src, u32 len, svm_fifo_chunk_t **last)
 
 CLIB_MARCH_FN (svm_fifo_copy_from_chunk, void, svm_fifo_t *f, svm_fifo_chunk_t *c, u32 head_idx, u8 *dst, u32 len, svm_fifo_chunk_t **last)
 
static void svm_fifo_copy_to_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 tail_idx, const u8 *src, u32 len, svm_fifo_chunk_t **last)
 
static void svm_fifo_copy_from_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c, u32 head_idx, u8 *dst, u32 len, svm_fifo_chunk_t **last)
 
static u8 position_lt (svm_fifo_t *f, u32 a, u32 b, u32 tail)
 
static u8 position_leq (svm_fifo_t *f, u32 a, u32 b, u32 tail)
 
static u8 position_gt (svm_fifo_t *f, u32 a, u32 b, u32 tail)
 
static u32 position_diff (svm_fifo_t *f, u32 a, u32 b, u32 tail)
 
static u32 ooo_segment_end_pos (svm_fifo_t *f, ooo_segment_t *s)
 
void svm_fifo_free_ooo_data (svm_fifo_t *f)
 Cleanup fifo ooo data. More...
 
static ooo_segment_tooo_segment_prev (svm_fifo_t *f, ooo_segment_t *s)
 
static ooo_segment_tooo_segment_next (svm_fifo_t *f, ooo_segment_t *s)
 
static ooo_segment_tooo_segment_alloc (svm_fifo_t *f, u32 start, u32 length)
 
static void ooo_segment_free (svm_fifo_t *f, u32 index)
 
static void ooo_segment_add (svm_fifo_t *f, u32 offset, u32 head, u32 tail, u32 length)
 Add segment to fifo's out-of-order segment list. More...
 
static int ooo_segment_try_collect (svm_fifo_t *f, u32 n_bytes_enqueued, u32 *tail)
 Removes segments that can now be enqueued because the fifo's tail has advanced. More...
 
static ooo_segment_tooo_segment_last (svm_fifo_t *f)
 
void svm_fifo_init (svm_fifo_t *f, u32 size)
 Initialize fifo. More...
 
void svm_fifo_init_chunks (svm_fifo_t *f)
 Initialize fifo chunks and rbtree. More...
 
svm_fifo_tsvm_fifo_create (u32 data_size_in_bytes)
 Creates a fifo in the current heap. More...
 
svm_fifo_chunk_tsvm_fifo_chunk_alloc (u32 size)
 Creates a fifo chunk in the current heap. More...
 
static u8 svm_fifo_chunk_includes_pos (svm_fifo_chunk_t *c, u32 pos)
 
static svm_fifo_chunk_tsvm_fifo_find_chunk (svm_fifo_t *f, u32 pos)
 Find chunk for given byte position. More...
 
static void svm_fifo_grow (svm_fifo_t *f, svm_fifo_chunk_t *c)
 
static void svm_fifo_try_grow (svm_fifo_t *f, u32 new_head)
 
void svm_fifo_add_chunk (svm_fifo_t *f, svm_fifo_chunk_t *c)
 Grow fifo size by adding chunk to chunk list. More...
 
svm_fifo_chunk_tsvm_fifo_collect_chunks (svm_fifo_t *f)
 Removes chunks that are after fifo end byte. More...
 
void svm_fifo_try_shrink (svm_fifo_t *f, u32 head, u32 tail)
 Try to shrink fifo size. More...
 
int svm_fifo_reduce_size (svm_fifo_t *f, u32 len, u8 try_shrink)
 Request to reduce fifo size by amount of bytes. More...
 
void svm_fifo_free_chunk_lookup (svm_fifo_t *f)
 Cleanup fifo chunk lookup rb tree. More...
 
void svm_fifo_free (svm_fifo_t *f)
 Free fifo and associated state. More...
 
void svm_fifo_overwrite_head (svm_fifo_t *f, u8 *src, u32 len)
 Overwrite fifo head with new data. More...
 
int svm_fifo_enqueue (svm_fifo_t *f, u32 len, const u8 *src)
 Enqueue data to fifo. More...
 
int svm_fifo_enqueue_with_offset (svm_fifo_t *f, u32 offset, u32 len, u8 *src)
 Enqueue a future segment. More...
 
void svm_fifo_enqueue_nocopy (svm_fifo_t *f, u32 len)
 Advance tail. More...
 
int svm_fifo_dequeue (svm_fifo_t *f, u32 len, u8 *dst)
 Dequeue data from fifo. More...
 
int svm_fifo_peek (svm_fifo_t *f, u32 offset, u32 len, u8 *dst)
 Peek data from fifo. More...
 
int svm_fifo_dequeue_drop (svm_fifo_t *f, u32 len)
 Dequeue and drop bytes from fifo. More...
 
void svm_fifo_dequeue_drop_all (svm_fifo_t *f)
 Dequeue and drop all bytes from fifo. More...
 
int svm_fifo_segments (svm_fifo_t *f, svm_fifo_seg_t *fs)
 
void svm_fifo_segments_free (svm_fifo_t *f, svm_fifo_seg_t *fs)
 
void svm_fifo_clone (svm_fifo_t *df, svm_fifo_t *sf)
 Clones fifo. More...
 
u32 svm_fifo_n_ooo_segments (svm_fifo_t *f)
 Number of out-of-order segments for fifo. More...
 
ooo_segment_tsvm_fifo_first_ooo_segment (svm_fifo_t *f)
 First out-of-order segment for fifo. More...
 
void svm_fifo_init_pointers (svm_fifo_t *f, u32 head, u32 tail)
 Set fifo pointers to requested offset. More...
 
void svm_fifo_add_subscriber (svm_fifo_t *f, u8 subscriber)
 Add io events subscriber to list. More...
 
void svm_fifo_del_subscriber (svm_fifo_t *f, u8 subscriber)
 Remove io events subscriber form list. More...
 
u8 svm_fifo_is_sane (svm_fifo_t *f)
 Check if fifo is sane. More...
 
u8format_ooo_segment (u8 *s, va_list *args)
 
u8svm_fifo_dump_trace (u8 *s, svm_fifo_t *f)
 
u8svm_fifo_replay (u8 *s, svm_fifo_t *f, u8 no_read, u8 verbose)
 
u8format_ooo_list (u8 *s, va_list *args)
 
u8format_svm_fifo (u8 *s, va_list *args)
 

Function Documentation

CLIB_MARCH_FN ( svm_fifo_copy_to_chunk  ,
void  ,
svm_fifo_t f,
svm_fifo_chunk_t c,
u32  tail_idx,
const u8 src,
u32  len,
svm_fifo_chunk_t **  last 
)

Definition at line 23 of file svm_fifo.c.

CLIB_MARCH_FN ( svm_fifo_copy_from_chunk  ,
void  ,
svm_fifo_t f,
svm_fifo_chunk_t c,
u32  head_idx,
u8 dst,
u32  len,
svm_fifo_chunk_t **  last 
)

Definition at line 53 of file svm_fifo.c.

u8* format_ooo_list ( u8 s,
va_list *  args 
)

Definition at line 1250 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

u8* format_ooo_segment ( u8 s,
va_list *  args 
)

Definition at line 1156 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

u8* format_svm_fifo ( u8 s,
va_list *  args 
)

Definition at line 1269 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static void ooo_segment_add ( svm_fifo_t f,
u32  offset,
u32  head,
u32  tail,
u32  length 
)
static

Add segment to fifo's out-of-order segment list.

Takes care of merging adjacent segments and removing overlapping ones.

Definition at line 197 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static ooo_segment_t* ooo_segment_alloc ( svm_fifo_t f,
u32  start,
u32  length 
)
inlinestatic

Definition at line 154 of file svm_fifo.c.

+ Here is the caller graph for this function:

static u32 ooo_segment_end_pos ( svm_fifo_t f,
ooo_segment_t s 
)
inlinestatic

Definition at line 126 of file svm_fifo.c.

+ Here is the caller graph for this function:

static void ooo_segment_free ( svm_fifo_t f,
u32  index 
)
inlinestatic

Definition at line 168 of file svm_fifo.c.

+ Here is the caller graph for this function:

static ooo_segment_t* ooo_segment_last ( svm_fifo_t f)
static

Definition at line 378 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static ooo_segment_t* ooo_segment_next ( svm_fifo_t f,
ooo_segment_t s 
)
inlinestatic

Definition at line 146 of file svm_fifo.c.

+ Here is the caller graph for this function:

static ooo_segment_t* ooo_segment_prev ( svm_fifo_t f,
ooo_segment_t s 
)
inlinestatic

Definition at line 138 of file svm_fifo.c.

+ Here is the caller graph for this function:

static int ooo_segment_try_collect ( svm_fifo_t f,
u32  n_bytes_enqueued,
u32 tail 
)
static

Removes segments that can now be enqueued because the fifo's tail has advanced.

Returns the number of bytes added to tail.

Definition at line 330 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static u32 position_diff ( svm_fifo_t f,
u32  a,
u32  b,
u32  tail 
)
inlinestatic

Definition at line 120 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static u8 position_gt ( svm_fifo_t f,
u32  a,
u32  b,
u32  tail 
)
inlinestatic

Definition at line 114 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static u8 position_leq ( svm_fifo_t f,
u32  a,
u32  b,
u32  tail 
)
inlinestatic

Definition at line 108 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static u8 position_lt ( svm_fifo_t f,
u32  a,
u32  b,
u32  tail 
)
inlinestatic

Definition at line 102 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_add_chunk ( svm_fifo_t f,
svm_fifo_chunk_t c 
)

Grow fifo size by adding chunk to chunk list.

If fifos are allocated on a segment, this should be called with the segment's heap pushed.

Parameters
ffifo to be extended
cchunk or linked list of chunks to be added

Definition at line 579 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_add_subscriber ( svm_fifo_t f,
u8  sub 
)

Add io events subscriber to list.

Parameters
ffifo
subsubscriber opaque index (typically app worker index)

Definition at line 1094 of file svm_fifo.c.

+ Here is the caller graph for this function:

svm_fifo_chunk_t* svm_fifo_chunk_alloc ( u32  size)

Creates a fifo chunk in the current heap.

Allocate a fifo chunk on heap.

Definition at line 471 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static u8 svm_fifo_chunk_includes_pos ( svm_fifo_chunk_t c,
u32  pos 
)
inlinestatic

Definition at line 489 of file svm_fifo.c.

+ Here is the caller graph for this function:

void svm_fifo_clone ( svm_fifo_t df,
svm_fifo_t sf 
)

Clones fifo.

Clone fifo.

Assumptions:

  • no prod and cons are accessing either dest or src fifo
  • fifo is not multi chunk

Definition at line 1049 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

svm_fifo_chunk_t* svm_fifo_collect_chunks ( svm_fifo_t f)

Removes chunks that are after fifo end byte.

Needs to be called with segment heap pushed.

Parameters
ffifo

Definition at line 634 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static void svm_fifo_copy_from_chunk ( svm_fifo_t f,
svm_fifo_chunk_t c,
u32  head_idx,
u8 dst,
u32  len,
svm_fifo_chunk_t **  last 
)
inlinestatic

Definition at line 94 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static void svm_fifo_copy_to_chunk ( svm_fifo_t f,
svm_fifo_chunk_t c,
u32  tail_idx,
const u8 src,
u32  len,
svm_fifo_chunk_t **  last 
)
inlinestatic

Definition at line 86 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

svm_fifo_t* svm_fifo_create ( u32  data_size_in_bytes)

Creates a fifo in the current heap.

Create fifo of requested size.

Fails vs blow up the process

Definition at line 436 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_del_subscriber ( svm_fifo_t f,
u8  subscriber 
)

Remove io events subscriber form list.

Parameters
ffifo
subsubscriber index to be removed

Definition at line 1102 of file svm_fifo.c.

+ Here is the caller graph for this function:

int svm_fifo_dequeue ( svm_fifo_t f,
u32  len,
u8 dst 
)

Dequeue data from fifo.

Data is dequeued to consumer provided buffer and head is atomically updated.

Parameters
ffifo
lenlength of data to dequeue
dstbuffer to where to dequeue the data
Returns
number of bytes dequeued or error

Definition at line 900 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int svm_fifo_dequeue_drop ( svm_fifo_t f,
u32  len 
)

Dequeue and drop bytes from fifo.

Advances fifo head by requested amount of bytes.

Parameters
ffifo
lennumber of bytes to drop
Returns
number of bytes dropped

Definition at line 948 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_dequeue_drop_all ( svm_fifo_t f)

Dequeue and drop all bytes from fifo.

Advances head to tail position.

Parameters
ffifo

Definition at line 980 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

u8* svm_fifo_dump_trace ( u8 s,
svm_fifo_t f 
)

Definition at line 1168 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int svm_fifo_enqueue ( svm_fifo_t f,
u32  len,
const u8 src 
)

Enqueue data to fifo.

Data is enqueued and tail pointer is updated atomically. If the new data enqueued partly overlaps or "touches" an out-of-order segment, said segment is "consumed" and the number of bytes returned is appropriately updated.

Parameters
ffifo
lenlength of data to copy
srcbuffer from where to copy the data
Returns
number of contiguous bytes that can be consumed or error

Definition at line 807 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_enqueue_nocopy ( svm_fifo_t f,
u32  len 
)

Advance tail.

Advance tail pointer.

Definition at line 883 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int svm_fifo_enqueue_with_offset ( svm_fifo_t f,
u32  offset,
u32  len,
u8 src 
)

Enqueue a future segment.

Enqueue data to fifo with offset.

Two choices: either copies the entire segment, or copies nothing Returns 0 of the entire segment was copied Returns -1 if none of the segment was copied due to lack of space

Definition at line 850 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

static svm_fifo_chunk_t* svm_fifo_find_chunk ( svm_fifo_t f,
u32  pos 
)
static

Find chunk for given byte position.

Parameters
ffifo
posnormalized position in fifo
Returns
chunk that includes given position or 0

Definition at line 503 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

ooo_segment_t* svm_fifo_first_ooo_segment ( svm_fifo_t f)

First out-of-order segment for fifo.

Parameters
ffifo
Returns
first out-of-order segment for fifo

Definition at line 1066 of file svm_fifo.c.

+ Here is the caller graph for this function:

void svm_fifo_free ( svm_fifo_t f)

Free fifo and associated state.

Parameters
ffifo

Definition at line 772 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_free_chunk_lookup ( svm_fifo_t f)

Cleanup fifo chunk lookup rb tree.

The rb tree is allocated in segment heap so this should be called with it pushed.

Parameters
ffifo to cleanup

Definition at line 766 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_free_ooo_data ( svm_fifo_t f)

Cleanup fifo ooo data.

The ooo data is allocated in producer process memory. The fifo segment heap should not be pushed.

Parameters
ffifo to cleanup

Definition at line 132 of file svm_fifo.c.

+ Here is the caller graph for this function:

static void svm_fifo_grow ( svm_fifo_t f,
svm_fifo_chunk_t c 
)
inlinestatic

Definition at line 546 of file svm_fifo.c.

+ Here is the caller graph for this function:

void svm_fifo_init ( svm_fifo_t f,
u32  size 
)

Initialize fifo.

Parameters
ffifo
sizesize for fifo

Definition at line 392 of file svm_fifo.c.

+ Here is the caller graph for this function:

void svm_fifo_init_chunks ( svm_fifo_t f)

Initialize fifo chunks and rbtree.

Parameters
ffifo

Definition at line 408 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_init_pointers ( svm_fifo_t f,
u32  head,
u32  tail 
)

Set fifo pointers to requested offset.

Init fifo head and tail.

Definition at line 1075 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

u8 svm_fifo_is_sane ( svm_fifo_t f)

Check if fifo is sane.

Debug only.

Parameters
ffifo
Returns
1 if sane, 0 otherwise

Definition at line 1117 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

u32 svm_fifo_n_ooo_segments ( svm_fifo_t f)

Number of out-of-order segments for fifo.

Parameters
ffifo
Returns
number of out of order segments

Definition at line 1060 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_overwrite_head ( svm_fifo_t f,
u8 src,
u32  len 
)

Overwrite fifo head with new data.

This should be typically used by dgram transport protocols that need to update the dgram header after dequeueing a chunk of data. It assumes that the dgram header is at most spread over two chunks.

Parameters
ffifo
srcsrc of new data
lenlength of new data

Definition at line 785 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int svm_fifo_peek ( svm_fifo_t f,
u32  offset,
u32  len,
u8 dst 
)

Peek data from fifo.

Data is copied from requested offset into provided dst buffer. Head is not updated.

Parameters
ffifo
offsetoffset from which to copy the data
lenlength of data to copy
dstbuffer to where to dequeue the data
Returns
number of bytes peeked

Definition at line 926 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int svm_fifo_reduce_size ( svm_fifo_t f,
u32  len,
u8  try_shrink 
)

Request to reduce fifo size by amount of bytes.

Because the producer might be enqueuing data when this is called, the actual size update is only applied when producer tries to enqueue new data, unless

Parameters
try_shrinkis set.
ffifo
lennumber of bytes to remove from fifo. The actual number of bytes to be removed will be less or equal to this value.
try_shrinkflg to indicate if it's safe to try to shrink fifo size. It should be set only if this is called by the producer of if the producer is not using the fifo
Returns
actual length fifo size will be reduced by

Definition at line 726 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

u8* svm_fifo_replay ( u8 s,
svm_fifo_t f,
u8  no_read,
u8  verbose 
)

Definition at line 1192 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

int svm_fifo_segments ( svm_fifo_t f,
svm_fifo_seg_t fs 
)

Definition at line 996 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_segments_free ( svm_fifo_t f,
svm_fifo_seg_t fs 
)

Definition at line 1028 of file svm_fifo.c.

+ Here is the caller graph for this function:

static void svm_fifo_try_grow ( svm_fifo_t f,
u32  new_head 
)
static

Definition at line 569 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function:

void svm_fifo_try_shrink ( svm_fifo_t f,
u32  head,
u32  tail 
)

Try to shrink fifo size.

Internal function.

Definition at line 653 of file svm_fifo.c.

+ Here is the call graph for this function:

+ Here is the caller graph for this function: