40 #ifndef included_vlib_buffer_funcs_h 41 #define included_vlib_buffer_funcs_h 64 u32 *buffer_indices,
u16 *thread_indices,
u32 n_packets,
65 int drop_on_congestion);
137 ASSERT (n_buffers <= ring_size);
145 u32 n = ring_size - start;
155 ASSERT (n_buffers <= ring_size);
163 u32 n = ring_size - start;
173 #if defined CLIB_HAVE_VEC512 174 b->as_u8x64[0] = bt->as_u8x64[0];
175 #elif defined (CLIB_HAVE_VEC256) 176 b->as_u8x32[0] = bt->as_u8x32[0];
177 b->as_u8x32[1] = bt->as_u8x32[1];
178 #elif defined (CLIB_HAVE_VEC128) 179 b->as_u8x16[0] = bt->as_u8x16[0];
180 b->as_u8x16[1] = bt->as_u8x16[1];
181 b->as_u8x16[2] = bt->as_u8x16[2];
182 b->as_u8x16[3] = bt->as_u8x16[3];
208 #ifdef CLIB_HAVE_VEC512 209 u64x8 of8 = u64x8_splat (buffer_mem_start + offset);
215 u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi));
216 u64x8 b1 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 8));
217 u64x8 b2 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 16));
218 u64x8 b3 = u64x8_from_u32x8 (u32x8_load_unaligned (bi + 24));
230 u64x8 b0 = u64x8_from_u32x8 (u32x8_load_unaligned (bi));
237 #elif defined CLIB_HAVE_VEC256 238 u64x4 off = u64x4_splat (buffer_mem_start + offset);
243 u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
244 u64x4 b1 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 4));
245 u64x4 b2 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 8));
246 u64x4 b3 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 12));
247 u64x4 b4 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 16));
248 u64x4 b5 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 20));
249 u64x4 b6 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 24));
250 u64x4 b7 = u64x4_from_u32x4 (u32x4_load_unaligned (bi + 28));
267 #ifdef CLIB_HAVE_VEC256 268 u64x4 b0 = u64x4_from_u32x4 (u32x4_load_unaligned (bi));
271 #elif defined (CLIB_HAVE_VEC128) 272 u64x2 off = u64x2_splat (buffer_mem_start + offset);
273 u32x4 bi4 = u32x4_load_unaligned (bi);
275 #if defined (__aarch64__) 346 #ifdef CLIB_HAVE_VEC256 347 u32x8
mask = { 0, 2, 4, 6, 1, 3, 5, 7 };
353 u64x4 v0 = u64x4_load_unaligned (b);
354 u64x4 v1 = u64x4_load_unaligned (b + 4);
369 u32x4_store_unaligned (u32x8_extract_lo (v2), bi);
370 u32x4_store_unaligned (u32x8_extract_lo (v3), bi + 4);
419 return (b->
flags & VLIB_BUFFER_NEXT_PRESENT
469 uword content_len = 0;
479 if (!(b->
flags & VLIB_BUFFER_NEXT_PRESENT))
507 #define vlib_prefetch_buffer_with_index(vm,bi,type) \ 509 vlib_buffer_t * _b = vlib_get_buffer (vm, bi); \ 510 vlib_prefetch_buffer_header (_b, type); \ 546 uword next_buffer_stride,
549 uword follow_buffer_next);
598 u8 buffer_pool_index)
606 if (VLIB_BUFFER_ALLOC_FAULT_INJECTOR > 0)
611 n_buffers = vlib_buffer_alloc_may_fail (vm, n_buffers);
624 if (len >= n_buffers)
729 ASSERT (n_buffers <= ring_size);
755 u8 buffer_pool_index)
759 ASSERT (n_buffers <= ring_size);
782 u32 n_cached, n_empty;
790 if (n_buffers <= n_empty)
799 buffers + n_buffers - n_empty, n_empty);
804 n_buffers - n_empty);
805 bp->
n_avail += n_buffers - n_empty;
813 const int queue_size = 128;
815 u8 buffer_pool_index = ~0;
816 u32 n_queue = 0, queue[queue_size + 4];
818 #if defined(CLIB_HAVE_VEC128) 822 .
flags = VLIB_BUFFER_NEXT_PRESENT,
834 #if defined(CLIB_HAVE_VEC128) 857 #if defined(CLIB_HAVE_VEC128) 859 p0 = u8x16_load_unaligned (b[0]);
860 p1 = u8x16_load_unaligned (b[1]);
861 p2 = u8x16_load_unaligned (b[2]);
862 p3 = u8x16_load_unaligned (b[3]);
864 r = p0 ^ bpi_vec.as_u8x16[0];
865 r |= p1 ^ bpi_vec.as_u8x16[0];
866 r |= p2 ^ bpi_vec.as_u8x16[0];
867 r |= p3 ^ bpi_vec.as_u8x16[0];
868 r &= bpi_mask.as_u8x16[0];
869 r |= (p0 | p1 | p2 | p3) & flags_refs_mask.as_u8x16[0];
871 sum = !u8x16_is_all_zero (r);
877 sum &= VLIB_BUFFER_NEXT_PRESENT;
908 if (n_queue >= queue_size)
925 if (
PREDICT_FALSE (buffer_pool_index != b[0]->buffer_pool_index))
935 #if defined(CLIB_HAVE_VEC128) 949 queue[n_queue++] = bi;
952 if (n_queue == queue_size)
958 if (maybe_next && (
flags & VLIB_BUFFER_NEXT_PRESENT))
1032 ASSERT (n_buffers <= ring_size);
1057 ASSERT (n_buffers <= ring_size);
1075 #define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK \ 1076 (VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID | \ 1077 VLIB_BUFFER_IS_TRACED | ~VLIB_BUFFER_FLAGS_ALL) 1089 while (s->
flags & VLIB_BUFFER_NEXT_PRESENT)
1168 ASSERT ((b->
flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1178 if (source + length <= destination)
1181 memmove (destination, source, length);
1207 ASSERT (n_buffers <= 256);
1209 ASSERT ((offset + head_end_offset) <
1214 buffers[0] = src_buffer;
1232 buffers[0] = src_buffer;
1258 VLIB_BUFFER_NEXT_PRESENT;
1268 while (s->
flags & VLIB_BUFFER_NEXT_PRESENT)
1298 while (n_buffers > 256)
1304 (buffers + n_cloned),
1305 256, head_end_offset, offset);
1310 n_buffers, head_end_offset, offset);
1332 head_end_offset, 0);
1346 ASSERT ((head->
flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1349 head->
flags |= VLIB_BUFFER_NEXT_PRESENT;
1350 head->
flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
1351 head->
flags &= ~VLIB_BUFFER_EXT_HDR_VALID;
1352 head->
flags |= (tail->
flags & VLIB_BUFFER_TOTAL_LENGTH_VALID);
1360 if (tail->
flags & VLIB_BUFFER_NEXT_PRESENT)
1373 first->
flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1374 first->
flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID;
1383 last->
flags |= VLIB_BUFFER_NEXT_PRESENT;
1385 next_buffer->
flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1450 uword n_packet_data_bytes,
1451 uword min_n_buffers_each_alloc,
1476 u32 bytes_left = 0, data_size;
1488 while (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
1499 if (is_cloned && bytes_left >= dst_left)
1502 u32 space_needed = bytes_left - dst_left;
1512 while (len < space_needed)
1520 b->
flags = VLIB_BUFFER_NEXT_PRESENT;
1527 to_free = first->next_buffer;
1528 first->next_buffer = tail;
1533 src_left = sb->current_length;
1546 dst_left = data_size;
1558 while (src_left == 0)
1560 ASSERT (sb->flags & VLIB_BUFFER_NEXT_PRESENT);
1562 src_left = sb->current_length;
1566 bytes_to_copy =
clib_min (dst_left, src_left);
1571 bytes_to_copy =
clib_min (bytes_to_copy, sp - dp);
1576 src_left -= bytes_to_copy;
1577 dst_left -= bytes_to_copy;
1578 dp += bytes_to_copy;
1579 sp += bytes_to_copy;
1580 bytes_left -= bytes_to_copy;
1586 if (is_cloned && to_free)
1590 if (db->
flags & VLIB_BUFFER_NEXT_PRESENT)
1592 db->
flags &= ~VLIB_BUFFER_NEXT_PRESENT;
1595 while (b->
flags & VLIB_BUFFER_NEXT_PRESENT)
1602 first->flags &= ~VLIB_BUFFER_TOTAL_LENGTH_VALID;
#define VLIB_BUFFER_COPY_CLONE_FLAGS_MASK
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
static uword vlib_buffer_get_current_pa(vlib_main_t *vm, vlib_buffer_t *b)
static __clib_warn_unused_result u32 vlib_buffer_alloc_to_ring_from_pool(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers into ring from specific buffer pool.
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer's data.
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
void() vlib_buffer_enqueue_to_single_next_fn_t(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *ers, u16 next_index, u32 count)
uword vlib_buffer_length_in_chain_slow_path(vlib_main_t *vm, vlib_buffer_t *b_first)
vl_api_wireguard_peer_flags_t flags
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
vnet_hw_if_output_node_runtime_t * r
#define clib_atomic_add_fetch(a, b)
format_function_t format_vlib_buffer_contents
static uword vlib_buffer_get_pa(vlib_main_t *vm, vlib_buffer_t *b)
format_function_t format_vlib_buffer
static_always_inline u64x2 u64x2_from_u32x4_high(u32x4 v)
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
vlib_buffer_enqueue_to_single_next_fn_t * buffer_enqueue_to_single_next_fn
static_always_inline vlib_buffer_pool_t * vlib_get_buffer_pool(vlib_main_t *vm, u8 buffer_pool_index)
u16 nexts[VLIB_FRAME_SIZE]
u16 vlib_buffer_chain_append_data_with_alloc(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t **last, void *data, u16 data_len)
static void vlib_buffer_attach_clone(vlib_main_t *vm, vlib_buffer_t *head, vlib_buffer_t *tail)
Attach cloned tail to the buffer.
static void vlib_buffer_chain_increase_length(vlib_buffer_t *first, vlib_buffer_t *last, i32 len)
static void vlib_buffer_move(vlib_main_t *vm, vlib_buffer_t *b, i16 offset)
#define VLIB_BUFFER_PRE_DATA_SIZE
int vlib_buffer_add_data(vlib_main_t *vm, u32 *buffer_index, void *data, u32 n_data_bytes)
u16 current_length
Nbytes between current data and the end of this buffer.
#define CLIB_LOG2_CACHE_LINE_BYTES
static __clib_warn_unused_result u32 vlib_buffer_alloc_to_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Allocate buffers into ring.
vlib_buffer_main_t * buffer_main
foreach_avx2_vec256i static foreach_avx2_vec256u u32x8 u32x8_permute(u32x8 v, u32x8 idx)
static_always_inline void vlib_get_buffers_with_offset(vlib_main_t *vm, u32 *bi, void **b, int count, i32 offset)
Translate array of buffer indices into buffer pointers with offset.
static vlib_buffer_known_state_t vlib_buffer_is_known(vlib_main_t *vm, u32 buffer_index)
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
static void vlib_buffer_copy_indices_to_ring(u32 *ring, u32 *src, u32 start, u32 ring_size, u32 n_buffers)
static vlib_buffer_t * vlib_buffer_copy(vlib_main_t *vm, vlib_buffer_t *b)
u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES]
STATIC_ASSERT_OFFSET_OF(vlib_buffer_t, template_end, 64)
u8 buffer_pool_index
index of buffer pool this buffer belongs.
static_always_inline void vlib_get_buffer_indices(vlib_main_t *vm, vlib_buffer_t **b, u32 *bi, uword count)
Translate array of buffer pointers into buffer indices.
format_function_t format_vlib_buffer_no_chain
vlib_buffer_pool_thread_t * threads
static_always_inline __clib_warn_unused_result uword vlib_buffer_pool_get(vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers, u32 n_buffers)
static __clib_warn_unused_result u32 vlib_buffer_alloc_on_numa(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u32 numa_node)
Allocate buffers from specific numa node into supplied array.
#define static_always_inline
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
vlib_buffer_enqueue_to_next_fn_t * buffer_enqueue_to_next_fn
static u32 vlib_get_buffer_index(vlib_main_t *vm, void *p)
Translate buffer pointer into buffer index.
static u16 vlib_buffer_chain_append_data(vlib_main_t *vm, vlib_buffer_t *first, vlib_buffer_t *last, void *data, u16 data_len)
epu8_epi32 epu16_epi32 u64x2
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
static heap_elt_t * first(heap_header_t *h)
static u32 vlib_buffer_chain_linearize(vlib_main_t *vm, vlib_buffer_t *b)
static vlib_buffer_t * vlib_buffer_copy_no_chain(vlib_main_t *vm, vlib_buffer_t *b, u32 *di)
u32 min_n_buffers_each_alloc
u8 * vlib_validate_buffer(vlib_main_t *vm, u32 buffer_index, uword follow_chain)
static_always_inline void vlib_buffer_copy_template(vlib_buffer_t *b, vlib_buffer_t *bt)
format_function_t format_vlib_buffer_and_data
static __clib_warn_unused_result u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
static void vlib_buffer_validate(vlib_main_t *vm, vlib_buffer_t *b)
vlib_buffer_t buffer_template
u8 * vlib_validate_buffers(vlib_main_t *vm, u32 *buffers, uword next_buffer_stride, uword n_buffers, vlib_buffer_known_state_t known_state, uword follow_buffer_next)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
static_always_inline u32x4 u32x4_shuffle(u32x4 v, const int a, const int b, const int c, const int d)
vlib_buffer_pool_t * buffer_pools
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
static_always_inline void vlib_get_buffer_indices_with_offset(vlib_main_t *vm, void **b, u32 *bi, uword count, i32 offset)
Translate array of buffer pointers into buffer indices with offset.
u32() vlib_buffer_enqueue_to_thread_fn_t(vlib_main_t *vm, vlib_node_runtime_t *node, u32 frame_queue_index, u32 *buffer_indices, u16 *thread_indices, u32 n_packets, int drop_on_congestion)
static uword vlib_buffer_contents(vlib_main_t *vm, u32 buffer_index, u8 *contents)
Copy buffer contents to memory.
static void vlib_buffer_chain_init(vlib_buffer_t *first)
#define clib_atomic_sub_fetch(a, b)
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
void() vlib_buffer_enqueue_to_next_fn_t(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
clib_spinlock_t buffer_known_hash_lockp
static void vlib_buffer_free_no_next(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers, does not free the buffer chain for each buffer.
static u16 vlib_buffer_clone(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset)
Create multiple clones of buffer and store them in the supplied array.
static_always_inline void vlib_buffer_pool_put(vlib_main_t *vm, u8 buffer_pool_index, u32 *buffers, u32 n_buffers)
sll srl srl sll sra u16x4 i
void vlib_buffer_chain_validate(vlib_main_t *vm, vlib_buffer_t *first)
#define vec_free(V)
Free vector's memory (no header).
#define VLIB_BUFFER_MAX_NUMA_NODES
static vlib_buffer_t * vlib_get_next_buffer(vlib_main_t *vm, vlib_buffer_t *b)
Get next buffer in buffer linklist, or zero for end of list.
static_always_inline void clib_memcpy_u32(u32 *dst, u32 *src, u32 n_left)
void vlib_buffer_validate_alloc_free(vlib_main_t *vm, u32 *buffers, uword n_buffers, vlib_buffer_known_state_t expected_state)
static u32 vlib_buffer_space_left_at_end(vlib_main_t *vm, vlib_buffer_t *b)
static uword round_pow2(uword x, uword pow2)
vlib_buffer_known_state_t
#define uword_to_pointer(u, type)
vlib buffer structure definition and a few select access methods.
static u16 vlib_buffer_clone_at_offset(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset, i16 offset)
Create multiple clones of buffer and store them in the supplied array.
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
static uword vlib_buffer_index_length_in_chain(vlib_main_t *vm, u32 bi)
Get length in bytes of the buffer index buffer chain.
vlib_buffer_func_main_t vlib_buffer_func_main
static_always_inline void * clib_memcpy_fast(void *restrict dst, const void *restrict src, size_t n)
nat44_ei_hairpin_src_next_t next_index
static void * vlib_buffer_ptr_from_index(uword buffer_mem_start, u32 buffer_index, uword offset)
static_always_inline void vlib_buffer_free_inline(vlib_main_t *vm, u32 *buffers, u32 n_buffers, int maybe_next)
static_always_inline u64x2 u64x2_from_u32x4(u32x4 v)
static uword pointer_to_uword(const void *p)
template key/value backing page structure
vlib_frame_queue_dequeue_fn_t * frame_queue_dequeue_fn
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
static_always_inline void vlib_buffer_copy_indices(u32 *dst, u32 *src, u32 n_indices)
static vlib_buffer_t * vlib_buffer_chain_buffer(vlib_main_t *vm, vlib_buffer_t *last, u32 next_bi)
#define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ
vlib_main_t vlib_node_runtime_t * node
u32 next_buffer
Next buffer for this linked-list of buffers.
u32 cached_buffers[VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ]
#define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
VLIB buffer representation.
static void vlib_buffer_copy_indices_from_ring(u32 *dst, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
u32() vlib_frame_queue_dequeue_fn_t(vlib_main_t *vm, vlib_frame_queue_main_t *fqm)
uword * buffer_known_hash
struct clib_bihash_value offset
template key/value backing page structure
vlib_buffer_enqueue_to_thread_fn_t * buffer_enqueue_to_thread_fn
void * vlib_packet_template_get_packet(vlib_main_t *vm, vlib_packet_template_t *t, u32 *bi_result)
static u16 vlib_buffer_clone_256(vlib_main_t *vm, u32 src_buffer, u32 *buffers, u16 n_buffers, u16 head_end_offset, i16 offset)
Create a maximum of 256 clones of buffer and store them in the supplied array.
static void vlib_packet_template_free(vlib_main_t *vm, vlib_packet_template_t *t)
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
#define CLIB_CACHE_LINE_BYTES
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
void vlib_packet_template_init(vlib_main_t *vm, vlib_packet_template_t *t, void *packet_data, uword n_packet_data_bytes, uword min_n_buffers_each_alloc, char *fmt,...)
static u8 vlib_buffer_pool_get_default_for_numa(vlib_main_t *vm, u32 numa_node)
static void vlib_buffer_free_from_ring_no_next(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring without freeing tail buffers.
volatile u8 ref_count
Reference count for this buffer.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
static __clib_warn_unused_result u32 vlib_buffer_alloc_from_pool(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers from specific pool into supplied array.