58 #ifndef CLIB_MARCH_VARIANT 64 interface_output_trace_t *t = va_arg (*va, interface_output_trace_t *);
86 #define _(bit, name, v, x) \ 87 if (v && (t->flags & VNET_BUFFER_F_##name)) \ 88 s = format (s, "%s ", v); 91 if (t->
flags & VNET_BUFFER_F_GSO)
93 s =
format (s,
"\n%Ugso_sz %d gso_l4_hdr_sz %d",
119 interface_output_trace_t *t0, *t1;
131 if (b0->
flags & VLIB_BUFFER_IS_TRACED)
141 if (b1->
flags & VLIB_BUFFER_IS_TRACED)
159 interface_output_trace_t *t0;
165 if (b0->
flags & VLIB_BUFFER_IS_TRACED)
186 int is_ip4 = (b->
flags & VNET_BUFFER_F_IS_IP4) != 0;
187 int is_ip6 = (b->
flags & VNET_BUFFER_F_IS_IP6) != 0;
189 ASSERT (!(is_ip4 && is_ip6));
199 if (b->
flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
201 if (b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
206 else if (b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
215 if (b->
flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
221 else if (b->
flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
228 b->
flags &= ~VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
229 b->
flags &= ~VNET_BUFFER_F_OFFLOAD_UDP_CKSUM;
230 b->
flags &= ~VNET_BUFFER_F_OFFLOAD_IP_CKSUM;
244 u16 n_bufs = (n_bytes_b0 - l234_sz + (size - 1)) /
size;
251 if (n_alloc < n_bufs)
265 nb0->
flags = VLIB_BUFFER_TOTAL_LENGTH_VALID |
flags;
276 u16 gso_size,
u8 ** p_dst_ptr,
u16 * p_dst_left,
289 tcp->seq_number = clib_host_to_net_u32 (next_tcp_seq);
301 tcp->flags = tcp_flags;
328 int is_ip4 = sb0->
flags & VNET_BUFFER_F_IS_IP4;
329 int is_ip6 = sb0->
flags & VNET_BUFFER_F_IS_IP6;
330 ASSERT (is_ip4 || is_ip6);
331 ASSERT (sb0->
flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID);
332 ASSERT (sb0->
flags & VNET_BUFFER_F_L3_HDR_OFFSET_VALID);
333 ASSERT (sb0->
flags & VNET_BUFFER_F_L4_HDR_OFFSET_VALID);
337 u8 save_tcp_flags = 0;
338 u8 tcp_flags_no_fin_psh = 0;
339 u32 next_tcp_seq = 0;
343 next_tcp_seq = clib_net_to_host_u32 (tcp->seq_number);
345 save_tcp_flags = tcp->flags;
346 tcp_flags_no_fin_psh = tcp->flags & ~(
TCP_FLAG_FIN | TCP_FLAG_PSH);
350 sb0->
flags & ~(VNET_BUFFER_F_GSO | VLIB_BUFFER_NEXT_PRESENT);
354 next_tcp_seq += first_data_size;
362 l234_sz + first_data_size);
364 u32 total_src_left = n_bytes_b0 - l234_sz - first_data_size;
368 u8 *src_ptr, *dst_ptr;
369 u16 src_left, dst_left;
388 &dst_left, next_tcp_seq, default_bflags);
392 while (total_src_left)
400 src_left -= bytes_to_copy;
401 src_ptr += bytes_to_copy;
402 total_src_left -= bytes_to_copy;
403 dst_left -= bytes_to_copy;
404 dst_ptr += bytes_to_copy;
405 next_tcp_seq += bytes_to_copy;
410 int has_next = (csb0->
flags & VLIB_BUFFER_NEXT_PRESENT);
423 ASSERT (total_src_left == 0);
427 if (0 == dst_left && total_src_left)
435 gso_size, &dst_ptr, &dst_left,
436 next_tcp_seq, default_bflags);
482 u32 n_left_to_tx, *from, *from_end, *to_tx;
483 u32 n_bytes, n_buffers, n_packets;
484 u32 n_bytes_b0, n_bytes_b1, n_bytes_b2, n_bytes_b3;
488 u32 current_config_index = ~0;
489 u8 arc = im->output_feature_arc_index;
530 from_end = from + n_buffers;
547 while (from < from_end)
553 while (from + 8 <= from_end && n_left_to_tx >= 4)
555 u32 bi0, bi1, bi2, bi3;
556 u32 tx_swif0, tx_swif1, tx_swif2, tx_swif3;
588 ASSERT (b[0]->current_length > 0);
589 ASSERT (b[1]->current_length > 0);
590 ASSERT (b[2]->current_length > 0);
591 ASSERT (b[3]->current_length > 0);
602 n_bytes += n_bytes_b0 + n_bytes_b1;
603 n_bytes += n_bytes_b2 + n_bytes_b3;
623 thread_index, tx_swif0, 1,
632 thread_index, tx_swif1, 1,
641 thread_index, tx_swif2, 1,
649 thread_index, tx_swif3, 1,
656 (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
657 VNET_BUFFER_F_OFFLOAD_UDP_CKSUM |
658 VNET_BUFFER_F_OFFLOAD_IP_CKSUM))
670 while (from + 1 <= from_end && n_left_to_tx >= 1)
683 ASSERT (b[0]->current_length > 0);
687 n_bytes += n_bytes_b0;
708 n_bytes -= n_bytes_b0;
726 u32 *from_tx_seg = ptd->split_buffers;
728 while (n_tx_bufs > 0)
730 if (n_tx_bufs >= n_left_to_tx)
732 while (n_left_to_tx > 0)
734 to_tx[0] = from_tx_seg[0];
744 to_tx, n_left_to_tx);
746 while (n_tx_bufs > 0)
748 to_tx[0] = from_tx_seg[0];
756 n_bytes += n_tx_bytes;
761 (im->combined_sw_if_counters +
763 _vec_len (ptd->split_buffers), n_tx_bytes);
766 _vec_len (ptd->split_buffers) = 0;
779 thread_index, tx_swif0, 1,
803 int sw_if_index_from_buffer)
805 u32 n_left_from, *from;
812 if (sw_if_index_from_buffer == 0)
823 while (n_left_from > 0)
825 int classify_filter_result;
833 classify_filter_result =
836 if (classify_filter_result)
841 if (sw_if_index_from_buffer)
849 #ifndef CLIB_MARCH_VARIANT 905 u32 n_left_to_next, *from, *to_next;
906 u32 n_left_from, next_index;
914 next_index = node->cached_next_index;
916 while (n_left_from > 0)
920 while (n_left_from >= 4 && n_left_to_next >= 2)
922 u32 bi0, bi1, next0, next1;
955 n_left_to_next, bi0, bi1, next0,
959 while (n_left_from > 0 && n_left_to_next > 0)
982 n_left_to_next, bi0, next0);
1014 u32 n_left, *buffers;
1035 if (b0->
flags & VLIB_BUFFER_IS_TRACED)
1040 if (b1->
flags & VLIB_BUFFER_IS_TRACED)
1059 if (b0->
flags & VLIB_BUFFER_IS_TRACED)
1080 vnet_error_disposition_t disposition)
1094 sw_if_index = sw_if_indices;
1148 sw_if_index = sw_if_indices + off;
1177 i16 save_current_data;
1178 u16 save_current_length;
1226 if (b0->
flags & VNET_BUFFER_F_L2_HDR_OFFSET_VALID)
1238 u32 error_node_index;
1239 int drop_string_len;
1242 int error_string_len =
1250 drop_string_len = error_string_len +
vec_len (n->
name) + 2;
1253 while (last->
flags & VLIB_BUFFER_NEXT_PRESENT)
1262 - drop_string_len)))
1275 b0->
flags &= ~(VLIB_BUFFER_TOTAL_LENGTH_VALID);
1297 #ifndef CLIB_MARCH_VARIANT 1335 .name =
"error-drop",
1336 .vector_size =
sizeof (
u32),
1347 .name =
"error-punt",
1348 .vector_size =
sizeof (
u32),
1359 .name =
"interface-output",
1360 .vector_size =
sizeof (
u32),
1369 u32 last_sw_if_index = ~0;
1372 u32 *from, *to_next = 0;
1377 while (n_left_from > 0)
1389 if (
PREDICT_FALSE ((last_sw_if_index != sw_if_index0) || to_frame == 0))
1396 last_sw_if_index = sw_if_index0;
1413 .name =
"interface-tx",
1414 .vector_size =
sizeof (
u32),
1423 .arc_name =
"interface-output",
1425 .last_in_arc =
"interface-tx",
1430 .arc_name =
"interface-output",
1431 .node_name =
"span-output",
1436 .arc_name =
"interface-output",
1437 .node_name =
"ipsec-if-output",
1442 .arc_name =
"interface-output",
1443 .node_name =
"interface-tx",
1448 #ifndef CLIB_MARCH_VARIANT 1461 (vnm->
vlib_main, vnet_per_buffer_interface_output_node.index,
1473 u32 hw_if_index,
u32 node_index)
1478 (vnm->
vlib_main, vnet_per_buffer_interface_output_node.index, node_index);
u8 * format_vnet_interface_output_trace(u8 *s, va_list *va)
vnet_config_main_t config_main
static_always_inline uword vnet_interface_output_node_inline_gso(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vnet_main_t *vnm, vnet_hw_interface_t *hi, int do_tx_offloads, int do_segmentation)
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
vlib_main_t vlib_global_main
VNET_FEATURE_ARC_INIT(interface_output, static)
#define hash_set(h, key, value)
void vnet_set_interface_output_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Set interface output node - for interface registered without its output/tx nodes created because its ...
static_always_inline uword vnet_interface_output_node_inline(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vnet_main_t *vnm, vnet_hw_interface_t *hi, int do_tx_offloads)
#define hash_unset(h, key)
static void vlib_increment_combined_counter(vlib_combined_counter_main_t *cm, u32 thread_index, u32 index, u64 n_packets, u64 n_bytes)
Increment a combined counter.
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
u8 runtime_data[0]
Function dependent node-runtime data.
vnet_main_t * vnet_get_main(void)
static vnet_hw_interface_t * vnet_get_sup_hw_interface(vnet_main_t *vnm, u32 sw_if_index)
vnet_interface_main_t interface_main
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
static_always_inline void drop_one_buffer_and_count(vlib_main_t *vm, vnet_main_t *vnm, vlib_node_runtime_t *node, u32 *pbi0, u32 drop_error_code)
#define clib_memcpy_fast(a, b, c)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u16 current_length
Nbytes between current data and the end of this buffer.
static heap_elt_t * last(heap_header_t *h)
static_always_inline void tso_init_buf_from_template(vlib_main_t *vm, vlib_buffer_t *nb0, vlib_buffer_t *b0, u16 template_data_sz, u16 gso_size, u8 **p_dst_ptr, u16 *p_dst_left, u32 next_tcp_seq, u32 flags)
clib_error_t * vnet_per_buffer_interface_output_hw_interface_add_del(vnet_main_t *vnm, u32 hw_if_index, u32 is_create)
static void vnet_interface_output_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, uword n_buffers)
static vnet_sw_interface_t * vnet_get_sw_interface(vnet_main_t *vnm, u32 sw_if_index)
static void vlib_increment_simple_counter(vlib_simple_counter_main_t *cm, u32 thread_index, u32 index, u64 increment)
Increment a simple counter.
static_always_inline int vnet_have_features(u8 arc, u32 sw_if_index)
void vnet_pcap_drop_trace_filter_add_del(u32 error_index, int is_add)
#define VLIB_NODE_FN(node)
static uword vlib_buffer_length_in_chain(vlib_main_t *vm, vlib_buffer_t *b)
Get length in bytes of the buffer chain.
struct _tcp_header tcp_header_t
format_function_t format_vnet_sw_if_index_name
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
#define static_always_inline
uword * pcap_drop_filter_hash
static_always_inline u32 tso_segment_buffer(vlib_main_t *vm, vnet_interface_per_thread_data_t *ptd, int do_tx_offloads, u32 sbi0, vlib_buffer_t *sb0, u32 n_bytes_b0)
Allocate the necessary number of ptd->split_buffers, and segment the possibly chained buffer(s) from ...
#define vlib_prefetch_buffer_with_index(vm, bi, type)
Prefetch buffer metadata by buffer index The first 64 bytes of buffer contains most header informatio...
vl_api_interface_index_t sw_if_index
static_always_inline void calc_checksums(vlib_main_t *vm, vlib_buffer_t *b)
struct vnet_error_trace_t_ vnet_error_trace_t
vnet_hw_interface_flags_t flags
#define vlib_prefetch_buffer_header(b, type)
Prefetch buffer metadata.
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u32 filter_classify_table_index
vlib_frame_t * vlib_get_frame_to_node(vlib_main_t *vm, u32 to_node_index)
#define vlib_get_new_next_frame(vm, node, next_index, vectors, n_vectors_left)
A collection of simple counters.
Use the vpp classifier to decide whether to trace packets.
static uword interface_tx_node_fn(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *from_frame)
vlib_error_t error
Error code for buffers to be enqueued to error handler.
format_function_t format_vnet_sw_interface_name
u8 data[128 - 3 *sizeof(u32)]
static_always_inline void tso_init_buf_from_template_base(vlib_buffer_t *nb0, vlib_buffer_t *b0, u32 flags, u16 length)
static u8 * format_vnet_error_trace(u8 *s, va_list *va)
vlib_error_main_t error_main
VNET_FEATURE_INIT(span_tx, static)
uword vnet_interface_output_node(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
uword vlib_error_drop_buffers(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u32 next_buffer_stride, u32 n_buffers, u32 next_index, u32 drop_error_node, u32 drop_error_code)
void vlib_put_frame_to_node(vlib_main_t *vm, u32 to_node_index, vlib_frame_t *f)
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
static void * vnet_get_config_data(vnet_config_main_t *cm, u32 *config_index, u32 *next_index, u32 n_data_bytes)
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
vlib_simple_counter_main_t * sw_if_counters
static_always_inline u16 tso_alloc_tx_bufs(vlib_main_t *vm, vnet_interface_per_thread_data_t *ptd, vlib_buffer_t *b0, u32 n_bytes_b0, u16 l234_sz, u16 gso_size)
format_function_t * format_buffer
u32 node_index
Node index.
#define vlib_validate_buffer_enqueue_x2(vm, node, next_index, to_next, n_left_to_next, bi0, bi1, next0, next1)
Finish enqueueing two buffers forward in the graph.
#define vlib_validate_buffer_enqueue_x1(vm, node, next_index, to_next, n_left_to_next, bi0, next0)
Finish enqueueing one buffer forward in the graph.
#define vlib_get_next_frame(vm, node, next_index, vectors, n_vectors_left)
Get pointer to next frame vector data by (vlib_node_runtime_t, next_index).
static void interface_trace_buffers(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame)
static_always_inline void tso_fixup_segmented_buf(vlib_buffer_t *b0, u8 tcp_flags, int is_ip6)
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
#define VLIB_REGISTER_NODE(x,...)
char ** error_strings_heap
static_always_inline void vlib_buffer_enqueue_to_next(vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts, uword count)
static void pcap_drop_trace(vlib_main_t *vm, vnet_interface_main_t *im, vnet_pcap_t *pp, vlib_frame_t *f)
#define pool_is_free_index(P, I)
Use free bitmap to query whether given index is free.
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
void vlib_put_next_frame(vlib_main_t *vm, vlib_node_runtime_t *r, u32 next_index, u32 n_vectors_left)
Release pointer to next frame vector data.
static int vnet_is_packet_traced_inline(vlib_buffer_t *b, u32 classify_table_index, int func)
vnet_is_packet_traced
u16 ip6_tcp_udp_icmp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip6_header_t *ip0, int *bogus_lengthp)
#define hash_create(elts, value_bytes)
u32 output_node_next_index
u8 output_feature_arc_index
u16 ip4_tcp_udp_compute_checksum(vlib_main_t *vm, vlib_buffer_t *p0, ip4_header_t *ip0)
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
static_always_inline void vnet_interface_pcap_tx_trace(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, int sw_if_index_from_buffer)
#define VNET_FEATURES(...)
static void * vlib_add_trace(vlib_main_t *vm, vlib_node_runtime_t *r, vlib_buffer_t *b, u32 n_data_bytes)
VNET_HW_INTERFACE_ADD_DEL_FUNCTION(vnet_per_buffer_interface_output_hw_interface_add_del)
#define VLIB_BUFFER_DEFAULT_DATA_SIZE
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static_always_inline u32 vnet_get_feature_config_index(u8 arc, u32 sw_if_index)
u32 next_buffer
Next buffer for this linked-list of buffers.
vlib_node_main_t node_main
VLIB buffer representation.
vnet_sw_interface_t * sw_interfaces
static void * vlib_frame_vector_args(vlib_frame_t *f)
Get pointer to frame vector data.
#define clib_strnlen(s, m)
static_always_inline uword clib_count_equal_u32(u32 *data, uword max_count)
static vlib_node_t * vlib_get_node(vlib_main_t *vm, u32 i)
Get vlib node by index.
u16 flags
Copy of main node flags.
static void vlib_buffer_free_one(vlib_main_t *vm, u32 buffer_index)
Free one buffer Shorthand to free a single buffer chain.
static_always_inline uword interface_drop_punt(vlib_main_t *vm, vlib_node_runtime_t *node, vlib_frame_t *frame, vnet_error_disposition_t disposition)
static_always_inline void vlib_get_buffers(vlib_main_t *vm, u32 *bi, vlib_buffer_t **b, int count)
Translate array of buffer indices into buffer pointers.
#define VLIB_NODE_FLAG_TRACE
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
static u32 vlib_buffer_alloc(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Allocate buffers into supplied array.
static_always_inline vnet_feature_config_main_t * vnet_feature_get_config_main(u16 arc)
static void pcap_add_buffer(pcap_main_t *pm, struct vlib_main_t *vm, u32 buffer_index, u32 n_bytes_in_trace)
Add buffer (vlib_buffer_t) to the trace.
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
static u16 ip4_header_checksum(ip4_header_t *i)
#define clib_panic(format, args...)
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.