FD.io VPP  v16.12-rc0-308-g931be3a
Vector Packet Processing
buffer.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.h: VLIB buffers
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_h
41 #define included_vlib_buffer_h
42 
43 #include <vppinfra/types.h>
44 #include <vppinfra/cache.h>
45 #include <vppinfra/serialize.h>
46 #include <vppinfra/vector.h>
47 #include <vlib/error.h> /* for vlib_error_t */
48 
49 #if DPDK > 0
50 #include <rte_config.h>
51 #define VLIB_BUFFER_DATA_SIZE (2048)
52 #define VLIB_BUFFER_PRE_DATA_SIZE RTE_PKTMBUF_HEADROOM
53 #else
54 #include <vlib/config.h> /* for __PRE_DATA_SIZE */
55 #define VLIB_BUFFER_DATA_SIZE (512)
56 #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
57 #endif
58 
59 #ifdef CLIB_HAVE_VEC128
60 typedef u8x16 vlib_copy_unit_t;
61 #else
63 #endif
64 
65 /** \file
66  vlib buffer structure definition and a few select
67  access methods. This structure and the buffer allocation
68  mechanism should perhaps live in vnet, but it would take a lot
69  of typing to make it so.
70 */
71 
72 /* VLIB buffer representation. */
73 typedef struct
74 {
75  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
76  /* Offset within data[] that we are currently processing.
77  If negative current header points into predata area. */
78  i16 current_data; /**< signed offset in data[], pre_data[]
79  that we are currently processing.
80  If negative current header points into predata area.
81  */
82  u16 current_length; /**< Nbytes between current data and
83  the end of this buffer.
84  */
85  u32 flags; /**< buffer flags:
86  <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
87  <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
88  <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
89  <br> VLIB_BUFFER_REPL_FAIL: packet replication failure
90  <br> VLIB_BUFFER_RECYCLE: as it says
91  <br> VLIB_BUFFER_FLOW_REPORT: buffer is a flow report,
92  set to avoid adding it to a flow report
93  <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
94  */
95 #define VLIB_BUFFER_IS_TRACED (1 << 0)
96 #define VLIB_BUFFER_LOG2_NEXT_PRESENT (1)
97 #define VLIB_BUFFER_NEXT_PRESENT (1 << VLIB_BUFFER_LOG2_NEXT_PRESENT)
98 #define VLIB_BUFFER_IS_RECYCLED (1 << 2)
99 #define VLIB_BUFFER_TOTAL_LENGTH_VALID (1 << 3)
100 #define VLIB_BUFFER_REPL_FAIL (1 << 4)
101 #define VLIB_BUFFER_RECYCLE (1 << 5)
102 #define VLIB_BUFFER_FLOW_REPORT (1 << 6)
103 
104  /* User defined buffer flags. */
105 #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
106 #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
107 
108  u32 free_list_index; /**< Buffer free list that this buffer was
109  allocated from and will be freed to.
110  */
111 
113  /**< Only valid for first buffer in chain. Current length plus
114  total length given here give total number of bytes in buffer chain.
115  */
116 
117  u32 next_buffer; /**< Next buffer for this linked-list of buffers.
118  Only valid if VLIB_BUFFER_NEXT_PRESENT flag is set.
119  */
120 
121  vlib_error_t error; /**< Error code for buffers to be enqueued
122  to error handler.
123  */
124  u32 current_config_index; /**< Used by feature subgraph arcs to
125  visit enabled feature nodes
126  */
127 
128  u32 dont_waste_me; /**< Available space in the (precious)
129  first 32 octets of buffer metadata
130  Before allocating any of it, discussion required!
131  */
132 
133  u32 opaque[8]; /**< Opaque data used by sub-graphs for their own purposes.
134  See .../vnet/vnet/buffer.h
135  */
136  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
137 
138  u32 trace_index; /**< Specifies index into trace buffer
139  if VLIB_PACKET_IS_TRACED flag is set.
140  */
141  u32 recycle_count; /**< Used by L2 path recycle code */
142  u32 opaque2[14]; /**< More opaque data, currently unused */
143 
144  /***** end of second cache line */
145  CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
146  u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]; /**< Space for inserting data
147  before buffer start.
148  Packet rewrite string will be
149  rewritten backwards and may extend
150  back before buffer->data[0].
151  Must come directly before packet data.
152  */
153 
154  u8 data[0]; /**< Packet data. Hardware DMA here */
155 } vlib_buffer_t; /* Must be a multiple of 64B. */
156 
157 #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
158 
159 /** \brief Prefetch buffer metadata.
160  The first 64 bytes of buffer contains most header information
161 
162  @param b - (vlib_buffer_t *) pointer to the buffer
163  @param type - LOAD, STORE. In most cases, STORE is the right answer
164 */
165 
166 #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
167 
170 {
171  return (void *) (b + 1) + buffer_bytes;
172 }
173 
174 always_inline void
176 {
177  ASSERT (sizeof (b[0]) % 64 == 0);
178 
179  /* Rewrite data must be before and contiguous with packet data. */
181 }
182 
183 /** \brief Get pointer to current data to process
184 
185  @param b - (vlib_buffer_t *) pointer to the buffer
186  @return - (void *) (b->data + b->current_data)
187 */
188 
189 always_inline void *
191 {
192  /* Check bounds. */
193  ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
194  return b->data + b->current_data;
195 }
196 
197 /** \brief Advance current data pointer by the supplied (signed!) amount
198 
199  @param b - (vlib_buffer_t *) pointer to the buffer
200  @param l - (word) signed increment
201 */
202 always_inline void
204 {
205  ASSERT (b->current_length >= l);
206  b->current_data += l;
207  b->current_length -= l;
208 }
209 
210 /** \brief Reset current header & length to state they were in when
211  packet was received.
212 
213  @param b - (vlib_buffer_t *) pointer to the buffer
214 */
215 
216 always_inline void
218 {
219  b->current_length += clib_max (b->current_data, 0);
220  b->current_data = 0;
221 }
222 
223 /** \brief Get pointer to buffer's opaque data array
224 
225  @param b - (vlib_buffer_t *) pointer to the buffer
226  @return - (void *) b->opaque
227 */
228 always_inline void *
230 {
231  return (void *) b->opaque;
232 }
233 
234 /** \brief Get pointer to buffer's opaque2 data array
235 
236  @param b - (vlib_buffer_t *) pointer to the buffer
237  @return - (void *) b->opaque2
238 */
239 always_inline void *
241 {
242  return (void *) b->opaque2;
243 }
244 
245 /* Forward declaration. */
246 struct vlib_main_t;
247 
249 {
250  /* Template buffer used to initialize first 16 bytes of buffers
251  allocated on this free list. */
253 
254  /* Our index into vlib_main_t's buffer_free_list_pool. */
256 
257  /* Number of data bytes for buffers in this free list. */
259 
260  /* Number of buffers to allocate when we need to allocate new buffers
261  from physmem heap. */
263 
264  /* Total number of buffers allocated from this free list. */
266 
267  /* Vector of free buffers. Each element is a byte offset into I/O heap.
268  Aligned vectors always has naturally aligned vlib_copy_unit_t sized chunks
269  of buffer indices. Unaligned vector has any left over. This is meant to
270  speed up copy routines. */
272 
273  /* Memory chunks allocated for this free list
274  recorded here so they can be freed when free list
275  is deleted. */
277 
278  /* Free list name. */
280 
281  /* Callback functions to initialize newly allocated buffers.
282  If null buffers are zeroed. */
283  void (*buffer_init_function) (struct vlib_main_t * vm,
284  struct vlib_buffer_free_list_t * fl,
285  u32 * buffers, u32 n_buffers);
286 
287  /* Callback function to announce that buffers have been
288  added to the freelist */
290  (struct vlib_main_t * vm, struct vlib_buffer_free_list_t * fl);
291 
293 } __attribute__ ((aligned (16))) vlib_buffer_free_list_t;
294 
295 typedef struct
296 {
297  /* Buffer free callback, for subversive activities */
298  u32 (*buffer_free_callback) (struct vlib_main_t * vm,
299  u32 * buffers,
300  u32 n_buffers, u32 follow_buffer_next);
301  /* Pool of buffer free lists.
302  Multiple free lists exist for packet generator which uses
303  separate free lists for each packet stream --- so as to avoid
304  initializing static data for each packet generated. */
306 #define VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX (0)
307 #define VLIB_BUFFER_DEFAULT_FREE_LIST_BYTES VLIB_BUFFER_DATA_SIZE
308 
309  /* Hash table mapping buffer size (rounded to next unit of
310  sizeof (vlib_buffer_t)) to free list index. */
312 
313  /* Hash table mapping buffer index into number
314  0 => allocated but free, 1 => allocated and not-free.
315  If buffer index is not in hash table then this buffer
316  has never been allocated. */
318 
319  /* List of free-lists needing Blue Light Special announcements */
321 
322  /* Vector of rte_mempools per socket */
323 #if DPDK == 1
324  struct rte_mempool **pktmbuf_pools;
325 #endif
327 
328 typedef struct
329 {
331 
332  u32 first_buffer, last_buffer;
333 
334  union
335  {
336  struct
337  {
338  /* Total accumulated bytes in chain starting with first_buffer. */
340 
341  /* Max number of bytes to accumulate in chain starting with first_buffer.
342  As this limit is reached buffers are enqueued to next node. */
344 
345  /* Next node to enqueue buffers to relative to current process node. */
347 
348  /* Free list to use to allocate new buffers. */
350  } tx;
351 
352  struct
353  {
354  /* CLIB fifo of buffer indices waiting to be unserialized. */
356 
357  /* Event type used to signal that RX buffers have been added to fifo. */
359  } rx;
360  };
362 
366  struct vlib_main_t *vm,
368 
371 void *vlib_set_buffer_free_callback (struct vlib_main_t *vm, void *fp);
372 
375 {
376  serialize_stream_t *s = &m->stream;
380  return sm->tx.n_total_data_bytes + s->current_buffer_index +
382 }
383 
384 #if DPDK > 0
385 #define rte_mbuf_from_vlib_buffer(x) (((struct rte_mbuf *)x) - 1)
386 #define vlib_buffer_from_rte_mbuf(x) ((vlib_buffer_t *)(x+1))
387 #endif
388 
389 /*
390  */
391 
392 /** \brief Compile time buffer trajectory tracing option
393  Turn this on if you run into "bad monkey" contexts,
394  and you want to know exactly which nodes they've visited...
395  See vlib/main.c...
396 */
397 #define VLIB_BUFFER_TRACE_TRAJECTORY 0
398 
399 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
400 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) (b)->pre_data[0]=0
401 #else
402 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
403 #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
404 
405 #endif /* included_vlib_buffer_h */
406 
407 /*
408  * fd.io coding-style-patch-verification: ON
409  *
410  * Local Variables:
411  * eval: (c-set-style "gnu")
412  * End:
413  */
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:68
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:124
u32 free_list_index
Buffer free list that this buffer was allocated from and will be freed to.
Definition: buffer.h:108
static void vlib_buffer_struct_is_sane(vlib_buffer_t *b)
Definition: buffer.h:175
vlib_buffer_t buffer_init_template
Definition: buffer.h:252
struct vlib_serialize_buffer_main_t::@22::@24 tx
struct vlib_main_t * vlib_main
Definition: buffer.h:330
u32 serialize_close_vlib_buffer(serialize_main_t *m)
Definition: dpdk_buffer.c:1246
void ** buffer_memory_allocated
Definition: buffer.h:276
u32 recycle_count
Used by L2 path recycle code.
Definition: buffer.h:141
u32 min_n_buffers_each_physmem_alloc
Definition: buffer.h:262
void * vlib_set_buffer_free_callback(struct vlib_main_t *vm, void *fp)
Definition: dpdk_buffer.c:689
static u32 serialize_vlib_buffer_n_bytes(serialize_main_t *m)
Definition: buffer.h:374
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:78
void(* buffer_init_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl, u32 *buffers, u32 n_buffers)
Definition: buffer.h:283
void unserialize_close_vlib_buffer(serialize_main_t *m)
Definition: dpdk_buffer.c:1271
#define always_inline
Definition: clib.h:84
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:190
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:146
unsigned long u64
Definition: types.h:89
vlib_buffer_free_list_t ** announce_list
Definition: buffer.h:320
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:52
u32 vlib_error_t
Definition: error.h:44
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:82
u32 dont_waste_me
Available space in the (precious) first 32 octets of buffer metadata Before allocating any of it...
Definition: buffer.h:128
static void * vlib_get_buffer_opaque(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque data array.
Definition: buffer.h:229
void serialize_open_vlib_buffer(serialize_main_t *m, struct vlib_main_t *vm, vlib_serialize_buffer_main_t *sm)
Definition: dpdk_buffer.c:1232
static void * vlib_get_buffer_opaque2(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque2 data array.
Definition: buffer.h:240
static void vlib_buffer_reset(vlib_buffer_t *b)
Reset current header & length to state they were in when packet was received.
Definition: buffer.h:217
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:121
#define uword_to_pointer(u, type)
Definition: types.h:136
struct rte_mempool ** pktmbuf_pools
Definition: buffer.h:324
serialize_stream_t stream
Definition: serialize.h:147
u32 current_buffer_index
Definition: serialize.h:62
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:203
void(* buffers_added_to_freelist_function)(struct vlib_main_t *vm, struct vlib_buffer_free_list_t *fl)
Definition: buffer.h:290
uword data_function_opaque
Definition: serialize.h:74
#define ASSERT(truth)
unsigned int u32
Definition: types.h:88
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:117
u64 vlib_copy_unit_t
Definition: buffer.h:62
void unserialize_open_vlib_buffer(serialize_main_t *m, struct vlib_main_t *vm, vlib_serialize_buffer_main_t *sm)
Definition: dpdk_buffer.c:1239
#define clib_max(x, y)
Definition: clib.h:319
u64 uword
Definition: types.h:112
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:112
unsigned short u16
Definition: types.h:57
i64 word
Definition: types.h:111
static vlib_buffer_t * vlib_buffer_next_contiguous(vlib_buffer_t *b, u32 buffer_bytes)
Definition: buffer.h:169
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
vlib_buffer_free_list_t * buffer_free_list_pool
Definition: buffer.h:305
uword * buffer_known_hash
Definition: buffer.h:317
u32 opaque[8]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:133
short i16
Definition: types.h:46
u8 data[0]
Packet data.
Definition: buffer.h:154
uword buffer_init_function_opaque
Definition: buffer.h:292
u32 flags
buffer flags: VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:85
u32 trace_index
Specifies index into trace buffer if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:138
uword * free_list_by_size
Definition: buffer.h:311
u32 opaque2[14]
More opaque data, currently unused.
Definition: buffer.h:142