FD.io VPP  v20.01-46-g7006026de
Vector Packet Processing
buffer.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * buffer.h: VLIB buffers
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #ifndef included_vlib_buffer_h
41 #define included_vlib_buffer_h
42 
43 #include <vppinfra/types.h>
44 #include <vppinfra/cache.h>
45 #include <vppinfra/serialize.h>
46 #include <vppinfra/vector.h>
47 #include <vppinfra/lock.h>
48 #include <vlib/error.h> /* for vlib_error_t */
49 
50 #include <vlib/config.h> /* for __PRE_DATA_SIZE */
51 #define VLIB_BUFFER_PRE_DATA_SIZE __PRE_DATA_SIZE
52 
53 #define VLIB_BUFFER_DEFAULT_DATA_SIZE (2048)
54 
55 /* Minimum buffer chain segment size. Does not apply to last buffer in chain.
56  Dataplane code can safely asume that specified amount of data is not split
57  into 2 chained buffers */
58 #define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE (128)
59 
60 /* Amount of head buffer data copied to each replica head buffer */
61 #define VLIB_BUFFER_CLONE_HEAD_SIZE (256)
62 
63 /** \file
64  vlib buffer structure definition and a few select
65  access methods. This structure and the buffer allocation
66  mechanism should perhaps live in vnet, but it would take a lot
67  of typing to make it so.
68 */
69 
70 /**
71  * Buffer Flags
72  */
73 #define foreach_vlib_buffer_flag \
74  _( 0, IS_TRACED, 0) \
75  _( 1, NEXT_PRESENT, "next-present") \
76  _( 2, TOTAL_LENGTH_VALID, 0) \
77  _( 3, EXT_HDR_VALID, "ext-hdr-valid")
78 
79 /* NOTE: only buffer generic flags should be defined here, please consider
80  using user flags. i.e. src/vnet/buffer.h */
81 
82 enum
83 {
84 #define _(bit, name, v) VLIB_BUFFER_##name = (1 << (bit)),
86 #undef _
87 };
88 
89 enum
90 {
91 #define _(bit, name, v) VLIB_BUFFER_LOG2_##name = (bit),
93 #undef _
94 };
95 
96  /* User defined buffer flags. */
97 #define LOG2_VLIB_BUFFER_FLAG_USER(n) (32 - (n))
98 #define VLIB_BUFFER_FLAG_USER(n) (1 << LOG2_VLIB_BUFFER_FLAG_USER(n))
99 #define VLIB_BUFFER_FLAGS_ALL (0x0f)
100 
101 /** VLIB buffer representation. */
102 typedef union
103 {
104  struct
105  {
106  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
107 
108  /** signed offset in data[], pre_data[] that we are currently
109  * processing. If negative current header points into predata area. */
111 
112  /** Nbytes between current data and the end of this buffer. */
114 
115  /** buffer flags:
116  <br> VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index,
117  <br> VLIB_BUFFER_IS_TRACED: trace this buffer.
118  <br> VLIB_BUFFER_NEXT_PRESENT: this is a multi-chunk buffer.
119  <br> VLIB_BUFFER_TOTAL_LENGTH_VALID: as it says
120  <br> VLIB_BUFFER_EXT_HDR_VALID: buffer contains valid external buffer manager header,
121  set to avoid adding it to a flow report
122  <br> VLIB_BUFFER_FLAG_USER(n): user-defined bit N
123  */
125 
126  /** Generic flow identifier */
128 
129  /** Reference count for this buffer. */
130  volatile u8 ref_count;
131 
132  /** index of buffer pool this buffer belongs. */
134 
135  /** Error code for buffers to be enqueued to error handler. */
137 
138  /** Next buffer for this linked-list of buffers. Only valid if
139  * VLIB_BUFFER_NEXT_PRESENT flag is set. */
141 
142  /** The following fields can be in a union because once a packet enters
143  * the punt path, it is no longer on a feature arc */
144  union
145  {
146  /** Used by feature subgraph arcs to visit enabled feature nodes */
148  /* the reason the packet once punted */
150  };
151 
152  /** Opaque data used by sub-graphs for their own purposes. */
153  u32 opaque[10];
154 
155  /** part of buffer metadata which is initialized on alloc ends here. */
156  STRUCT_MARK (template_end);
157 
158  /** start of 2nd cache line */
159  CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
160 
161  /** Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is
162  * set. */
164 
165  /** Only valid for first buffer in chain. Current length plus total length
166  * given here give total number of bytes in buffer chain. */
168 
169  /**< More opaque data, see ../vnet/vnet/buffer.h */
170  u32 opaque2[14];
171 
172  /** start of third cache line */
173  CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
174 
175  /** Space for inserting data before buffer start. Packet rewrite string
176  * will be rewritten backwards and may extend back before
177  * buffer->data[0]. Must come directly before packet data. */
179 
180  /** Packet data */
181  u8 data[0];
182  };
183 #ifdef CLIB_HAVE_VEC128
184  u8x16 as_u8x16[4];
185 #endif
186 #ifdef CLIB_HAVE_VEC256
187  u8x32 as_u8x32[2];
188 #endif
189 #ifdef CLIB_HAVE_VEC512
190  u8x64 as_u8x64[1];
191 #endif
192 } vlib_buffer_t;
193 
194 #define VLIB_BUFFER_HDR_SIZE (sizeof(vlib_buffer_t) - VLIB_BUFFER_PRE_DATA_SIZE)
195 
196 /** \brief Prefetch buffer metadata.
197  The first 64 bytes of buffer contains most header information
198 
199  @param b - (vlib_buffer_t *) pointer to the buffer
200  @param type - LOAD, STORE. In most cases, STORE is the right answer
201 */
202 
203 #define vlib_prefetch_buffer_header(b,type) CLIB_PREFETCH (b, 64, type)
204 #define vlib_prefetch_buffer_data(b,type) \
205  CLIB_PREFETCH (vlib_buffer_get_current(b), CLIB_CACHE_LINE_BYTES, type)
206 
207 always_inline void
209 {
210  ASSERT (sizeof (b[0]) % 64 == 0);
211 
212  /* Rewrite data must be before and contiguous with packet data. */
214 }
215 
218 {
219  return pointer_to_uword (b->data);
220 }
221 
222 /** \brief Get pointer to current data to process
223 
224  @param b - (vlib_buffer_t *) pointer to the buffer
225  @return - (void *) (b->data + b->current_data)
226 */
227 
228 always_inline void *
230 {
231  /* Check bounds. */
232  ASSERT ((signed) b->current_data >= (signed) -VLIB_BUFFER_PRE_DATA_SIZE);
233  return b->data + b->current_data;
234 }
235 
238 {
239  return vlib_buffer_get_va (b) + b->current_data;
240 }
241 
242 /** \brief Advance current data pointer by the supplied (signed!) amount
243 
244  @param b - (vlib_buffer_t *) pointer to the buffer
245  @param l - (word) signed increment
246 */
247 always_inline void
249 {
250  ASSERT (b->current_length >= l);
251  b->current_data += l;
252  b->current_length -= l;
253 
254  ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0 ||
256 }
257 
258 /** \brief Check if there is enough space in buffer to advance
259 
260  @param b - (vlib_buffer_t *) pointer to the buffer
261  @param l - (word) size to check
262  @return - 0 if there is less space than 'l' in buffer
263 */
266 {
267  return b->current_length >= l;
268 }
269 
270 /** \brief Reset current header & length to state they were in when
271  packet was received.
272 
273  @param b - (vlib_buffer_t *) pointer to the buffer
274 */
275 
276 always_inline void
278 {
279  b->current_length += clib_max (b->current_data, 0);
280  b->current_data = 0;
281 }
282 
283 /** \brief Get pointer to buffer's opaque data array
284 
285  @param b - (vlib_buffer_t *) pointer to the buffer
286  @return - (void *) b->opaque
287 */
288 always_inline void *
290 {
291  return (void *) b->opaque;
292 }
293 
294 /** \brief Get pointer to buffer's opaque2 data array
295 
296  @param b - (vlib_buffer_t *) pointer to the buffer
297  @return - (void *) b->opaque2
298 */
299 always_inline void *
301 {
302  return (void *) b->opaque2;
303 }
304 
305 /** \brief Get pointer to the end of buffer's data
306  * @param b pointer to the buffer
307  * @return pointer to tail of packet's data
308  */
311 {
312  return b->data + b->current_data + b->current_length;
313 }
314 
315 /** \brief Append uninitialized data to buffer
316  * @param b pointer to the buffer
317  * @param size number of uninitialized bytes
318  * @return pointer to beginning of uninitialized data
319  */
320 always_inline void *
322 {
323  void *p = vlib_buffer_get_tail (b);
324  /* XXX make sure there's enough space */
325  b->current_length += size;
326  return p;
327 }
328 
329 /** \brief Prepend uninitialized data to buffer
330  * @param b pointer to the buffer
331  * @param size number of uninitialized bytes
332  * @return pointer to beginning of uninitialized data
333  */
334 always_inline void *
336 {
338  b->current_data -= size;
339  b->current_length += size;
340 
341  return vlib_buffer_get_current (b);
342 }
343 
344 /** \brief Make head room, typically for packet headers
345  * @param b pointer to the buffer
346  * @param size number of head room bytes
347  * @return pointer to start of buffer (current data)
348  */
349 always_inline void *
351 {
352  b->current_data += size;
353  return vlib_buffer_get_current (b);
354 }
355 
356 /** \brief Construct a trace handle from thread and pool index
357  * @param thread Thread id
358  * @param pool_index Pool index
359  * @return trace handle
360  */
363 {
364  u32 rv;
365  ASSERT (thread < 0xff);
366  ASSERT (pool_index < 0x00FFFFFF);
367  rv = (thread << 24) | (pool_index & 0x00FFFFFF);
368  return rv;
369 }
370 
371 /** \brief Extract the thread id from a trace handle
372  * @param trace_handle the trace handle
373  * @return the thread id
374  */
377 {
378  u32 trace_handle = b->trace_handle;
379 
380  return trace_handle >> 24;
381 }
382 
383 /** \brief Extract the trace (pool) index from a trace handle
384  * @param trace_handle the trace handle
385  * @return the trace index
386  */
389 {
390  u32 trace_handle = b->trace_handle;
391  return trace_handle & 0x00FFFFFF;
392 }
393 
394 /** \brief Retrieve bytes from buffer head
395  * @param b pointer to the buffer
396  * @param size number of bytes to pull
397  * @return pointer to start of buffer (current data)
398  */
399 always_inline void *
401 {
403  return 0;
404 
405  void *data = vlib_buffer_get_current (b);
406  vlib_buffer_advance (b, size);
407  return data;
408 }
409 
410 /* Forward declaration. */
411 struct vlib_main_t;
412 
413 #define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ 512
414 
415 typedef struct
416 {
417  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
421 
422 typedef struct
423 {
424  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
437 
438  /* per-thread data */
440 
441  /* buffer metadata template */
444 
445 #define VLIB_BUFFER_MAX_NUMA_NODES 32
446 
447 typedef struct
448 {
449  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
450  /* Virtual memory address and size of buffer memory, used for calculating
451  buffer index */
455 
456  /* Hash table mapping buffer index into number
457  0 => allocated but free, 1 => allocated and not-free.
458  If buffer index is not in hash table then this buffer
459  has never been allocated. */
462  u8 default_buffer_pool_index_for_numa[VLIB_BUFFER_MAX_NUMA_NODES];
463 
464  /* config */
468 
469  /* logging */
472 
474 
475 /*
476  */
477 
478 /** \brief Compile time buffer trajectory tracing option
479  Turn this on if you run into "bad monkey" contexts,
480  and you want to know exactly which nodes they've visited...
481  See vlib/main.c...
482 */
483 #define VLIB_BUFFER_TRACE_TRAJECTORY 0
484 
485 #if VLIB_BUFFER_TRACE_TRAJECTORY > 0
486 extern void (*vlib_buffer_trace_trajectory_cb) (vlib_buffer_t * b, u32 index);
487 extern void (*vlib_buffer_trace_trajectory_init_cb) (vlib_buffer_t * b);
488 extern void vlib_buffer_trace_trajectory_init (vlib_buffer_t * b);
489 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b) \
490  vlib_buffer_trace_trajectory_init (b);
491 #else
492 #define VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b)
493 #endif /* VLIB_BUFFER_TRACE_TRAJECTORY */
494 
495 extern u16 __vlib_buffer_external_hdr_size;
496 #define VLIB_BUFFER_SET_EXT_HDR_SIZE(x) \
497 static void __clib_constructor \
498 vnet_buffer_set_ext_hdr_size() \
499 { \
500  if (__vlib_buffer_external_hdr_size) \
501  clib_error ("buffer external header space already set"); \
502  __vlib_buffer_external_hdr_size = CLIB_CACHE_LINE_ROUND (x); \
503 }
504 
505 #endif /* included_vlib_buffer_h */
506 
507 /*
508  * fd.io coding-style-patch-verification: ON
509  *
510  * Local Variables:
511  * eval: (c-set-style "gnu")
512  * End:
513  */
u32 opaque2[14]
Definition: buffer.h:170
u32 flags
buffer flags: VLIB_BUFFER_FREE_LIST_INDEX_MASK: bits used to store free list index, VLIB_BUFFER_IS_TRACED: trace this buffer.
Definition: buffer.h:124
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
static u8 * vlib_buffer_get_tail(vlib_buffer_t *b)
Get pointer to the end of buffer&#39;s data.
Definition: buffer.h:310
static uword vlib_buffer_get_current_va(vlib_buffer_t *b)
Definition: buffer.h:237
static u32 vlib_buffer_make_trace_handle(u32 thread, u32 pool_index)
Construct a trace handle from thread and pool index.
Definition: buffer.h:362
static void vlib_buffer_reset(vlib_buffer_t *b)
Reset current header & length to state they were in when packet was received.
Definition: buffer.h:277
static u32 vlib_buffer_get_trace_index(vlib_buffer_t *b)
Extract the trace (pool) index from a trace handle.
Definition: buffer.h:388
i16 current_data
signed offset in data[], pre_data[] that we are currently processing.
Definition: buffer.h:110
#define VLIB_BUFFER_PRE_DATA_SIZE
Definition: buffer.h:51
u32 physmem_map_index
Definition: buffer.h:430
u16 current_length
Nbytes between current data and the end of this buffer.
Definition: buffer.h:113
u8 data[0]
Packet data.
Definition: buffer.h:181
u16 vlib_error_t
Definition: error.h:43
#define STRUCT_MARK(mark)
Definition: clib.h:70
uword log2_page_size
Definition: buffer.h:427
unsigned char u8
Definition: types.h:56
u8 buffer_pool_index
index of buffer pool this buffer belongs.
Definition: buffer.h:133
vlib_buffer_pool_thread_t * threads
Definition: buffer.h:439
u32 vlib_log_class_t
Definition: vlib.h:51
i64 word
Definition: types.h:111
uword buffer_mem_size
Definition: buffer.h:453
pthread_t thread[MAX_CONNS]
Definition: main.c:142
unsigned int u32
Definition: types.h:88
vlib_error_t error
Error code for buffers to be enqueued to error handler.
Definition: buffer.h:136
static void * vlib_buffer_make_headroom(vlib_buffer_t *b, u8 size)
Make head room, typically for packet headers.
Definition: buffer.h:350
unsigned short u16
Definition: types.h:57
u64 size
Definition: vhost_user.h:140
vlib_buffer_t buffer_template
Definition: buffer.h:442
static void * vlib_buffer_get_current(vlib_buffer_t *b)
Get pointer to current data to process.
Definition: buffer.h:229
vlib_buffer_pool_t * buffer_pools
Definition: buffer.h:454
u32 trace_handle
Specifies trace buffer handle if VLIB_PACKET_IS_TRACED flag is set.
Definition: buffer.h:163
#define always_inline
Definition: ipsec.h:28
vlib_main_t * vm
Definition: in2out_ed.c:1810
vlib_log_class_t log_default
Definition: buffer.h:470
u32 punt_reason
Definition: buffer.h:149
clib_spinlock_t buffer_known_hash_lockp
Definition: buffer.h:461
u32 flow_id
Generic flow identifier.
Definition: buffer.h:127
#define VLIB_BUFFER_MAX_NUMA_NODES
Definition: buffer.h:445
static u32 vlib_buffer_get_trace_thread(vlib_buffer_t *b)
Extract the thread id from a trace handle.
Definition: buffer.h:376
clib_spinlock_t lock
Definition: buffer.h:436
static void * vlib_get_buffer_opaque2(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque2 data array.
Definition: buffer.h:300
u32 current_config_index
Used by feature subgraph arcs to visit enabled feature nodes.
Definition: buffer.h:147
u32 buffers_per_numa
Definition: buffer.h:465
#define ASSERT(truth)
u8 pre_data[VLIB_BUFFER_PRE_DATA_SIZE]
Space for inserting data before buffer start.
Definition: buffer.h:178
u8 data[128]
Definition: ipsec_types.api:87
static void vlib_buffer_advance(vlib_buffer_t *b, word l)
Advance current data pointer by the supplied (signed!) amount.
Definition: buffer.h:248
static uword pointer_to_uword(const void *p)
Definition: types.h:131
#define clib_max(x, y)
Definition: clib.h:288
static void * vlib_buffer_push_uninit(vlib_buffer_t *b, u8 size)
Prepend uninitialized data to buffer.
Definition: buffer.h:335
static u8 vlib_buffer_has_space(vlib_buffer_t *b, word l)
Check if there is enough space in buffer to advance.
Definition: buffer.h:265
#define VLIB_BUFFER_POOL_PER_THREAD_CACHE_SZ
Definition: buffer.h:413
static void * vlib_get_buffer_opaque(vlib_buffer_t *b)
Get pointer to buffer&#39;s opaque data array.
Definition: buffer.h:289
u32 next_buffer
Next buffer for this linked-list of buffers.
Definition: buffer.h:140
VLIB buffer representation.
Definition: buffer.h:102
u64 uword
Definition: types.h:112
uword buffer_mem_start
Definition: buffer.h:452
uword * buffer_known_hash
Definition: buffer.h:460
u32 default_data_size
Definition: buffer.h:467
#define VLIB_BUFFER_MIN_CHAIN_SEG_SIZE
Definition: buffer.h:58
clib_error_t * vlib_buffer_main_init(struct vlib_main_t *vm)
Definition: buffer.c:804
static uword vlib_buffer_get_va(vlib_buffer_t *b)
Definition: buffer.h:217
static void * vlib_buffer_pull(vlib_buffer_t *b, u8 size)
Retrieve bytes from buffer head.
Definition: buffer.h:400
#define foreach_vlib_buffer_flag
Buffer Flags.
Definition: buffer.h:73
static void * vlib_buffer_put_uninit(vlib_buffer_t *b, u16 size)
Append uninitialized data to buffer.
Definition: buffer.h:321
u32 total_length_not_including_first_buffer
Only valid for first buffer in chain.
Definition: buffer.h:167
static void vlib_buffer_struct_is_sane(vlib_buffer_t *b)
Definition: buffer.h:208
volatile u8 ref_count
Reference count for this buffer.
Definition: buffer.h:130
u32 opaque[10]
Opaque data used by sub-graphs for their own purposes.
Definition: buffer.h:153
signed short i16
Definition: types.h:46