FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
mem.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus
17 
18  Permission is hereby granted, free of charge, to any person obtaining
19  a copy of this software and associated documentation files (the
20  "Software"), to deal in the Software without restriction, including
21  without limitation the rights to use, copy, modify, merge, publish,
22  distribute, sublicense, and/or sell copies of the Software, and to
23  permit persons to whom the Software is furnished to do so, subject to
24  the following conditions:
25 
26  The above copyright notice and this permission notice shall be
27  included in all copies or substantial portions of the Software.
28 
29  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
36 */
37 
38 #ifndef _included_clib_mem_h
39 #define _included_clib_mem_h
40 
41 #include <stdarg.h>
42 #include <unistd.h>
43 #include <sys/mman.h>
44 
45 #include <vppinfra/clib.h> /* uword, etc */
46 #include <vppinfra/clib_error.h>
47 
48 #include <vppinfra/dlmalloc.h>
49 
50 #include <vppinfra/os.h>
51 #include <vppinfra/string.h> /* memcpy, clib_memset */
52 #include <vppinfra/sanitizer.h>
53 
54 #define CLIB_MAX_MHEAPS 256
55 #define CLIB_MAX_NUMAS 8
56 
57 /* Unspecified NUMA socket */
58 #define VEC_NUMA_UNSPECIFIED (0xFF)
59 
60 /* Per CPU heaps. */
63 
64 always_inline void *
66 {
67  int cpu = os_get_thread_index ();
68  return clib_per_cpu_mheaps[cpu];
69 }
70 
71 always_inline void *
73 {
74  int cpu = os_get_thread_index ();
75  void *old = clib_per_cpu_mheaps[cpu];
76  clib_per_cpu_mheaps[cpu] = new_heap;
77  return old;
78 }
79 
80 always_inline void *
82 {
84  return clib_per_numa_mheaps[numa_id];
85 }
86 
87 always_inline void *
89 {
90  int numa = os_get_numa_index ();
91  void *old = clib_per_numa_mheaps[numa];
92  clib_per_numa_mheaps[numa] = new_heap;
93  return old;
94 }
95 
96 always_inline void
98 {
99  /*
100  * Find an unused slot in the per-cpu-mheaps array,
101  * and grab it for this thread. We need to be able to
102  * push/pop the thread heap without affecting other thread(s).
103  */
104  int i;
105  if (__os_thread_index != 0)
106  return;
107  for (i = 0; i < ARRAY_LEN (clib_per_cpu_mheaps); i++)
109  0, clib_per_cpu_mheaps[0]))
110  {
112  break;
113  }
114  ASSERT (__os_thread_index > 0);
115 }
116 
119 {
121 }
122 
123 /* Memory allocator which may call os_out_of_memory() if it fails */
124 always_inline void *
126  int os_out_of_memory_on_failure)
127 {
128  void *heap, *p;
129  uword cpu;
130 
131  if (align_offset > align)
132  {
133  if (align > 0)
134  align_offset %= align;
135  else
136  align_offset = align;
137  }
138 
139  cpu = os_get_thread_index ();
140  heap = clib_per_cpu_mheaps[cpu];
141 
142  p = mspace_get_aligned (heap, size, align, align_offset);
143 
144  if (PREDICT_FALSE (0 == p))
145  {
146  if (os_out_of_memory_on_failure)
147  os_out_of_memory ();
148  return 0;
149  }
150 
151  CLIB_MEM_UNPOISON (p, size);
152  return p;
153 }
154 
155 /* Memory allocator which calls os_out_of_memory() when it fails */
156 always_inline void *
158 {
159  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
160  /* align_offset */ 0,
161  /* os_out_of_memory */ 1);
162 }
163 
164 always_inline void *
166 {
167  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
168  /* os_out_of_memory */ 1);
169 }
170 
171 /* Memory allocator which calls os_out_of_memory() when it fails */
172 always_inline void *
174 {
175  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
176  /* align_offset */ 0,
177  /* os_out_of_memory */ 0);
178 }
179 
180 always_inline void *
182 {
183  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
184  /* os_out_of_memory */ 0);
185 }
186 
187 
188 
189 /* Memory allocator which panics when it fails.
190  Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
191 #define clib_mem_alloc_aligned_no_fail(size,align) \
192 ({ \
193  uword _clib_mem_alloc_size = (size); \
194  void * _clib_mem_alloc_p; \
195  _clib_mem_alloc_p = clib_mem_alloc_aligned (_clib_mem_alloc_size, (align)); \
196  if (! _clib_mem_alloc_p) \
197  clib_panic ("failed to allocate %d bytes", _clib_mem_alloc_size); \
198  _clib_mem_alloc_p; \
199 })
200 
201 #define clib_mem_alloc_no_fail(size) clib_mem_alloc_aligned_no_fail(size,1)
202 
203 /* Alias to stack allocator for naming consistency. */
204 #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
205 
208 {
209  void *heap = clib_mem_get_per_cpu_heap ();
210 
211  return mspace_is_heap_object (heap, p);
212 }
213 
214 always_inline void
215 clib_mem_free (void *p)
216 {
217  u8 *heap = clib_mem_get_per_cpu_heap ();
218 
219  /* Make sure object is in the correct heap. */
221 
223 
224  mspace_put (heap, p);
225 }
226 
227 always_inline void *
228 clib_mem_realloc (void *p, uword new_size, uword old_size)
229 {
230  /* By default use alloc, copy and free to emulate realloc. */
231  void *q = clib_mem_alloc (new_size);
232  if (q)
233  {
234  uword copy_size;
235  if (old_size < new_size)
236  copy_size = old_size;
237  else
238  copy_size = new_size;
239  clib_memcpy_fast (q, p, copy_size);
240  clib_mem_free (p);
241  }
242  return q;
243 }
244 
246 clib_mem_size (void *p)
247 {
249  return clib_mem_size_nocheck (p);
250 }
251 
252 always_inline void
254 {
255  uword size = clib_mem_size (p);
256  CLIB_MEM_UNPOISON (p, size);
257  memset_s_inline (p, size, 0, size);
258  clib_mem_free (p);
259 }
260 
261 always_inline void *
263 {
264  return clib_mem_get_per_cpu_heap ();
265 }
266 
267 always_inline void *
268 clib_mem_set_heap (void *heap)
269 {
270  return clib_mem_set_per_cpu_heap (heap);
271 }
272 
273 void *clib_mem_init (void *heap, uword size);
276  u8 numa);
277 
278 void clib_mem_exit (void);
279 
281 
282 void clib_mem_validate (void);
283 
284 void clib_mem_trace (int enable);
285 
286 int clib_mem_is_traced (void);
287 
288 typedef struct
289 {
290  /* Total number of objects allocated. */
292 
293  /* Total allocated bytes. Bytes used and free.
294  used + free = total */
295  uword bytes_total, bytes_used, bytes_free;
296 
297  /* Number of bytes used by mheap data structure overhead
298  (e.g. free lists, mheap header). */
300 
301  /* Amount of free space returned to operating system. */
303 
304  /* For malloc which puts small objects in sbrk region and
305  large objects in mmap'ed regions. */
308 
309  /* Max. number of bytes in this heap. */
312 
314 
315 u8 *format_clib_mem_usage (u8 * s, va_list * args);
316 
317 /* Allocate virtual address space. */
318 always_inline void *
320 {
321  void *mmap_addr;
322  uword flags = MAP_PRIVATE;
323 
324 #ifdef MAP_ANONYMOUS
325  flags |= MAP_ANONYMOUS;
326 #endif
327 
328  mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0);
329  if (mmap_addr == (void *) -1)
330  mmap_addr = 0;
331  else
332  CLIB_MEM_UNPOISON (mmap_addr, size);
333 
334  return mmap_addr;
335 }
336 
337 always_inline void
339 {
340  munmap (addr, size);
341 }
342 
343 always_inline void *
345 {
346  void *mmap_addr;
347  uword flags = MAP_PRIVATE | MAP_FIXED;
348 
349  /* To unmap we "map" with no protection. If we actually called
350  munmap then other callers could steal the address space. By
351  changing to PROT_NONE the kernel can free up the pages which is
352  really what we want "unmap" to mean. */
353  mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0);
354  if (mmap_addr == (void *) -1)
355  mmap_addr = 0;
356  else
357  CLIB_MEM_UNPOISON (mmap_addr, size);
358 
359  return mmap_addr;
360 }
361 
362 always_inline void *
364 {
365  void *mmap_addr;
366  uword flags = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS;
367 
368  mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0);
369  if (mmap_addr == (void *) -1)
370  mmap_addr = 0;
371  else
372  CLIB_MEM_UNPOISON (mmap_addr, size);
373 
374  return mmap_addr;
375 }
376 
377 typedef struct
378 {
379 #define CLIB_MEM_VM_F_SHARED (1 << 0)
380 #define CLIB_MEM_VM_F_HUGETLB (1 << 1)
381 #define CLIB_MEM_VM_F_NUMA_PREFER (1 << 2)
382 #define CLIB_MEM_VM_F_NUMA_FORCE (1 << 3)
383 #define CLIB_MEM_VM_F_HUGETLB_PREALLOC (1 << 4)
384 #define CLIB_MEM_VM_F_LOCKED (1 << 5)
385  u32 flags; /**< vm allocation flags:
386  <br> CLIB_MEM_VM_F_SHARED: request shared memory, file
387  descriptor will be provided on successful allocation.
388  <br> CLIB_MEM_VM_F_HUGETLB: request hugepages.
389  <br> CLIB_MEM_VM_F_NUMA_PREFER: numa_node field contains valid
390  numa node preference.
391  <br> CLIB_MEM_VM_F_NUMA_FORCE: fail if setting numa policy fails.
392  <br> CLIB_MEM_VM_F_HUGETLB_PREALLOC: pre-allocate hugepages if
393  number of available pages is not sufficient.
394  <br> CLIB_MEM_VM_F_LOCKED: request locked memory.
395  */
396  char *name; /**< Name for memory allocation, set by caller. */
397  uword size; /**< Allocation size, set by caller. */
398  int numa_node; /**< numa node preference. Valid if CLIB_MEM_VM_F_NUMA_PREFER set. */
399  void *addr; /**< Pointer to allocated memory, set on successful allocation. */
400  int fd; /**< File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set. */
401  int log2_page_size; /* Page size in log2 format, set on successful allocation. */
402  int n_pages; /* Number of pages. */
403  uword requested_va; /**< Request fixed position mapping */
405 
406 clib_error_t *clib_mem_create_fd (char *name, int *fdp);
407 clib_error_t *clib_mem_create_hugetlb_fd (char *name, int *fdp);
412 int clib_mem_get_fd_log2_page_size (int fd);
413 uword clib_mem_vm_reserve (uword start, uword size, u32 log2_page_sz);
414 u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages);
415 void clib_mem_destroy_mspace (void *mspace);
416 void clib_mem_destroy (void);
417 
418 typedef struct
419 {
420  uword size; /**< Map size */
421  int fd; /**< File descriptor to be mapped */
422  uword requested_va; /**< Request fixed position mapping */
423  void *addr; /**< Pointer to mapped memory, if successful */
426 
428 void clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size);
429 void mheap_trace (void *v, int enable);
431 void clib_mem_trace (int enable);
432 
433 #include <vppinfra/error.h> /* clib_panic */
434 
435 #endif /* _included_clib_mem_h */
436 
437 /*
438  * fd.io coding-style-patch-verification: ON
439  *
440  * Local Variables:
441  * eval: (c-set-style "gnu")
442  * End:
443  */
uword bytes_overhead
Definition: mem.h:299
void clib_mem_validate(void)
Definition: mem_dlmalloc.c:455
#define CLIB_MEM_UNPOISON(a, s)
Definition: sanitizer.h:47
vhost_user_memory_t memory
Definition: vhost_user.h:112
void * clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]
Definition: mem_dlmalloc.c:24
void * mspace
Definition: dlmalloc.h:1306
static void * clib_mem_alloc_aligned_at_offset(uword size, uword align, uword align_offset, int os_out_of_memory_on_failure)
Definition: mem.h:125
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:103
a
Definition: bitmap.h:538
int clib_mem_get_fd_log2_page_size(int fd)
Definition: mem.c:100
uword bytes_free_reclaimed
Definition: mem.h:302
Optimized string handling code, including c11-compliant "safe C library" variants.
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:193
unsigned long u64
Definition: types.h:89
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
void * addr
Pointer to allocated memory, set on successful allocation.
Definition: mem.h:399
uword requested_va
Request fixed position mapping.
Definition: mem.h:403
static void * clib_mem_realloc(void *p, uword new_size, uword old_size)
Definition: mem.h:228
void os_out_of_memory(void)
Definition: unix-misc.c:220
int numa_node
numa node preference.
Definition: mem.h:398
static void usage(void)
Definition: health_check.c:14
uword bytes_used_sbrk
Definition: mem.h:306
uword clib_mem_vm_reserve(uword start, uword size, u32 log2_page_sz)
Definition: mem.c:348
clib_error_t * clib_mem_vm_ext_map(clib_mem_vm_map_t *a)
Definition: mem.c:419
vhost_vring_addr_t addr
Definition: vhost_user.h:111
#define CLIB_MAX_MHEAPS
Definition: mem.h:54
uword bytes_used
Definition: mem.h:295
unsigned char u8
Definition: types.h:56
uword object_count
Definition: mem.h:291
DLMALLOC_EXPORT int mspace_is_heap_object(mspace msp, void *p)
static_always_inline uword os_get_numa_index(void)
Definition: os.h:75
static void clib_mem_set_thread_index(void)
Definition: mem.h:97
uword requested_va
Request fixed position mapping.
Definition: mem.h:422
static errno_t memset_s_inline(void *s, rsize_t smax, int c, rsize_t n)
Definition: string.h:185
char * name
Name for memory allocation, set by caller.
Definition: mem.h:396
void mheap_trace(void *v, int enable)
Definition: mem_dlmalloc.c:461
uword size
Allocation size, set by caller.
Definition: mem.h:397
void * clib_per_numa_mheaps[CLIB_MAX_NUMAS]
Definition: mem_dlmalloc.c:25
unsigned int u32
Definition: types.h:88
uword clib_mem_trace_enable_disable(uword enable)
Definition: mem_dlmalloc.c:491
int fd
File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
Definition: mem.h:400
u64 memory_size
Definition: vhost_user.h:105
u32 size
Definition: vhost_user.h:106
static void * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:65
u8 * format_clib_mem_usage(u8 *s, va_list *args)
Definition: mem_dlmalloc.c:282
uword bytes_used_mmap
Definition: mem.h:307
#define PREDICT_FALSE(x)
Definition: clib.h:120
#define always_inline
Definition: ipsec.h:28
u64 * clib_mem_vm_get_paddr(void *mem, int log2_page_size, int n_pages)
Definition: mem.c:380
void * clib_mem_init(void *heap, uword size)
Definition: mem_dlmalloc.c:227
static void * clib_mem_alloc_or_null(uword size)
Definition: mem.h:173
uword bytes_max
Definition: mem.h:310
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:268
static uword clib_mem_size(void *p)
Definition: mem.h:246
#define ARRAY_LEN(x)
Definition: clib.h:67
string name[64]
Definition: ip.api:44
clib_error_t * clib_mem_create_fd(char *name, int *fdp)
Definition: mem.c:126
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file descriptor will be provided ...
Definition: mem.h:385
static void * clib_mem_get_heap(void)
Definition: mem.h:262
int clib_mem_is_traced(void)
Definition: mem_dlmalloc.c:485
static void * clib_mem_alloc_aligned_or_null(uword size, uword align)
Definition: mem.h:181
void clib_mem_destroy_mspace(void *mspace)
Definition: mem_dlmalloc.c:241
void clib_mem_destroy(void)
Definition: mem_dlmalloc.c:252
#define align_offset(A)
Definition: dlmalloc.c:200
#define ASSERT(truth)
uword size
Map size.
Definition: mem.h:420
#define CLIB_MAX_NUMAS
Definition: mem.h:55
static void clib_mem_free(void *p)
Definition: mem.h:215
static uword clib_mem_is_heap_object(void *p)
Definition: mem.h:207
static void * clib_mem_get_per_numa_heap(u32 numa_id)
Definition: mem.h:81
clib_error_t * clib_mem_create_hugetlb_fd(char *name, int *fdp)
Definition: mem.c:146
static void clib_mem_free_s(void *p)
Definition: mem.h:253
static void * clib_mem_alloc(uword size)
Definition: mem.h:157
static void * clib_mem_set_per_cpu_heap(u8 *new_heap)
Definition: mem.h:72
#define clib_atomic_bool_cmp_and_swap(addr, old, new)
Definition: atomics.h:38
static void clib_mem_vm_free(void *addr, uword size)
Definition: mem.h:338
uword clib_mem_get_default_hugepage_size(void)
Definition: mem.c:57
void clib_mem_vm_ext_free(clib_mem_vm_alloc_t *a)
Definition: mem.c:337
void clib_mem_usage(clib_mem_usage_t *usage)
Definition: mem_dlmalloc.c:431
u64 uword
Definition: types.h:112
static uword clib_mem_size_nocheck(void *p)
Definition: mem.h:118
u64 clib_mem_get_fd_page_size(int fd)
Definition: mem.c:91
DLMALLOC_EXPORT void * mspace_get_aligned(mspace msp, unsigned long n_user_data_bytes, unsigned long align, unsigned long align_offset)
static_always_inline uword os_get_thread_index(void)
Definition: os.h:63
void * clib_mem_init_thread_safe_numa(void *memory, uword memory_size, u8 numa)
Definition: mem_dlmalloc.c:258
void * mem
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:165
int fd
File descriptor to be mapped.
Definition: mem.h:421
static void * clib_mem_vm_unmap(void *addr, uword size)
Definition: mem.h:344
DLMALLOC_EXPORT void mspace_put(mspace msp, void *p)
void clib_mem_vm_randomize_va(uword *requested_va, u32 log2_page_size)
Definition: mem.c:106
int log2_page_size
Definition: mem.h:401
DLMALLOC_EXPORT size_t mspace_usable_size_with_delta(const void *p)
void clib_mem_trace(int enable)
Definition: mem_dlmalloc.c:470
void * clib_mem_init_thread_safe(void *memory, uword memory_size)
Definition: mem_dlmalloc.c:234
uword clib_mem_get_page_size(void)
Definition: mem.c:51
static void * clib_mem_vm_alloc(uword size)
Definition: mem.h:319
static_always_inline void os_set_thread_index(uword thread_index)
Definition: os.h:69
void clib_mem_exit(void)
void * addr
Pointer to mapped memory, if successful.
Definition: mem.h:423
static void * clib_mem_set_per_numa_heap(u8 *new_heap)
Definition: mem.h:88
#define CLIB_MEM_POISON(a, s)
Definition: sanitizer.h:46
static void * clib_mem_vm_map(void *addr, uword size)
Definition: mem.h:363