FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
mem.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus
17 
18  Permission is hereby granted, free of charge, to any person obtaining
19  a copy of this software and associated documentation files (the
20  "Software"), to deal in the Software without restriction, including
21  without limitation the rights to use, copy, modify, merge, publish,
22  distribute, sublicense, and/or sell copies of the Software, and to
23  permit persons to whom the Software is furnished to do so, subject to
24  the following conditions:
25 
26  The above copyright notice and this permission notice shall be
27  included in all copies or substantial portions of the Software.
28 
29  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
36 */
37 
38 #ifndef _included_clib_mem_h
39 #define _included_clib_mem_h
40 
41 #include <stdarg.h>
42 #include <unistd.h>
43 #include <sys/mman.h>
44 
45 #include <vppinfra/clib.h> /* uword, etc */
46 #include <vppinfra/clib_error.h>
47 
48 #if USE_DLMALLOC == 0
50 #else
51 #include <vppinfra/dlmalloc.h>
52 #endif
53 
54 #include <vppinfra/os.h>
55 #include <vppinfra/string.h> /* memcpy, clib_memset */
56 #include <vppinfra/valgrind.h>
57 
58 #define CLIB_MAX_MHEAPS 256
59 
60 /* Per CPU heaps. */
62 
63 always_inline void *
65 {
66  int cpu = os_get_thread_index ();
67  return clib_per_cpu_mheaps[cpu];
68 }
69 
70 always_inline void *
72 {
73  int cpu = os_get_thread_index ();
74  void *old = clib_per_cpu_mheaps[cpu];
75  clib_per_cpu_mheaps[cpu] = new_heap;
76  return old;
77 }
78 
79 /* Memory allocator which may call os_out_of_memory() if it fails */
80 always_inline void *
82  int os_out_of_memory_on_failure)
83 {
84  void *heap, *p;
85  uword cpu;
86 
87  if (align_offset > align)
88  {
89  if (align > 0)
90  align_offset %= align;
91  else
92  align_offset = align;
93  }
94 
95  cpu = os_get_thread_index ();
96  heap = clib_per_cpu_mheaps[cpu];
97 
98 #if USE_DLMALLOC == 0
99  uword offset;
100  heap = mheap_get_aligned (heap, size, align, align_offset, &offset);
101  clib_per_cpu_mheaps[cpu] = heap;
102 
103  if (offset != ~0)
104  {
105  p = heap + offset;
106 #if CLIB_DEBUG > 0
107  VALGRIND_MALLOCLIKE_BLOCK (p, mheap_data_bytes (heap, offset), 0, 0);
108 #endif
109  return p;
110  }
111  else
112  {
113  if (os_out_of_memory_on_failure)
114  os_out_of_memory ();
115  return 0;
116  }
117 #else
118  p = mspace_get_aligned (heap, size, align, align_offset);
119  if (PREDICT_FALSE (p == 0))
120  {
121  if (os_out_of_memory_on_failure)
122  os_out_of_memory ();
123  return 0;
124  }
125 
126  return p;
127 #endif /* USE_DLMALLOC */
128 }
129 
130 /* Memory allocator which calls os_out_of_memory() when it fails */
131 always_inline void *
133 {
134  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
135  /* align_offset */ 0,
136  /* os_out_of_memory */ 1);
137 }
138 
139 always_inline void *
141 {
142  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
143  /* os_out_of_memory */ 1);
144 }
145 
146 /* Memory allocator which calls os_out_of_memory() when it fails */
147 always_inline void *
149 {
150  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
151  /* align_offset */ 0,
152  /* os_out_of_memory */ 0);
153 }
154 
155 always_inline void *
157 {
158  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
159  /* os_out_of_memory */ 0);
160 }
161 
162 
163 
164 /* Memory allocator which panics when it fails.
165  Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
166 #define clib_mem_alloc_aligned_no_fail(size,align) \
167 ({ \
168  uword _clib_mem_alloc_size = (size); \
169  void * _clib_mem_alloc_p; \
170  _clib_mem_alloc_p = clib_mem_alloc_aligned (_clib_mem_alloc_size, (align)); \
171  if (! _clib_mem_alloc_p) \
172  clib_panic ("failed to allocate %d bytes", _clib_mem_alloc_size); \
173  _clib_mem_alloc_p; \
174 })
175 
176 #define clib_mem_alloc_no_fail(size) clib_mem_alloc_aligned_no_fail(size,1)
177 
178 /* Alias to stack allocator for naming consistency. */
179 #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
180 
183 {
184 #if USE_DLMALLOC == 0
185  void *heap = clib_mem_get_per_cpu_heap ();
186  uword offset = (uword) p - (uword) heap;
187  mheap_elt_t *e, *n;
188 
189  if (offset >= vec_len (heap))
190  return 0;
191 
192  e = mheap_elt_at_uoffset (heap, offset);
193  n = mheap_next_elt (e);
194 
195  /* Check that heap forward and reverse pointers agree. */
196  return e->n_user_data == n->prev_n_user_data;
197 #else
198  void *heap = clib_mem_get_per_cpu_heap ();
199 
200  return mspace_is_heap_object (heap, p);
201 #endif /* USE_DLMALLOC */
202 }
203 
204 always_inline void
205 clib_mem_free (void *p)
206 {
207  u8 *heap = clib_mem_get_per_cpu_heap ();
208 
209  /* Make sure object is in the correct heap. */
211 
212 #if USE_DLMALLOC == 0
213  mheap_put (heap, (u8 *) p - heap);
214 #else
215  mspace_put (heap, p);
216 #endif
217 
218 #if CLIB_DEBUG > 0
220 #endif
221 }
222 
223 always_inline void *
224 clib_mem_realloc (void *p, uword new_size, uword old_size)
225 {
226  /* By default use alloc, copy and free to emulate realloc. */
227  void *q = clib_mem_alloc (new_size);
228  if (q)
229  {
230  uword copy_size;
231  if (old_size < new_size)
232  copy_size = old_size;
233  else
234  copy_size = new_size;
235  clib_memcpy_fast (q, p, copy_size);
236  clib_mem_free (p);
237  }
238  return q;
239 }
240 
242 clib_mem_size (void *p)
243 {
244 #if USE_DLMALLOC == 0
247  return mheap_elt_data_bytes (e);
248 #else
251 #endif
252 }
253 
254 always_inline void *
256 {
257  return clib_mem_get_per_cpu_heap ();
258 }
259 
260 always_inline void *
261 clib_mem_set_heap (void *heap)
262 {
263  return clib_mem_set_per_cpu_heap (heap);
264 }
265 
266 void *clib_mem_init (void *heap, uword size);
268 
269 void clib_mem_exit (void);
270 
272 
273 void clib_mem_validate (void);
274 
275 void clib_mem_trace (int enable);
276 
277 int clib_mem_is_traced (void);
278 
279 typedef struct
280 {
281  /* Total number of objects allocated. */
283 
284  /* Total allocated bytes. Bytes used and free.
285  used + free = total */
286  uword bytes_total, bytes_used, bytes_free;
287 
288  /* Number of bytes used by mheap data structure overhead
289  (e.g. free lists, mheap header). */
291 
292  /* Amount of free space returned to operating system. */
294 
295  /* For malloc which puts small objects in sbrk region and
296  large objects in mmap'ed regions. */
299 
300  /* Max. number of bytes in this heap. */
303 
305 
306 u8 *format_clib_mem_usage (u8 * s, va_list * args);
307 
308 /* Allocate virtual address space. */
309 always_inline void *
311 {
312  void *mmap_addr;
313  uword flags = MAP_PRIVATE;
314 
315 #ifdef MAP_ANONYMOUS
316  flags |= MAP_ANONYMOUS;
317 #endif
318 
319  mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0);
320  if (mmap_addr == (void *) -1)
321  mmap_addr = 0;
322 
323  return mmap_addr;
324 }
325 
326 always_inline void
328 {
329  munmap (addr, size);
330 }
331 
332 always_inline void *
334 {
335  void *mmap_addr;
336  uword flags = MAP_PRIVATE | MAP_FIXED;
337 
338  /* To unmap we "map" with no protection. If we actually called
339  munmap then other callers could steal the address space. By
340  changing to PROT_NONE the kernel can free up the pages which is
341  really what we want "unmap" to mean. */
342  mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0);
343  if (mmap_addr == (void *) -1)
344  mmap_addr = 0;
345 
346  return mmap_addr;
347 }
348 
349 always_inline void *
351 {
352  void *mmap_addr;
353  uword flags = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS;
354 
355  mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0);
356  if (mmap_addr == (void *) -1)
357  mmap_addr = 0;
358 
359  return mmap_addr;
360 }
361 
362 typedef struct
363 {
364 #define CLIB_MEM_VM_F_SHARED (1 << 0)
365 #define CLIB_MEM_VM_F_HUGETLB (1 << 1)
366 #define CLIB_MEM_VM_F_NUMA_PREFER (1 << 2)
367 #define CLIB_MEM_VM_F_NUMA_FORCE (1 << 3)
368 #define CLIB_MEM_VM_F_HUGETLB_PREALLOC (1 << 4)
369 #define CLIB_MEM_VM_F_LOCKED (1 << 5)
370  u32 flags; /**< vm allocation flags:
371  <br> CLIB_MEM_VM_F_SHARED: request shared memory, file
372  descriptor will be provided on successful allocation.
373  <br> CLIB_MEM_VM_F_HUGETLB: request hugepages.
374  <br> CLIB_MEM_VM_F_NUMA_PREFER: numa_node field contains valid
375  numa node preference.
376  <br> CLIB_MEM_VM_F_NUMA_FORCE: fail if setting numa policy fails.
377  <br> CLIB_MEM_VM_F_HUGETLB_PREALLOC: pre-allocate hugepages if
378  number of available pages is not sufficient.
379  <br> CLIB_MEM_VM_F_LOCKED: request locked memory.
380  */
381  char *name; /**< Name for memory allocation, set by caller. */
382  uword size; /**< Allocation size, set by caller. */
383  int numa_node; /**< numa node preference. Valid if CLIB_MEM_VM_F_NUMA_PREFER set. */
384  void *addr; /**< Pointer to allocated memory, set on successful allocation. */
385  int fd; /**< File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set. */
386  int log2_page_size; /* Page size in log2 format, set on successful allocation. */
387  int n_pages; /* Number of pages. */
388  uword requested_va; /**< Request fixed position mapping */
390 
391 clib_error_t *clib_mem_create_fd (char *name, int *fdp);
392 clib_error_t *clib_mem_create_hugetlb_fd (char *name, int *fdp);
397 int clib_mem_get_fd_log2_page_size (int fd);
398 u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages);
399 
400 typedef struct
401 {
402  uword size; /**< Map size */
403  int fd; /**< File descriptor to be mapped */
404  uword requested_va; /**< Request fixed position mapping */
405  void *addr; /**< Pointer to mapped memory, if successful */
407 
409 void clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size);
410 void mheap_trace (void *v, int enable);
411 
412 #include <vppinfra/error.h> /* clib_panic */
413 
414 #endif /* _included_clib_mem_h */
415 
416 /*
417  * fd.io coding-style-patch-verification: ON
418  *
419  * Local Variables:
420  * eval: (c-set-style "gnu")
421  * End:
422  */
uword bytes_overhead
Definition: mem.h:290
void clib_mem_validate(void)
Definition: mem_dlmalloc.c:390
u32 flags
Definition: vhost_user.h:115
vhost_user_memory_t memory
Definition: vhost_user.h:122
void * clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]
Definition: mem_dlmalloc.c:23
static void * clib_mem_alloc_aligned_at_offset(uword size, uword align, uword align_offset, int os_out_of_memory_on_failure)
Definition: mem.h:81
a
Definition: bitmap.h:538
int clib_mem_get_fd_log2_page_size(int fd)
Definition: mem.c:100
uword bytes_free_reclaimed
Definition: mem.h:293
Optimized string handling code, including c11-compliant "safe C library" variants.
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:193
unsigned long u64
Definition: types.h:89
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
void * addr
Pointer to allocated memory, set on successful allocation.
Definition: mem.h:384
uword requested_va
Request fixed position mapping.
Definition: mem.h:388
static void * clib_mem_realloc(void *p, uword new_size, uword old_size)
Definition: mem.h:224
void os_out_of_memory(void)
Definition: unix-misc.c:219
int numa_node
numa node preference.
Definition: mem.h:383
static void usage(void)
Definition: health_check.c:14
uword bytes_used_sbrk
Definition: mem.h:297
clib_error_t * clib_mem_vm_ext_map(clib_mem_vm_map_t *a)
Definition: mem.c:386
vhost_vring_addr_t addr
Definition: vhost_user.h:121
#define CLIB_MAX_MHEAPS
Definition: mem.h:58
uword bytes_used
Definition: mem.h:286
unsigned char u8
Definition: types.h:56
uword object_count
Definition: mem.h:282
DLMALLOC_EXPORT int mspace_is_heap_object(mspace msp, void *p)
uword requested_va
Request fixed position mapping.
Definition: mem.h:404
#define always_inline
Definition: clib.h:98
char * name
Name for memory allocation, set by caller.
Definition: mem.h:381
static uword mheap_elt_data_bytes(mheap_elt_t *e)
void mheap_trace(void *v, int enable)
Definition: mem_dlmalloc.c:396
uword size
Allocation size, set by caller.
Definition: mem.h:382
unsigned int u32
Definition: types.h:88
uword size
int fd
File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
Definition: mem.h:385
u64 memory_size
Definition: vhost_user.h:115
static void * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:64
u8 * format_clib_mem_usage(u8 *s, va_list *args)
Definition: mem_dlmalloc.c:232
uword bytes_used_mmap
Definition: mem.h:298
#define PREDICT_FALSE(x)
Definition: clib.h:111
u8 name[64]
Definition: memclnt.api:152
u64 * clib_mem_vm_get_paddr(void *mem, int log2_page_size, int n_pages)
Definition: mem.c:347
void * clib_mem_init(void *heap, uword size)
Definition: mem_dlmalloc.c:205
#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
Definition: valgrind.h:3894
static void * clib_mem_alloc_or_null(uword size)
Definition: mem.h:148
void * mheap_get_aligned(void *v, uword n_user_data_bytes, uword align, uword align_offset, uword *offset_return)
Definition: mheap.c:643
uword bytes_max
Definition: mem.h:301
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:261
static uword clib_mem_size(void *p)
Definition: mem.h:242
clib_error_t * clib_mem_create_fd(char *name, int *fdp)
Definition: mem.c:126
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file descriptor will be provided ...
Definition: mem.h:370
static void * clib_mem_get_heap(void)
Definition: mem.h:255
int clib_mem_is_traced(void)
Definition: mem_dlmalloc.c:420
static void * clib_mem_alloc_aligned_or_null(uword size, uword align)
Definition: mem.h:156
#define ASSERT(truth)
#define align_offset(A)
Definition: dlmalloc.c:199
uword size
Map size.
Definition: mem.h:402
static void clib_mem_free(void *p)
Definition: mem.h:205
static uword clib_mem_is_heap_object(void *p)
Definition: mem.h:182
clib_error_t * clib_mem_create_hugetlb_fd(char *name, int *fdp)
Definition: mem.c:146
static void * clib_mem_alloc(uword size)
Definition: mem.h:132
static void * clib_mem_set_per_cpu_heap(u8 *new_heap)
Definition: mem.h:71
static mheap_elt_t * mheap_elt_at_uoffset(void *v, uword uo)
template key/value backing page structure
Definition: bihash_doc.h:44
static void clib_mem_vm_free(void *addr, uword size)
Definition: mem.h:327
uword clib_mem_get_default_hugepage_size(void)
Definition: mem.c:57
void mheap_put(void *v, uword uoffset)
Definition: mheap.c:771
void clib_mem_vm_ext_free(clib_mem_vm_alloc_t *a)
Definition: mem.c:336
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void clib_mem_usage(clib_mem_usage_t *usage)
Definition: mem_dlmalloc.c:381
u64 uword
Definition: types.h:112
u64 clib_mem_get_fd_page_size(int fd)
Definition: mem.c:91
DLMALLOC_EXPORT void * mspace_get_aligned(mspace msp, unsigned long n_user_data_bytes, unsigned long align, unsigned long align_offset)
static_always_inline uword os_get_thread_index(void)
Definition: os.h:62
static mheap_elt_t * mheap_next_elt(mheap_elt_t *e)
struct clib_bihash_value offset
template key/value backing page structure
void * mem
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:140
int fd
File descriptor to be mapped.
Definition: mem.h:403
static void * clib_mem_vm_unmap(void *addr, uword size)
Definition: mem.h:333
DLMALLOC_EXPORT void mspace_put(mspace msp, void *p)
void clib_mem_vm_randomize_va(uword *requested_va, u32 log2_page_size)
Definition: mem.c:106
#define VALGRIND_FREELIKE_BLOCK(addr, rzB)
Definition: valgrind.h:3905
static uword mheap_data_bytes(void *v, uword uo)
int log2_page_size
Definition: mem.h:386
DLMALLOC_EXPORT size_t mspace_usable_size_with_delta(const void *p)
void clib_mem_trace(int enable)
Definition: mem_dlmalloc.c:405
void * clib_mem_init_thread_safe(void *memory, uword memory_size)
Definition: mem_dlmalloc.c:226
uword clib_mem_get_page_size(void)
Definition: mem.c:51
static void * clib_mem_vm_alloc(uword size)
Definition: mem.h:310
void clib_mem_exit(void)
Definition: mem_mheap.c:49
static mheap_elt_t * mheap_user_pointer_to_elt(void *v)
void * addr
Pointer to mapped memory, if successful.
Definition: mem.h:405
static void * clib_mem_vm_map(void *addr, uword size)
Definition: mem.h:350