FD.io VPP  v17.01-9-ge7dcee4
Vector Packet Processing
mem.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus
17 
18  Permission is hereby granted, free of charge, to any person obtaining
19  a copy of this software and associated documentation files (the
20  "Software"), to deal in the Software without restriction, including
21  without limitation the rights to use, copy, modify, merge, publish,
22  distribute, sublicense, and/or sell copies of the Software, and to
23  permit persons to whom the Software is furnished to do so, subject to
24  the following conditions:
25 
26  The above copyright notice and this permission notice shall be
27  included in all copies or substantial portions of the Software.
28 
29  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
33  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
34  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
35  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
36 */
37 
38 #ifndef _included_clib_mem_h
39 #define _included_clib_mem_h
40 
41 #include <stdarg.h>
42 
43 #include <vppinfra/clib.h> /* uword, etc */
45 #include <vppinfra/os.h>
46 #include <vppinfra/string.h> /* memcpy, memset */
47 #include <vppinfra/valgrind.h>
48 
49 #define CLIB_MAX_MHEAPS 256
50 
51 /* Per CPU heaps. */
53 
54 always_inline void *
56 {
57  int cpu = os_get_cpu_number ();
58  return clib_per_cpu_mheaps[cpu];
59 }
60 
61 always_inline void *
63 {
64  int cpu = os_get_cpu_number ();
65  void *old = clib_per_cpu_mheaps[cpu];
66  clib_per_cpu_mheaps[cpu] = new_heap;
67  return old;
68 }
69 
70 /* Memory allocator which may call os_out_of_memory() if it fails */
71 always_inline void *
73  int os_out_of_memory_on_failure)
74 {
75  void *heap, *p;
76  uword offset, cpu;
77 
78  if (align_offset > align)
79  {
80  if (align > 0)
81  align_offset %= align;
82  else
83  align_offset = align;
84  }
85 
86  cpu = os_get_cpu_number ();
87  heap = clib_per_cpu_mheaps[cpu];
88  heap = mheap_get_aligned (heap, size, align, align_offset, &offset);
89  clib_per_cpu_mheaps[cpu] = heap;
90 
91  if (offset != ~0)
92  {
93  p = heap + offset;
94 #if CLIB_DEBUG > 0
95  VALGRIND_MALLOCLIKE_BLOCK (p, mheap_data_bytes (heap, offset), 0, 0);
96 #endif
97  return p;
98  }
99  else
100  {
101  if (os_out_of_memory_on_failure)
102  os_out_of_memory ();
103  return 0;
104  }
105 }
106 
107 /* Memory allocator which calls os_out_of_memory() when it fails */
108 always_inline void *
110 {
111  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
112  /* align_offset */ 0,
113  /* os_out_of_memory */ 1);
114 }
115 
116 always_inline void *
118 {
119  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
120  /* os_out_of_memory */ 1);
121 }
122 
123 /* Memory allocator which calls os_out_of_memory() when it fails */
124 always_inline void *
126 {
127  return clib_mem_alloc_aligned_at_offset (size, /* align */ 1,
128  /* align_offset */ 0,
129  /* os_out_of_memory */ 0);
130 }
131 
132 always_inline void *
134 {
135  return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0,
136  /* os_out_of_memory */ 0);
137 }
138 
139 
140 
141 /* Memory allocator which panics when it fails.
142  Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */
143 #define clib_mem_alloc_aligned_no_fail(size,align) \
144 ({ \
145  uword _clib_mem_alloc_size = (size); \
146  void * _clib_mem_alloc_p; \
147  _clib_mem_alloc_p = clib_mem_alloc_aligned (_clib_mem_alloc_size, (align)); \
148  if (! _clib_mem_alloc_p) \
149  clib_panic ("failed to allocate %d bytes", _clib_mem_alloc_size); \
150  _clib_mem_alloc_p; \
151 })
152 
153 #define clib_mem_alloc_no_fail(size) clib_mem_alloc_aligned_no_fail(size,1)
154 
155 /* Alias to stack allocator for naming consistency. */
156 #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes)
157 
160 {
161  void *heap = clib_mem_get_per_cpu_heap ();
162  uword offset = (uword) p - (uword) heap;
163  mheap_elt_t *e, *n;
164 
165  if (offset >= vec_len (heap))
166  return 0;
167 
168  e = mheap_elt_at_uoffset (heap, offset);
169  n = mheap_next_elt (e);
170 
171  /* Check that heap forward and reverse pointers agree. */
172  return e->n_user_data == n->prev_n_user_data;
173 }
174 
175 always_inline void
176 clib_mem_free (void *p)
177 {
178  u8 *heap = clib_mem_get_per_cpu_heap ();
179 
180  /* Make sure object is in the correct heap. */
182 
183  mheap_put (heap, (u8 *) p - heap);
184 
185 #if CLIB_DEBUG > 0
187 #endif
188 }
189 
190 always_inline void *
191 clib_mem_realloc (void *p, uword new_size, uword old_size)
192 {
193  /* By default use alloc, copy and free to emulate realloc. */
194  void *q = clib_mem_alloc (new_size);
195  if (q)
196  {
197  uword copy_size;
198  if (old_size < new_size)
199  copy_size = old_size;
200  else
201  copy_size = new_size;
202  clib_memcpy (q, p, copy_size);
203  clib_mem_free (p);
204  }
205  return q;
206 }
207 
209 clib_mem_size (void *p)
210 {
213  return mheap_elt_data_bytes (e);
214 }
215 
216 always_inline void *
218 {
219  return clib_mem_get_per_cpu_heap ();
220 }
221 
222 always_inline void *
223 clib_mem_set_heap (void *heap)
224 {
225  return clib_mem_set_per_cpu_heap (heap);
226 }
227 
228 void *clib_mem_init (void *heap, uword size);
229 
230 void clib_mem_exit (void);
231 
233 
234 void clib_mem_validate (void);
235 
236 void clib_mem_trace (int enable);
237 
238 typedef struct
239 {
240  /* Total number of objects allocated. */
242 
243  /* Total allocated bytes. Bytes used and free.
244  used + free = total */
245  uword bytes_total, bytes_used, bytes_free;
246 
247  /* Number of bytes used by mheap data structure overhead
248  (e.g. free lists, mheap header). */
250 
251  /* Amount of free space returned to operating system. */
253 
254  /* For malloc which puts small objects in sbrk region and
255  large objects in mmap'ed regions. */
258 
259  /* Max. number of bytes in this heap. */
262 
264 
265 u8 *format_clib_mem_usage (u8 * s, va_list * args);
266 
267 /* Include appropriate VM functions depending on whether
268  we are compiling for linux kernel, for Unix or standalone. */
269 #ifdef CLIB_LINUX_KERNEL
271 #endif
272 
273 #ifdef CLIB_UNIX
274 #include <vppinfra/vm_unix.h>
275 #endif
276 
277 #ifdef CLIB_STANDALONE
278 #include <vppinfra/vm_standalone.h>
279 #endif
280 
281 #include <vppinfra/error.h> /* clib_panic */
282 
283 #endif /* _included_clib_mem_h */
284 
285 /*
286  * fd.io coding-style-patch-verification: ON
287  *
288  * Local Variables:
289  * eval: (c-set-style "gnu")
290  * End:
291  */
uword bytes_overhead
Definition: mem.h:249
void clib_mem_validate(void)
Definition: mem_mheap.c:142
void * clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]
Definition: mem_mheap.c:46
static void * clib_mem_alloc_aligned_at_offset(uword size, uword align, uword align_offset, int os_out_of_memory_on_failure)
Definition: mem.h:72
uword bytes_free_reclaimed
Definition: mem.h:252
static void * clib_mem_realloc(void *p, uword new_size, uword old_size)
Definition: mem.h:191
void os_out_of_memory(void)
Definition: unix-misc.c:217
static void usage(void)
Definition: health_check.c:14
uword bytes_used_sbrk
Definition: mem.h:256
#define CLIB_MAX_MHEAPS
Definition: mem.h:49
uword bytes_used
Definition: mem.h:245
uword object_count
Definition: mem.h:241
#define always_inline
Definition: clib.h:84
static uword mheap_elt_data_bytes(mheap_elt_t *e)
uword os_get_cpu_number(void)
Definition: unix-misc.c:224
static void * clib_mem_get_per_cpu_heap(void)
Definition: mem.h:55
u8 * format_clib_mem_usage(u8 *s, va_list *args)
Definition: mem_mheap.c:126
uword bytes_used_mmap
Definition: mem.h:257
void * clib_mem_init(void *heap, uword size)
Definition: mem_mheap.c:60
#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed)
Definition: valgrind.h:3894
static void * clib_mem_alloc_or_null(uword size)
Definition: mem.h:125
void * mheap_get_aligned(void *v, uword n_user_data_bytes, uword align, uword align_offset, uword *offset_return)
Definition: mheap.c:649
uword bytes_max
Definition: mem.h:260
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:223
#define clib_memcpy(a, b, c)
Definition: string.h:69
static uword clib_mem_size(void *p)
Definition: mem.h:209
static void * clib_mem_get_heap(void)
Definition: mem.h:217
static void * clib_mem_alloc_aligned_or_null(uword size, uword align)
Definition: mem.h:133
#define ASSERT(truth)
u64 size
Definition: vhost-user.h:74
static void clib_mem_free(void *p)
Definition: mem.h:176
static uword clib_mem_is_heap_object(void *p)
Definition: mem.h:159
static void * clib_mem_alloc(uword size)
Definition: mem.h:109
u64 uword
Definition: types.h:112
static void * clib_mem_set_per_cpu_heap(u8 *new_heap)
Definition: mem.h:62
static mheap_elt_t * mheap_elt_at_uoffset(void *v, uword uo)
template key/value backing page structure
Definition: bihash_doc.h:44
void mheap_put(void *v, uword uoffset)
Definition: mheap.c:761
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
void clib_mem_usage(clib_mem_usage_t *usage)
Definition: mem_mheap.c:133
static mheap_elt_t * mheap_next_elt(mheap_elt_t *e)
struct clib_bihash_value offset
template key/value backing page structure
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:117
#define VALGRIND_FREELIKE_BLOCK(addr, rzB)
Definition: valgrind.h:3905
static uword mheap_data_bytes(void *v, uword uo)
void clib_mem_trace(int enable)
Definition: mem_mheap.c:154
uword clib_mem_get_page_size(void)
Definition: mem_mheap.c:110
void clib_mem_exit(void)
Definition: mem_mheap.c:49
static mheap_elt_t * mheap_user_pointer_to_elt(void *v)