FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
mem_dlmalloc.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include <vppinfra/format.h>
17 #include <vppinfra/dlmalloc.h>
18 #include <vppinfra/os.h>
19 #include <vppinfra/lock.h>
20 #include <vppinfra/hash.h>
21 #include <vppinfra/elf_clib.h>
22 #include <vppinfra/sanitizer.h>
23 
26 
27 typedef struct
28 {
29  /* Address of callers: outer first, inner last. */
30  uword callers[12];
31 
32  /* Count of allocations with this traceback. */
34 
35  /* Count of bytes allocated with this traceback. */
37 
38  /* Offset of this item */
41 
42 typedef struct
43 {
46 
48 
49  /* Indices of free traces. */
51 
52  /* Hash table mapping callers to trace index. */
54 
55  /* Hash table mapping mheap offset to trace index. */
57 
58  /* So we can easily shut off current segment trace, if any */
60 
62 
64 
65 void
67 {
69  mheap_trace_t *t;
70  uword i, n_callers, trace_index, *p;
72  uword save_enabled;
73 
74  if (tm->enabled == 0 || (clib_mem_get_heap () != tm->current_traced_mheap))
75  return;
76 
77  /* Spurious Coverity warnings be gone. */
78  clib_memset (&trace, 0, sizeof (trace));
79 
80  /* Skip our frame and mspace_get_aligned's frame */
81  n_callers = clib_backtrace (trace.callers, ARRAY_LEN (trace.callers), 2);
82  if (n_callers == 0)
83  return;
84 
85  clib_spinlock_lock (&tm->lock);
86 
87  /* Turn off tracing to avoid embarrassment... */
88  save_enabled = tm->enabled;
89  tm->enabled = 0;
90 
91  if (!tm->trace_by_callers)
92  tm->trace_by_callers =
93  hash_create_shmem (0, sizeof (trace.callers), sizeof (uword));
94 
95  p = hash_get_mem (tm->trace_by_callers, &trace.callers);
96  if (p)
97  {
98  trace_index = p[0];
99  t = tm->traces + trace_index;
100  }
101  else
102  {
103  i = vec_len (tm->trace_free_list);
104  if (i > 0)
105  {
106  trace_index = tm->trace_free_list[i - 1];
107  _vec_len (tm->trace_free_list) = i - 1;
108  }
109  else
110  {
111  mheap_trace_t *old_start = tm->traces;
112  mheap_trace_t *old_end = vec_end (tm->traces);
113 
114  vec_add2 (tm->traces, t, 1);
115 
116  if (tm->traces != old_start)
117  {
118  hash_pair_t *p;
119  mheap_trace_t *q;
120  /* *INDENT-OFF* */
122  ({
123  q = uword_to_pointer (p->key, mheap_trace_t *);
124  ASSERT (q >= old_start && q < old_end);
125  p->key = pointer_to_uword (tm->traces + (q - old_start));
126  }));
127  /* *INDENT-ON* */
128  }
129  trace_index = t - tm->traces;
130  }
131 
132  t = tm->traces + trace_index;
133  t[0] = trace;
134  t->n_allocations = 0;
135  t->n_bytes = 0;
136  hash_set_mem (tm->trace_by_callers, t->callers, trace_index);
137  }
138 
139  t->n_allocations += 1;
140  t->n_bytes += size;
141  t->offset = offset; /* keep a sample to autopsy */
142  hash_set (tm->trace_index_by_offset, offset, t - tm->traces);
143  tm->enabled = save_enabled;
144  clib_spinlock_unlock (&tm->lock);
145 }
146 
147 void
149 {
150  mheap_trace_t *t;
151  uword trace_index, *p;
153  uword save_enabled;
154 
155  if (tm->enabled == 0)
156  return;
157 
158  clib_spinlock_lock (&tm->lock);
159 
160  /* Turn off tracing for a moment */
161  save_enabled = tm->enabled;
162  tm->enabled = 0;
163 
164  p = hash_get (tm->trace_index_by_offset, offset);
165  if (!p)
166  {
167  tm->enabled = save_enabled;
168  clib_spinlock_unlock (&tm->lock);
169  return;
170  }
171 
172  trace_index = p[0];
173  hash_unset (tm->trace_index_by_offset, offset);
174  ASSERT (trace_index < vec_len (tm->traces));
175 
176  t = tm->traces + trace_index;
177  ASSERT (t->n_allocations > 0);
178  ASSERT (t->n_bytes >= size);
179  t->n_allocations -= 1;
180  t->n_bytes -= size;
181  if (t->n_allocations == 0)
182  {
184  vec_add1 (tm->trace_free_list, trace_index);
185  clib_memset (t, 0, sizeof (t[0]));
186  }
187  tm->enabled = save_enabled;
188  clib_spinlock_unlock (&tm->lock);
189 }
190 
191 always_inline void
193 {
194  vec_free (tm->traces);
198 }
199 
200 /* Initialize CLIB heap based on memory/size given by user.
201  Set memory to 0 and CLIB will try to allocate its own heap. */
202 static void *
204 {
205  u8 *heap;
206 
207  if (memory)
208  {
209  heap = create_mspace_with_base (memory, memory_size, 1 /* locked */ );
210  mspace_disable_expand (heap);
211  }
212  else
213  heap = create_mspace (memory_size, 1 /* locked */ );
214 
216 
217  if (set_heap)
218  clib_mem_set_heap (heap);
219 
220  if (mheap_trace_main.lock == 0)
221  clib_spinlock_init (&mheap_trace_main.lock);
222 
223  return heap;
224 }
225 
226 void *
228 {
229  return clib_mem_init_internal (memory, memory_size,
230  1 /* do clib_mem_set_heap */ );
231 }
232 
233 void *
235 {
236  return clib_mem_init_internal (memory, memory_size,
237  1 /* do clib_mem_set_heap */ );
238 }
239 
240 void
242 {
244 
245  if (tm->enabled && mspace == tm->current_traced_mheap)
246  tm->enabled = 0;
247 
248  destroy_mspace (mspace);
249 }
250 
251 void
253 {
255 }
256 
257 void *
259 {
260  clib_mem_vm_alloc_t alloc = { 0 };
261  clib_error_t *err;
262  void *heap;
263 
264  alloc.size = memory_size;
266  alloc.numa_node = numa;
267  if ((err = clib_mem_vm_ext_alloc (&alloc)))
268  {
269  clib_error_report (err);
270  return 0;
271  }
272 
273  heap = clib_mem_init_internal (memory, memory_size,
274  0 /* do NOT clib_mem_set_heap */ );
275 
276  ASSERT (heap);
277 
278  return heap;
279 }
280 
281 u8 *
282 format_clib_mem_usage (u8 * s, va_list * va)
283 {
284  int verbose = va_arg (*va, int);
285  return format (s, "$$$$ heap at %llx verbose %d", clib_mem_get_heap (),
286  verbose);
287 }
288 
289 /*
290  * Magic decoder ring for mallinfo stats (ala dlmalloc):
291  *
292  * size_t arena; / * Non-mmapped space allocated (bytes) * /
293  * size_t ordblks; / * Number of free chunks * /
294  * size_t smblks; / * Number of free fastbin blocks * /
295  * size_t hblks; / * Number of mmapped regions * /
296  * size_t hblkhd; / * Space allocated in mmapped regions (bytes) * /
297  * size_t usmblks; / * Maximum total allocated space (bytes) * /
298  * size_t fsmblks; / * Space in freed fastbin blocks (bytes) * /
299  * size_t uordblks; / * Total allocated space (bytes) * /
300  * size_t fordblks; / * Total free space (bytes) * /
301  * size_t keepcost; / * Top-most, releasable space (bytes) * /
302  *
303  */
304 
305 u8 *
306 format_msize (u8 * s, va_list * va)
307 {
308  uword a = va_arg (*va, uword);
309 
310  if (a >= 1ULL << 30)
311  s = format (s, "%.2fG", (((f64) a) / ((f64) (1ULL << 30))));
312  else if (a >= 1ULL << 20)
313  s = format (s, "%.2fM", (((f64) a) / ((f64) (1ULL << 20))));
314  else if (a >= 1ULL << 10)
315  s = format (s, "%.2fK", (((f64) a) / ((f64) (1ULL << 10))));
316  else
317  s = format (s, "%lld", a);
318  return s;
319 }
320 
321 static int
322 mheap_trace_sort (const void *_t1, const void *_t2)
323 {
324  const mheap_trace_t *t1 = _t1;
325  const mheap_trace_t *t2 = _t2;
326  word cmp;
327 
328  cmp = (word) t2->n_bytes - (word) t1->n_bytes;
329  if (!cmp)
330  cmp = (word) t2->n_allocations - (word) t1->n_allocations;
331  return cmp;
332 }
333 
334 u8 *
335 format_mheap_trace (u8 * s, va_list * va)
336 {
337  mheap_trace_main_t *tm = va_arg (*va, mheap_trace_main_t *);
338  int verbose = va_arg (*va, int);
339  int have_traces = 0;
340  int i;
341 
342  clib_spinlock_lock (&tm->lock);
343  if (vec_len (tm->traces) > 0 &&
345  {
346  have_traces = 1;
347 
348  /* Make a copy of traces since we'll be sorting them. */
349  mheap_trace_t *t, *traces_copy;
350  u32 indent, total_objects_traced;
351 
352  traces_copy = vec_dup (tm->traces);
353 
354  qsort (traces_copy, vec_len (traces_copy), sizeof (traces_copy[0]),
356 
357  total_objects_traced = 0;
358  s = format (s, "\n");
359  vec_foreach (t, traces_copy)
360  {
361  /* Skip over free elements. */
362  if (t->n_allocations == 0)
363  continue;
364 
365  total_objects_traced += t->n_allocations;
366 
367  /* When not verbose only report allocations of more than 1k. */
368  if (!verbose && t->n_bytes < 1024)
369  continue;
370 
371  if (t == traces_copy)
372  s = format (s, "%=9s%=9s %=10s Traceback\n", "Bytes", "Count",
373  "Sample");
374  s = format (s, "%9d%9d %p", t->n_bytes, t->n_allocations, t->offset);
375  indent = format_get_indent (s);
376  for (i = 0; i < ARRAY_LEN (t->callers) && t->callers[i]; i++)
377  {
378  if (i > 0)
379  s = format (s, "%U", format_white_space, indent);
380 #if defined(CLIB_UNIX) && !defined(__APPLE__)
381  /* $$$$ does this actually work? */
382  s =
384  t->callers[i]);
385 #else
386  s = format (s, " %p\n", t->callers[i]);
387 #endif
388  }
389  }
390 
391  s = format (s, "%d total traced objects\n", total_objects_traced);
392 
393  vec_free (traces_copy);
394  }
395  clib_spinlock_unlock (&tm->lock);
396  if (have_traces == 0)
397  s = format (s, "no traced allocations\n");
398 
399  return s;
400 }
401 
402 
403 u8 *
404 format_mheap (u8 * s, va_list * va)
405 {
406  void *heap = va_arg (*va, u8 *);
407  int verbose = va_arg (*va, int);
408  struct dlmallinfo mi;
410 
411  mi = mspace_mallinfo (heap);
412 
413  s = format (s, "total: %U, used: %U, free: %U, trimmable: %U",
414  format_msize, mi.arena,
417  if (verbose > 0)
418  {
419  s = format (s, "\n free chunks %llu free fastbin blks %llu",
420  mi.ordblks, mi.smblks);
421  s =
422  format (s, "\n max total allocated %U", format_msize, mi.usmblks);
423  }
424 
425  if (mspace_is_traced (heap))
426  s = format (s, "\n%U", format_mheap_trace, tm, verbose);
427  return s;
428 }
429 
430 void
432 {
433  clib_warning ("unimp");
434 }
435 
436 void
438 {
439  struct dlmallinfo mi = mspace_mallinfo (heap);
440 
441  /* TODO: Fill in some more values */
442  usage->object_count = 0;
443  usage->bytes_total = mi.arena;
444  usage->bytes_overhead = 0;
445  usage->bytes_max = 0;
446  usage->bytes_used = mi.uordblks;
447  usage->bytes_free = mi.fordblks;
448  usage->bytes_free_reclaimed = 0;
449 }
450 
451 /* Call serial number for debugger breakpoints. */
453 
454 void
456 {
457  clib_warning ("unimp");
458 }
459 
460 void
461 mheap_trace (void *v, int enable)
462 {
463  (void) mspace_enable_disable_trace (v, enable);
464 
465  if (enable == 0)
466  mheap_trace_main_free (&mheap_trace_main);
467 }
468 
469 void
470 clib_mem_trace (int enable)
471 {
473  void *current_heap = clib_mem_get_heap ();
474 
475  tm->enabled = enable;
476  mheap_trace (current_heap, enable);
477 
478  if (enable)
479  tm->current_traced_mheap = current_heap;
480  else
481  tm->current_traced_mheap = 0;
482 }
483 
484 int
486 {
488 }
489 
490 uword
492 {
493  uword rv;
495 
496  rv = tm->enabled;
497  tm->enabled = enable;
498  return rv;
499 }
500 
501 /*
502  * These API functions seem like layering violations, but
503  * by introducing them we greatly reduce the number
504  * of code changes required to use dlmalloc spaces
505  */
506 void *
508 {
509  void *rv;
510  if (memory == 0)
511  return create_mspace (size, locked);
512  else
513  {
514  rv = create_mspace_with_base (memory, size, locked);
515  if (rv)
517  return rv;
518  }
519 }
520 
521 /*
522  * fd.io coding-style-patch-verification: ON
523  *
524  * Local Variables:
525  * eval: (c-set-style "gnu")
526  * End:
527  */
void * clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]
Definition: mem_dlmalloc.c:24
uword bytes_overhead
Definition: mem.h:299
uword bytes_total
Definition: mem.h:295
static vlib_cli_command_t trace
(constructor) VLIB_CLI_COMMAND (trace)
Definition: vlib_api_cli.c:899
#define hash_set(h, key, value)
Definition: hash.h:255
uword bytes_free
Definition: mem.h:295
vhost_user_memory_t memory
Definition: vhost_user.h:112
static_always_inline void clib_spinlock_unlock(clib_spinlock_t *p)
Definition: lock.h:119
static_always_inline void clib_spinlock_lock(clib_spinlock_t *p)
Definition: lock.h:80
void * mspace
Definition: dlmalloc.h:1306
#define hash_unset(h, key)
Definition: hash.h:261
a
Definition: bitmap.h:538
void * clib_per_numa_mheaps[CLIB_MAX_NUMAS]
Definition: mem_dlmalloc.c:25
uword callers[12]
Definition: mem_dlmalloc.c:30
uword bytes_free_reclaimed
Definition: mem.h:302
void * current_traced_mheap
Definition: mem_dlmalloc.c:59
void * clib_mem_init(void *memory, uword memory_size)
Definition: mem_dlmalloc.c:227
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
uword clib_backtrace(uword *callers, uword max_callers, uword n_frames_to_skip)
Definition: backtrace.c:226
u8 * format_msize(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:306
void clib_mem_destroy(void)
Definition: mem_dlmalloc.c:252
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:592
void mheap_trace(void *v, int enable)
Definition: mem_dlmalloc.c:461
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:630
int numa_node
numa node preference.
Definition: mem.h:398
static void usage(void)
Definition: health_check.c:14
static u32 format_get_indent(u8 *s)
Definition: format.h:72
#define hash_set_mem(h, key, value)
Definition: hash.h:275
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
DLMALLOC_EXPORT struct dlmallinfo mspace_mallinfo(mspace msp)
#define CLIB_MAX_MHEAPS
Definition: mem.h:54
uword bytes_used
Definition: mem.h:295
unsigned char u8
Definition: types.h:56
double f64
Definition: types.h:142
uword object_count
Definition: mem.h:291
DLMALLOC_EXPORT mspace create_mspace_with_base(void *base, size_t capacity, int locked)
MALLINFO_FIELD_TYPE uordblks
Definition: dlmalloc.h:807
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:193
i64 word
Definition: types.h:111
MALLINFO_FIELD_TYPE arena
Definition: dlmalloc.h:800
u8 * format_white_space(u8 *s, va_list *va)
Definition: std-formats.c:129
uword size
Allocation size, set by caller.
Definition: mem.h:397
unsigned int u32
Definition: types.h:88
#define vec_end(v)
End (last data address) of vector.
MALLINFO_FIELD_TYPE ordblks
Definition: dlmalloc.h:801
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
u8 * format_mheap(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:404
#define hash_get(h, key)
Definition: hash.h:249
#define hash_unset_mem(h, key)
Definition: hash.h:291
mheap_trace_main_t mheap_trace_main
Definition: mem_dlmalloc.c:63
MALLINFO_FIELD_TYPE usmblks
Definition: dlmalloc.h:805
void clib_mem_usage(clib_mem_usage_t *u)
Definition: mem_dlmalloc.c:431
DLMALLOC_EXPORT void mspace_disable_expand(mspace msp)
u64 memory_size
Definition: vhost_user.h:105
u32 size
Definition: vhost_user.h:106
#define hash_free(h)
Definition: hash.h:310
mheap_trace_t * traces
Definition: mem_dlmalloc.c:47
#define CLIB_MEM_VM_F_NUMA_FORCE
Definition: mem.h:382
#define vec_dup(V)
Return copy of vector (no header, no alignment)
Definition: vec.h:429
#define always_inline
Definition: ipsec.h:28
void clib_mem_validate(void)
Definition: mem_dlmalloc.c:455
static int mheap_trace_sort(const void *_t1, const void *_t2)
Definition: mem_dlmalloc.c:322
DLMALLOC_EXPORT int mspace_is_traced(mspace msp)
DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked)
uword bytes_max
Definition: mem.h:310
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:380
static void * clib_mem_set_heap(void *heap)
Definition: mem.h:268
MALLINFO_FIELD_TYPE keepcost
Definition: dlmalloc.h:809
#define clib_warning(format, args...)
Definition: error.h:59
DLMALLOC_EXPORT int mspace_enable_disable_trace(mspace msp, int enable)
#define ARRAY_LEN(x)
Definition: clib.h:67
uword * trace_by_callers
Definition: mem_dlmalloc.c:53
u8 * format_mheap_trace(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:335
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file descriptor will be provided ...
Definition: mem.h:385
static void * clib_mem_get_heap(void)
Definition: mem.h:262
static void mheap_trace_main_free(mheap_trace_main_t *tm)
Definition: mem_dlmalloc.c:192
int clib_mem_is_traced(void)
Definition: mem_dlmalloc.c:485
MALLINFO_FIELD_TYPE smblks
Definition: dlmalloc.h:802
#define hash_create_shmem(elts, key_bytes, value_bytes)
Definition: hash.h:684
#define ASSERT(truth)
uword clib_mem_trace_enable_disable(uword enable)
Definition: mem_dlmalloc.c:491
#define CLIB_MAX_NUMAS
Definition: mem.h:55
void mheap_get_trace(uword offset, uword size)
Definition: mem_dlmalloc.c:66
void * mheap_alloc_with_lock(void *memory, uword size, int locked)
Definition: mem_dlmalloc.c:507
uword clib_mem_validate_serial
Definition: mem_dlmalloc.c:452
#define clib_error_report(e)
Definition: error.h:113
template key/value backing page structure
Definition: bihash_doc.h:44
void mheap_usage(void *heap, clib_mem_usage_t *usage)
Definition: mem_dlmalloc.c:437
void qsort(void *base, uword n, uword size, int(*compar)(const void *, const void *))
Definition: qsort.c:56
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
#define hash_foreach_pair(p, v, body)
Iterate over hash pairs.
Definition: hash.h:373
u64 uword
Definition: types.h:112
static void * clib_mem_init_internal(void *memory, uword memory_size, int set_heap)
Definition: mem_dlmalloc.c:203
void mheap_put_trace(uword offset, uword size)
Definition: mem_dlmalloc.c:148
format_function_t format_clib_elf_symbol_with_address
Definition: elf_clib.h:134
#define hash_get_mem(h, key)
Definition: hash.h:269
struct clib_bihash_value offset
template key/value backing page structure
DLMALLOC_EXPORT size_t destroy_mspace(mspace msp)
DLMALLOC_EXPORT size_t mspace_footprint(mspace msp)
#define vec_foreach(var, vec)
Vector iterator.
void clib_mem_destroy_mspace(void *mspace)
Definition: mem_dlmalloc.c:241
void clib_mem_trace(int enable)
Definition: mem_dlmalloc.c:470
void * clib_mem_init_thread_safe(void *memory, uword memory_size)
Definition: mem_dlmalloc.c:234
MALLINFO_FIELD_TYPE fordblks
Definition: dlmalloc.h:808
uword * trace_index_by_offset
Definition: mem_dlmalloc.c:56
clib_spinlock_t lock
Definition: mem_dlmalloc.c:44
u8 * format_clib_mem_usage(u8 *s, va_list *va)
Definition: mem_dlmalloc.c:282
DLMALLOC_EXPORT void * mspace_least_addr(mspace msp)
void * clib_mem_init_thread_safe_numa(void *memory, uword memory_size, u8 numa)
Definition: mem_dlmalloc.c:258
#define CLIB_MEM_POISON(a, s)
Definition: sanitizer.h:46