FD.io VPP  v18.11-rc0-18-g2a3fb1a
Vector Packet Processing
physmem.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * physmem.c: Unix physical memory
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <unistd.h>
41 #include <sys/types.h>
42 #include <sys/mount.h>
43 #include <sys/mman.h>
44 #include <sys/fcntl.h>
45 #include <sys/stat.h>
46 #include <unistd.h>
47 
48 #include <vppinfra/linux/syscall.h>
49 #include <vppinfra/linux/sysfs.h>
50 #include <vlib/vlib.h>
51 #include <vlib/physmem.h>
52 #include <vlib/unix/unix.h>
53 #include <vlib/pci/pci.h>
54 #include <vlib/linux/vfio.h>
55 
56 static void *
58  uword n_bytes, uword alignment)
59 {
61  uword lo_offset, hi_offset;
62  uword *to_free = 0;
63 
64  if (pr->heap == 0)
65  return 0;
66 
67  /* IO memory is always at least cache aligned. */
68  alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES);
69 
70  while (1)
71  {
72  mheap_get_aligned (pr->heap, n_bytes,
73  /* align */ alignment,
74  /* align offset */ 0,
75  &lo_offset);
76 
77  /* Allocation failed? */
78  if (lo_offset == ~0)
79  break;
80 
81  /* Make sure allocation does not span DMA physical chunk boundary. */
82  hi_offset = lo_offset + n_bytes - 1;
83 
84  if (((pointer_to_uword (pr->heap) + lo_offset) >> pr->log2_page_size) ==
85  ((pointer_to_uword (pr->heap) + hi_offset) >> pr->log2_page_size))
86  break;
87 
88  /* Allocation would span chunk boundary, queue it to be freed as soon as
89  we find suitable chunk. */
90  vec_add1 (to_free, lo_offset);
91  }
92 
93  if (to_free != 0)
94  {
95  uword i;
96  for (i = 0; i < vec_len (to_free); i++)
97  mheap_put (pr->heap, to_free[i]);
98  vec_free (to_free);
99  }
100 
101  return lo_offset != ~0 ? pr->heap + lo_offset : 0;
102 }
103 
104 static void
106 {
108  /* Return object to region's heap. */
109  mheap_put (pr->heap, x - pr->heap);
110 }
111 
112 static clib_error_t *
114  u8 numa_node, u32 flags,
116 {
119  clib_error_t *error = 0;
120  clib_mem_vm_alloc_t alloc = { 0 };
121  int i;
122 
123  pool_get (vpm->regions, pr);
124 
125  if ((pr - vpm->regions) >= 256)
126  {
127  error = clib_error_return (0, "maximum number of regions reached");
128  goto error;
129  }
130 
131  alloc.name = name;
132  alloc.size = size;
133  alloc.numa_node = numa_node;
134 
135  alloc.flags = (flags & VLIB_PHYSMEM_F_SHARED) ?
137 
138  if ((flags & VLIB_PHYSMEM_F_HUGETLB))
139  {
140  alloc.flags |= CLIB_MEM_VM_F_HUGETLB;
143  }
144  else
145  {
147  }
148 
149  error = clib_mem_vm_ext_alloc (&alloc);
150  if (error)
151  goto error;
152 
153  pr->index = pr - vpm->regions;
154  pr->flags = flags;
155  pr->fd = alloc.fd;
156  pr->mem = alloc.addr;
157  pr->log2_page_size = alloc.log2_page_size;
158  pr->n_pages = alloc.n_pages;
159  pr->size = (u64) pr->n_pages << (u64) pr->log2_page_size;
160  pr->page_mask = (1 << pr->log2_page_size) - 1;
161  pr->numa_node = numa_node;
162  pr->name = format (0, "%s%c", name, 0);
163 
164  for (i = 0; i < pr->n_pages; i++)
165  {
166  void *ptr = pr->mem + ((u64) i << pr->log2_page_size);
167  int node;
168  if ((move_pages (0, 1, &ptr, 0, &node, 0) == 0) && (numa_node != node))
169  {
170  clib_warning ("physmem page for region \'%s\' allocated on the"
171  " wrong numa node (requested %u actual %u)",
172  pr->name, pr->numa_node, node, i);
173  break;
174  }
175  }
176 
178  pr->n_pages);
179 
181 
182  if (flags & VLIB_PHYSMEM_F_INIT_MHEAP)
183  {
184  pr->heap = mheap_alloc_with_flags (pr->mem, pr->size,
185  /* Don't want mheap mmap/munmap with IO memory. */
188  }
189 
190  *idx = pr->index;
191 
192  goto done;
193 
194 error:
195  memset (pr, 0, sizeof (*pr));
196  pool_put (vpm->regions, pr);
197 
198 done:
199  return error;
200 }
201 
202 static void
204 {
207 
208  if (pr->fd > 0)
209  close (pr->fd);
210  munmap (pr->mem, pr->size);
211  vec_free (pr->name);
212  pool_put (vpm->regions, pr);
213 }
214 
215 clib_error_t *
217 {
219  clib_error_t *error = 0;
220  u64 *pt = 0;
221 
222  /* Avoid multiple calls. */
223  if (vm->os_physmem_alloc_aligned)
224  return error;
225 
226  /* check if pagemap is accessible */
227  pt = clib_mem_vm_get_paddr (&pt, min_log2 (sysconf (_SC_PAGESIZE)), 1);
228  if (pt[0])
230  vec_free (pt);
231 
232  if ((error = linux_vfio_init (vm)))
233  return error;
234 
239 
240  return error;
241 }
242 
243 static clib_error_t *
245  unformat_input_t * input, vlib_cli_command_t * cmd)
246 {
249 
250  /* *INDENT-OFF* */
251  pool_foreach (pr, vpm->regions, (
252  {
253  vlib_cli_output (vm, "index %u name '%s' page-size %uKB num-pages %d "
254  "numa-node %u fd %d\n",
255  pr->index, pr->name, (1 << (pr->log2_page_size -10)),
256  pr->n_pages, pr->numa_node, pr->fd);
257  if (pr->heap)
258  vlib_cli_output (vm, " %U", format_mheap, pr->heap, /* verbose */ 1);
259  else
260  vlib_cli_output (vm, " no heap\n");
261  }));
262  /* *INDENT-ON* */
263  return 0;
264 }
265 
266 /* *INDENT-OFF* */
267 VLIB_CLI_COMMAND (show_physmem_command, static) = {
268  .path = "show physmem",
269  .short_help = "Show physical memory allocation",
270  .function = show_physmem,
271 };
272 /* *INDENT-ON* */
273 
274 /*
275  * fd.io coding-style-patch-verification: ON
276  *
277  * Local Variables:
278  * eval: (c-set-style "gnu")
279  * End:
280  */
#define CLIB_MEM_VM_F_HUGETLB
Definition: mem.h:327
#define VLIB_PHYSMEM_F_INIT_MHEAP
Definition: physmem.h:57
void linux_vfio_dma_map_regions(vlib_main_t *vm)
Definition: vfio.c:66
#define VLIB_PHYSMEM_F_HUGETLB
Definition: physmem.h:58
void(* os_physmem_region_free)(struct vlib_main_t *vm, vlib_physmem_region_index_t idx)
Definition: main.h:122
#define CLIB_MEM_VM_F_NUMA_PREFER
Definition: mem.h:328
unsigned long u64
Definition: types.h:89
void * addr
Pointer to allocated memory, set on successful allocation.
Definition: mem.h:346
static clib_error_t * unix_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
Definition: physmem.c:113
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:523
int i
int numa_node
numa node preference.
Definition: mem.h:345
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define MHEAP_FLAG_THREAD_SAFE
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:228
unsigned char u8
Definition: types.h:56
static uword min_log2(uword x)
Definition: clib.h:138
static void * unix_physmem_alloc_aligned(vlib_main_t *vm, vlib_physmem_region_index_t idx, uword n_bytes, uword alignment)
Definition: physmem.c:57
#define VLIB_PHYSMEM_F_SHARED
Definition: physmem.h:59
#define VLIB_PHYSMEM_MAIN_F_HAVE_PAGEMAP
Definition: physmem.h:71
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:80
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:443
void(* os_physmem_free)(struct vlib_main_t *vm, vlib_physmem_region_index_t idx, void *x)
Definition: main.h:128
#define MHEAP_FLAG_DISABLE_VM
char * name
Name for memory allocation, set by caller.
Definition: mem.h:343
void *(* os_physmem_alloc_aligned)(struct vlib_main_t *vm, vlib_physmem_region_index_t idx, uword n_bytes, uword alignment)
Definition: main.h:125
#define clib_error_return(e, args...)
Definition: error.h:99
uword size
Allocation size, set by caller.
Definition: mem.h:344
unsigned int u32
Definition: types.h:88
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
Definition: physmem_funcs.h:44
#define CLIB_MEM_VM_F_SHARED
Definition: mem.h:326
uword size
int fd
File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
Definition: mem.h:347
struct _unformat_input_t unformat_input_t
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:274
#define CLIB_MEM_VM_F_NUMA_FORCE
Definition: mem.h:329
clib_error_t * linux_vfio_init(vlib_main_t *vm)
Definition: vfio.c:218
u32 flags
Definition: vhost_user.h:110
vlib_main_t * vm
Definition: buffer.c:294
void * mheap_get_aligned(void *v, uword n_user_data_bytes, uword align, uword align_offset, uword *offset_return)
Definition: mheap.c:643
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:339
#define clib_warning(format, args...)
Definition: error.h:59
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file descriptor will be provided ...
Definition: mem.h:332
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
void * mheap_alloc_with_flags(void *memory, uword memory_size, uword flags)
Definition: mheap.c:885
#define CLIB_MEM_VM_F_HUGETLB_PREALLOC
Definition: mem.h:330
#define CLIB_MEM_VM_F_LOCKED
Definition: mem.h:331
static void unix_physmem_free(vlib_main_t *vm, vlib_physmem_region_index_t idx, void *x)
Definition: physmem.c:105
static uword pointer_to_uword(const void *p)
Definition: types.h:131
vlib_physmem_region_index_t index
Definition: physmem.h:47
#define clib_max(x, y)
Definition: clib.h:282
static clib_error_t * show_physmem(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: physmem.c:244
void mheap_put(void *v, uword uoffset)
Definition: mheap.c:771
static void unix_physmem_region_free(vlib_main_t *vm, vlib_physmem_region_index_t idx)
Definition: physmem.c:203
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
u64 uword
Definition: types.h:112
u64 * clib_mem_vm_get_paddr(void *mem, int log2_page_size, int n_pages)
Definition: mem.c:256
clib_error_t * unix_physmem_init(vlib_main_t *vm)
Definition: physmem.c:216
vlib_physmem_main_t physmem_main
Definition: main.c:64
int log2_page_size
Definition: mem.h:348
static long move_pages(int pid, unsigned long count, void **pages, const int *nodes, int *status, int flags)
Definition: syscall.h:36
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:62
clib_error_t *(* os_physmem_region_alloc)(struct vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
Definition: main.h:116
u8 vlib_physmem_region_index_t
Definition: physmem.h:43
vlib_physmem_region_t * regions
Definition: physmem.h:73