FD.io VPP  v18.01-8-g0eacf49
Vector Packet Processing
physmem.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * physmem.c: Unix physical memory
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <unistd.h>
41 #include <sys/types.h>
42 #include <sys/mount.h>
43 #include <sys/mman.h>
44 #include <sys/fcntl.h>
45 #include <sys/stat.h>
46 #include <linux/vfio.h>
47 #include <unistd.h>
48 
49 #include <vppinfra/linux/syscall.h>
50 #include <vppinfra/linux/sysfs.h>
51 #include <vlib/vlib.h>
52 #include <vlib/physmem.h>
53 #include <vlib/unix/unix.h>
54 
55 static int vfio_container_fd = -1;
56 
57 static void *
59  uword n_bytes, uword alignment)
60 {
62  uword lo_offset, hi_offset;
63  uword *to_free = 0;
64 
65  if (pr->heap == 0)
66  return 0;
67 
68  /* IO memory is always at least cache aligned. */
69  alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES);
70 
71  while (1)
72  {
73  mheap_get_aligned (pr->heap, n_bytes,
74  /* align */ alignment,
75  /* align offset */ 0,
76  &lo_offset);
77 
78  /* Allocation failed? */
79  if (lo_offset == ~0)
80  break;
81 
82  if (pr->flags & VLIB_PHYSMEM_F_FAKE)
83  break;
84 
85  /* Make sure allocation does not span DMA physical chunk boundary. */
86  hi_offset = lo_offset + n_bytes - 1;
87 
88  if (((pointer_to_uword (pr->heap) + lo_offset) >> pr->log2_page_size) ==
89  ((pointer_to_uword (pr->heap) + hi_offset) >> pr->log2_page_size))
90  break;
91 
92  /* Allocation would span chunk boundary, queue it to be freed as soon as
93  we find suitable chunk. */
94  vec_add1 (to_free, lo_offset);
95  }
96 
97  if (to_free != 0)
98  {
99  uword i;
100  for (i = 0; i < vec_len (to_free); i++)
101  mheap_put (pr->heap, to_free[i]);
102  vec_free (to_free);
103  }
104 
105  return lo_offset != ~0 ? pr->heap + lo_offset : 0;
106 }
107 
108 static void
110 {
112  /* Return object to region's heap. */
113  mheap_put (pr->heap, x - pr->heap);
114 }
115 
116 static clib_error_t *
117 scan_vfio_fd (void *arg, u8 * path_name, u8 * file_name)
118 {
119  const char fn[] = "/dev/vfio/vfio";
120  char buff[sizeof (fn)] = { 0 };
121 
122  if (readlink ((char *) path_name, buff, sizeof (fn)) + 1 != sizeof (fn))
123  return 0;
124 
125  if (strncmp (fn, buff, sizeof (fn)))
126  return 0;
127 
128  vfio_container_fd = atoi ((char *) file_name);
129  return 0;
130 }
131 
132 static clib_error_t *
134 {
135  struct vfio_iommu_type1_dma_map dma_map = { 0 };
136  int i, fd;
137 
138  if (vfio_container_fd == -1)
139  foreach_directory_file ("/proc/self/fd", scan_vfio_fd, 0, 0);
140 
141  fd = vfio_container_fd;
142 
143  if (fd < 0)
144  return 0;
145 
146  if (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION)
147  return 0;
148 
149  if (ioctl (fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) == 0)
150  return 0;
151 
152  dma_map.argsz = sizeof (struct vfio_iommu_type1_dma_map);
153  dma_map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
154 
156  {
157  dma_map.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size);
158  dma_map.size = 1 << pr->log2_page_size;
159  dma_map.iova = pr->page_table[i];
160  if (ioctl (fd, VFIO_IOMMU_MAP_DMA, &dma_map) != 0)
161  return clib_error_return_unix (0, "ioctl (VFIO_IOMMU_MAP_DMA)");
162  }
163  return 0;
164 }
165 
166 static clib_error_t *
168  u8 numa_node, u32 flags,
170 {
171  vlib_physmem_main_t *vpm = &vm->physmem_main;
173  clib_error_t *error = 0;
174  clib_mem_vm_alloc_t alloc = { 0 };
175 
176 
177  if (geteuid () != 0 && (flags & VLIB_PHYSMEM_F_FAKE) == 0)
178  return clib_error_return (0, "not allowed");
179 
180  pool_get (vpm->regions, pr);
181 
182  if ((pr - vpm->regions) >= 256)
183  {
184  error = clib_error_return (0, "maximum number of regions reached");
185  goto error;
186  }
187 
188  alloc.name = name;
189  alloc.size = size;
190  alloc.numa_node = numa_node;
191  alloc.flags = CLIB_MEM_VM_F_SHARED;
192 
193  if ((flags & VLIB_PHYSMEM_F_FAKE) == 0)
194  {
195  alloc.flags |= CLIB_MEM_VM_F_HUGETLB;
198  }
199  else
200  {
202  }
203 
204  error = clib_mem_vm_ext_alloc (&alloc);
205  if (error)
206  goto error;
207 
208  pr->index = pr - vpm->regions;
209  pr->flags = flags;
210  pr->fd = alloc.fd;
211  pr->mem = alloc.addr;
212  pr->log2_page_size = alloc.log2_page_size;
213  pr->n_pages = alloc.n_pages;
214  pr->size = (u64) pr->n_pages << (u64) pr->log2_page_size;
215  pr->page_mask = (1 << pr->log2_page_size) - 1;
216  pr->numa_node = numa_node;
217  pr->name = format (0, "%s", name);
218 
219  if ((flags & VLIB_PHYSMEM_F_FAKE) == 0)
220  {
221  int i;
222  for (i = 0; i < pr->n_pages; i++)
223  {
224  void *ptr = pr->mem + (i << pr->log2_page_size);
225  int node;
226  move_pages (0, 1, &ptr, 0, &node, 0);
227  if (numa_node != node)
228  {
229  clib_warning ("physmem page for region \'%s\' allocated on the"
230  " wrong numa node (requested %u actual %u)",
231  pr->name, pr->numa_node, node, i);
232  break;
233  }
234  }
236  pr->n_pages);
238  if (error)
239  clib_error_report (error);
240  }
241 
242  if (flags & VLIB_PHYSMEM_F_INIT_MHEAP)
243  {
244  pr->heap = mheap_alloc_with_flags (pr->mem, pr->size,
245  /* Don't want mheap mmap/munmap with IO memory. */
248  }
249 
250  *idx = pr->index;
251 
252  goto done;
253 
254 error:
255  memset (pr, 0, sizeof (*pr));
256  pool_put (vpm->regions, pr);
257 
258 done:
259  return error;
260 }
261 
262 static void
264 {
265  vlib_physmem_main_t *vpm = &vm->physmem_main;
267 
268  if (pr->fd > 0)
269  close (pr->fd);
270  munmap (pr->mem, pr->size);
271  vec_free (pr->name);
272  pool_put (vpm->regions, pr);
273 }
274 
275 clib_error_t *
277 {
278  clib_error_t *error = 0;
279 
280  /* Avoid multiple calls. */
281  if (vm->os_physmem_alloc_aligned)
282  return error;
283 
288 
289  return error;
290 }
291 
292 static clib_error_t *
294  unformat_input_t * input, vlib_cli_command_t * cmd)
295 {
296  vlib_physmem_main_t *vpm = &vm->physmem_main;
298 
299  /* *INDENT-OFF* */
300  pool_foreach (pr, vpm->regions, (
301  {
302  vlib_cli_output (vm, "index %u name '%s' page-size %uKB num-pages %d "
303  "numa-node %u fd %d\n",
304  pr->index, pr->name, (1 << (pr->log2_page_size -10)),
305  pr->n_pages, pr->numa_node, pr->fd);
306  if (pr->heap)
307  vlib_cli_output (vm, " %U", format_mheap, pr->heap, /* verbose */ 1);
308  else
309  vlib_cli_output (vm, " no heap\n");
310  }));
311  /* *INDENT-ON* */
312  return 0;
313 }
314 
315 /* *INDENT-OFF* */
316 VLIB_CLI_COMMAND (show_physmem_command, static) = {
317  .path = "show physmem",
318  .short_help = "Show physical memory allocation",
319  .function = show_physmem,
320 };
321 /* *INDENT-ON* */
322 
323 /*
324  * fd.io coding-style-patch-verification: ON
325  *
326  * Local Variables:
327  * eval: (c-set-style "gnu")
328  * End:
329  */
#define CLIB_MEM_VM_F_HUGETLB
Definition: mem.h:327
#define vec_foreach_index(var, v)
Iterate over vector indices.
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
#define VLIB_PHYSMEM_F_INIT_MHEAP
Definition: physmem.h:57
void(* os_physmem_region_free)(struct vlib_main_t *vm, vlib_physmem_region_index_t idx)
Definition: main.h:119
vlib_physmem_main_t physmem_main
Definition: main.h:108
#define CLIB_MEM_VM_F_NUMA_PREFER
Definition: mem.h:328
void * addr
Pointer to allocated memory, set on successful allocation.
Definition: mem.h:344
static clib_error_t * unix_physmem_region_alloc(vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
Definition: physmem.c:167
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:518
static clib_error_t * scan_vfio_fd(void *arg, u8 *path_name, u8 *file_name)
Definition: physmem.c:117
int numa_node
numa node preference.
Definition: mem.h:343
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define MHEAP_FLAG_THREAD_SAFE
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:225
static void * unix_physmem_alloc_aligned(vlib_main_t *vm, vlib_physmem_region_index_t idx, uword n_bytes, uword alignment)
Definition: physmem.c:58
static int vfio_container_fd
Definition: physmem.c:55
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:58
#define pool_foreach(VAR, POOL, BODY)
Iterate through pool.
Definition: pool.h:438
void(* os_physmem_free)(struct vlib_main_t *vm, vlib_physmem_region_index_t idx, void *x)
Definition: main.h:125
#define MHEAP_FLAG_DISABLE_VM
char * name
Name for memory allocation, set by caller.
Definition: mem.h:341
void *(* os_physmem_alloc_aligned)(struct vlib_main_t *vm, vlib_physmem_region_index_t idx, uword n_bytes, uword alignment)
Definition: main.h:122
#define clib_error_return(e, args...)
Definition: error.h:99
unsigned long u64
Definition: types.h:89
uword size
Allocation size, set by caller.
Definition: mem.h:342
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static vlib_physmem_region_t * vlib_physmem_get_region(vlib_main_t *vm, u8 index)
Definition: physmem_funcs.h:44
#define CLIB_MEM_VM_F_SHARED
Definition: mem.h:326
int fd
File desriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
Definition: mem.h:345
struct _unformat_input_t unformat_input_t
#define clib_error_return_unix(e, args...)
Definition: error.h:102
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:271
#define CLIB_MEM_VM_F_NUMA_FORCE
Definition: mem.h:329
vlib_main_t * vm
Definition: buffer.c:283
void * mheap_get_aligned(void *v, uword n_user_data_bytes, uword align, uword align_offset, uword *offset_return)
Definition: mheap.c:643
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
#define clib_warning(format, args...)
Definition: error.h:59
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file destiptor will be provided o...
Definition: mem.h:331
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
void * mheap_alloc_with_flags(void *memory, uword memory_size, uword flags)
Definition: mheap.c:869
#define CLIB_MEM_VM_F_HUGETLB_PREALLOC
Definition: mem.h:330
unsigned int u32
Definition: types.h:88
static clib_error_t * unix_physmem_region_iommu_register(vlib_physmem_region_t *pr)
Definition: physmem.c:133
static void unix_physmem_free(vlib_main_t *vm, vlib_physmem_region_index_t idx, void *x)
Definition: physmem.c:109
u64 size
Definition: vhost-user.h:76
#define clib_error_report(e)
Definition: error.h:113
vlib_physmem_region_index_t index
Definition: physmem.h:47
#define clib_max(x, y)
Definition: clib.h:333
u64 uword
Definition: types.h:112
static clib_error_t * show_physmem(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: physmem.c:293
void mheap_put(void *v, uword uoffset)
Definition: mheap.c:755
static void unix_physmem_region_free(vlib_main_t *vm, vlib_physmem_region_index_t idx)
Definition: physmem.c:263
clib_error_t * foreach_directory_file(char *dir_name, clib_error_t *(*f)(void *arg, u8 *path_name, u8 *file_name), void *arg, int scan_dirs)
Definition: util.c:49
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
u64 * clib_mem_vm_get_paddr(void *mem, int log2_page_size, int n_pages)
Definition: mem.c:220
#define VLIB_PHYSMEM_F_FAKE
Definition: physmem.h:58
clib_error_t * unix_physmem_init(vlib_main_t *vm)
Definition: physmem.c:276
int log2_page_size
Definition: mem.h:346
u32 flags
Definition: vhost-user.h:77
static long move_pages(int pid, unsigned long count, void **pages, const int *nodes, int *status, int flags)
Definition: syscall.h:36
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
clib_error_t *(* os_physmem_region_alloc)(struct vlib_main_t *vm, char *name, u32 size, u8 numa_node, u32 flags, vlib_physmem_region_index_t *idx)
Definition: main.h:113
u8 vlib_physmem_region_index_t
Definition: physmem.h:43
vlib_physmem_region_t * regions
Definition: physmem.h:69