FD.io VPP  v18.01-8-g0eacf49
Vector Packet Processing
mem.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #define _GNU_SOURCE
17 #include <stdlib.h>
18 #include <sys/types.h>
19 #include <sys/stat.h>
20 #include <unistd.h>
21 #include <sys/mount.h>
22 #include <sys/mman.h>
23 #include <fcntl.h>
24 #include <linux/mempolicy.h>
25 #include <linux/memfd.h>
26 
27 #include <vppinfra/clib.h>
28 #include <vppinfra/mem.h>
29 #include <vppinfra/format.h>
30 #include <vppinfra/clib_error.h>
31 #include <vppinfra/linux/syscall.h>
32 #include <vppinfra/linux/sysfs.h>
33 
34 #ifndef F_LINUX_SPECIFIC_BASE
35 #define F_LINUX_SPECIFIC_BASE 1024
36 #endif
37 
38 #ifndef F_ADD_SEALS
39 #define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
40 #define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10)
41 
42 #define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */
43 #define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */
44 #define F_SEAL_GROW 0x0004 /* prevent file from growing */
45 #define F_SEAL_WRITE 0x0008 /* prevent writes */
46 #endif
47 
48 int
50 {
51  struct stat st = { 0 };
52  if (fstat (fd, &st) == -1)
53  return 0;
54  return min_log2 (st.st_blksize);
55 }
56 
59 {
60  int fd = -1;
61  clib_error_t *err = 0;
62  void *addr = 0;
63  u8 *filename = 0;
64  int mmap_flags = MAP_SHARED;
65  int log2_page_size;
66  int n_pages;
67  int old_mpol = -1;
68  u64 old_mask[16] = { 0 };
69 
70  /* save old numa mem policy if needed */
72  {
73  int rv;
74  rv =
75  get_mempolicy (&old_mpol, old_mask, sizeof (old_mask) * 8 + 1, 0, 0);
76 
77  if (rv == -1)
78  {
79  if ((a->flags & CLIB_MEM_VM_F_NUMA_FORCE) != 0)
80  {
81  err = clib_error_return_unix (0, "get_mempolicy");
82  goto error;
83  }
84  else
85  old_mpol = -1;
86  }
87  }
88 
89  /* if we are creating shared segment, we need file descriptor */
90  if (a->flags & CLIB_MEM_VM_F_SHARED)
91  {
92  /* if hugepages are needed we need to create mount point */
94  {
95  char *mount_dir;
96  char template[] = "/tmp/hugepage_mount.XXXXXX";
97 
98  mount_dir = mkdtemp (template);
99  if (mount_dir == 0)
100  return clib_error_return_unix (0, "mkdtemp \'%s\'", template);
101 
102  if (mount ("none", (char *) mount_dir, "hugetlbfs", 0, NULL))
103  {
104  err = clib_error_return_unix (0, "mount hugetlb directory '%s'",
105  mount_dir);
106  goto error;
107  }
108 
109  filename = format (0, "%s/%s%c", mount_dir, a->name, 0);
110 
111  if ((fd = open ((char *) filename, O_CREAT | O_RDWR, 0755)) == -1)
112  {
113  err = clib_error_return_unix (0, "open");
114  goto error;
115  }
116  umount2 ((char *) mount_dir, MNT_DETACH);
117  rmdir ((char *) mount_dir);
118  mmap_flags |= MAP_LOCKED;
119  }
120  else
121  {
122  if ((fd = memfd_create (a->name, MFD_ALLOW_SEALING)) == -1)
123  {
124  err = clib_error_return_unix (0, "memfd_create");
125  goto error;
126  }
127 
128  if ((fcntl (fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1)
129  {
130  err = clib_error_return_unix (0, "fcntl (F_ADD_SEALS)");
131  goto error;
132  }
133  }
134  log2_page_size = clib_mem_vm_get_log2_page_size (fd);
135 
136  if (log2_page_size == 0)
137  {
138  err = clib_error_return_unix (0, "cannot determine page size");
139  goto error;
140  }
141  }
142  else /* not CLIB_MEM_VM_F_SHARED */
143  {
144  if (a->flags & CLIB_MEM_VM_F_HUGETLB)
145  {
146  mmap_flags |= MAP_HUGETLB | MAP_PRIVATE | MAP_ANONYMOUS;
147  log2_page_size = 21;
148  }
149  else
150  {
151  mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
152  log2_page_size = min_log2 (sysconf (_SC_PAGESIZE));
153  }
154  }
155 
156  n_pages = ((a->size - 1) >> log2_page_size) + 1;
157 
158 
160  {
162  1 << (log2_page_size - 10),
163  n_pages);
164  if (err)
165  goto error;
166 
167  }
168 
169  if (fd != -1)
170  if ((ftruncate (fd, a->size)) == -1)
171  {
172  err = clib_error_return_unix (0, "ftruncate");
173  goto error;
174  }
175 
176  if (old_mpol != -1)
177  {
178  int rv;
179  u64 mask[16] = { 0 };
180  mask[0] = 1 << a->numa_node;
181  rv = set_mempolicy (MPOL_BIND, mask, sizeof (mask) * 8 + 1);
182  if (rv)
183  {
184  err = clib_error_return_unix (0, "set_mempolicy");
185  goto error;
186  }
187  }
188 
189  addr = mmap (0, a->size, (PROT_READ | PROT_WRITE), mmap_flags, fd, 0);
190  if (addr == MAP_FAILED)
191  {
192  err = clib_error_return_unix (0, "mmap");
193  goto error;
194  }
195 
196  /* re-apply ole numa memory policy */
197  if (old_mpol != -1 &&
198  set_mempolicy (old_mpol, old_mask, sizeof (old_mask) * 8 + 1) == -1)
199  {
200  err = clib_error_return_unix (0, "set_mempolicy");
201  goto error;
202  }
203 
204  a->log2_page_size = log2_page_size;
205  a->n_pages = n_pages;
206  a->addr = addr;
207  a->fd = fd;
208  goto done;
209 
210 error:
211  if (fd != -1)
212  close (fd);
213 
214 done:
215  vec_free (filename);
216  return err;
217 }
218 
219 u64 *
220 clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages)
221 {
222  int pagesize = sysconf (_SC_PAGESIZE);
223  int fd;
224  int i;
225  u64 *r = 0;
226 
227  if ((fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1)
228  return 0;
229 
230  for (i = 0; i < n_pages; i++)
231  {
232  u64 seek, pagemap = 0;
233  uword vaddr = pointer_to_uword (mem) + (((u64) i) << log2_page_size);
234  seek = ((u64) vaddr / pagesize) * sizeof (u64);
235  if (lseek (fd, seek, SEEK_SET) != seek)
236  goto done;
237 
238  if (read (fd, &pagemap, sizeof (pagemap)) != (sizeof (pagemap)))
239  goto done;
240 
241  if ((pagemap & (1ULL << 63)) == 0)
242  goto done;
243 
244  pagemap &= pow2_mask (55);
245  vec_add1 (r, pagemap * pagesize);
246  }
247 
248 done:
249  close (fd);
250  if (vec_len (r) != n_pages)
251  {
252  vec_free (r);
253  return 0;
254  }
255  return r;
256 }
257 
258 
259 
260 /*
261  * fd.io coding-style-patch-verification: ON
262  *
263  * Local Variables:
264  * eval: (c-set-style "gnu")
265  * End:
266  */
#define CLIB_MEM_VM_F_HUGETLB
Definition: mem.h:327
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:337
#define MFD_ALLOW_SEALING
Definition: memfd.h:177
a
Definition: bitmap.h:516
#define CLIB_MEM_VM_F_NUMA_PREFER
Definition: mem.h:328
#define F_ADD_SEALS
Definition: mem.c:39
#define NULL
Definition: clib.h:55
void * addr
Pointer to allocated memory, set on successful allocation.
Definition: mem.h:344
static int memfd_create(const char *name, unsigned int flags)
Definition: syscall.h:43
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:518
int numa_node
numa node preference.
Definition: mem.h:343
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
static uword min_log2(uword x)
Definition: clib.h:197
clib_error_t * clib_mem_vm_ext_alloc(clib_mem_vm_alloc_t *a)
Definition: mem.c:58
static long set_mempolicy(int mode, const unsigned long *nodemask, unsigned long maxnode)
Definition: syscall.h:23
static uword pow2_mask(uword x)
Definition: clib.h:265
char * name
Name for memory allocation, set by caller.
Definition: mem.h:341
unsigned long u64
Definition: types.h:89
uword size
Allocation size, set by caller.
Definition: mem.h:342
#define F_SEAL_SHRINK
Definition: mem.c:43
static uword pointer_to_uword(const void *p)
Definition: types.h:131
#define CLIB_MEM_VM_F_SHARED
Definition: mem.h:326
int fd
File desriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set.
Definition: mem.h:345
#define clib_error_return_unix(e, args...)
Definition: error.h:102
#define CLIB_MEM_VM_F_NUMA_FORCE
Definition: mem.h:329
static int get_mempolicy(int *mode, unsigned long *nodemask, unsigned long maxnode, void *addr, unsigned long flags)
Definition: syscall.h:29
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:336
u32 flags
vm allocation flags: CLIB_MEM_VM_F_SHARED: request shared memory, file destiptor will be provided o...
Definition: mem.h:331
int clib_mem_vm_get_log2_page_size(int fd)
Definition: mem.c:49
#define CLIB_MEM_VM_F_HUGETLB_PREALLOC
Definition: mem.h:330
u64 uword
Definition: types.h:112
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
u64 * clib_mem_vm_get_paddr(void *mem, int log2_page_size, int n_pages)
Definition: mem.c:220
int log2_page_size
Definition: mem.h:346
vhost_vring_addr_t addr
Definition: vhost-user.h:83
clib_error_t * clib_sysfs_prealloc_hugepages(int numa_node, int page_size, int nr)
Definition: sysfs.c:224