FD.io VPP  v17.04-9-g99c0734
Vector Packet Processing
physmem.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 Cisco and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 /*
16  * physmem.c: Unix physical memory
17  *
18  * Copyright (c) 2008 Eliot Dresselhaus
19  *
20  * Permission is hereby granted, free of charge, to any person obtaining
21  * a copy of this software and associated documentation files (the
22  * "Software"), to deal in the Software without restriction, including
23  * without limitation the rights to use, copy, modify, merge, publish,
24  * distribute, sublicense, and/or sell copies of the Software, and to
25  * permit persons to whom the Software is furnished to do so, subject to
26  * the following conditions:
27  *
28  * The above copyright notice and this permission notice shall be
29  * included in all copies or substantial portions of the Software.
30  *
31  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38  */
39 
40 #include <vlib/unix/physmem.h>
41 
43 
44 static void *
46  uword alignment)
47 {
48  vlib_main_t *vm = vlib_get_main ();
50  uword lo_offset, hi_offset;
51  uword *to_free = 0;
52 
54  clib_warning ("unsafe alloc!");
55 
56  /* IO memory is always at least cache aligned. */
57  alignment = clib_max (alignment, CLIB_CACHE_LINE_BYTES);
58 
59  while (1)
60  {
61  mheap_get_aligned (pm->heap, n_bytes,
62  /* align */ alignment,
63  /* align offset */ 0,
64  &lo_offset);
65 
66  /* Allocation failed? */
67  if (lo_offset == ~0)
68  break;
69 
70  /* Make sure allocation does not span DMA physical chunk boundary. */
71  hi_offset = lo_offset + n_bytes - 1;
72 
73  if ((lo_offset >> vpm->log2_n_bytes_per_page) ==
74  (hi_offset >> vpm->log2_n_bytes_per_page))
75  break;
76 
77  /* Allocation would span chunk boundary, queue it to be freed as soon as
78  we find suitable chunk. */
79  vec_add1 (to_free, lo_offset);
80  }
81 
82  if (to_free != 0)
83  {
84  uword i;
85  for (i = 0; i < vec_len (to_free); i++)
86  mheap_put (pm->heap, to_free[i]);
87  vec_free (to_free);
88  }
89 
90  return lo_offset != ~0 ? pm->heap + lo_offset : 0;
91 }
92 
93 static void
95 {
97 
98  /* Return object to region's heap. */
99  mheap_put (pm->heap, x - pm->heap);
100 }
101 
102 static void
104 {
106 
107  if (!pm->shmid)
108  return;
109  shmctl (pm->shmid, IPC_RMID, 0);
110  pm->shmid = 0;
111 }
112 
113 /* try to use huge TLB pgs if possible */
114 static int
116 {
117  vlib_physmem_main_t *vpm = &vm->physmem_main;
119  u64 hugepagesize, pagesize;
120  u64 pfn, seek_loc;
121  u64 cur, physaddr, ptbits;
122  int fd, i;
123 
124  pm->shmid = shmget (11 /* key, my amp goes to 11 */ , pm->mem_size,
125  IPC_CREAT | SHM_HUGETLB | SHM_R | SHM_W);
126  if (pm->shmid < 0)
127  {
128  clib_unix_warning ("shmget");
129  return 0;
130  }
131 
132  pm->mem = shmat (pm->shmid, NULL, 0 /* flags */ );
133  if (pm->mem == 0)
134  {
135  shmctl (pm->shmid, IPC_RMID, 0);
136  return 0;
137  }
138 
139  memset (pm->mem, 0, pm->mem_size);
140 
141  /* $$$ get page size info from /proc/meminfo */
142  hugepagesize = 2 << 20;
143  pagesize = 4 << 10;
144  vpm->log2_n_bytes_per_page = min_log2 (hugepagesize);
145  vec_resize (vpm->page_table, pm->mem_size / hugepagesize);
146 
148  vpm->virtual.start = pointer_to_uword (pm->mem);
149  vpm->virtual.size = pm->mem_size;
150  vpm->virtual.end = vpm->virtual.start + vpm->virtual.size;
151 
152  fd = open ("/proc/self/pagemap", O_RDONLY);
153 
154  if (fd < 0)
155  {
156  (void) shmdt (pm->mem);
157  return 0;
158  }
159 
160  pm->heap = mheap_alloc_with_flags (pm->mem, pm->mem_size,
161  /* Don't want mheap mmap/munmap with IO memory. */
163 
164  cur = pointer_to_uword (pm->mem);
165  i = 0;
166 
167  while (cur < pointer_to_uword (pm->mem) + pm->mem_size)
168  {
169  pfn = (u64) cur / pagesize;
170  seek_loc = pfn * sizeof (u64);
171  if (lseek (fd, seek_loc, SEEK_SET) != seek_loc)
172  {
173  clib_unix_warning ("lseek to 0x%llx", seek_loc);
174  shmctl (pm->shmid, IPC_RMID, 0);
175  close (fd);
176  return 0;
177  }
178  if (read (fd, &ptbits, sizeof (ptbits)) != (sizeof (ptbits)))
179  {
180  clib_unix_warning ("read ptbits");
181  shmctl (pm->shmid, IPC_RMID, 0);
182  close (fd);
183  return 0;
184  }
185 
186  /* bits 0-54 are the physical page number */
187  physaddr = (ptbits & 0x7fffffffffffffULL) * pagesize;
188  if (CLIB_DEBUG > 1)
189  fformat (stderr, "pm: virtual 0x%llx physical 0x%llx\n",
190  cur, physaddr);
191  vpm->page_table[i++] = physaddr;
192 
193  cur += hugepagesize;
194  }
195  close (fd);
196  atexit (htlb_shutdown);
197  return 1;
198 }
199 
201  physmem_main_t * pm, int) __attribute__ ((weak));
202 int
204 {
205  return 0;
206 }
207 
208 clib_error_t *
209 unix_physmem_init (vlib_main_t * vm, int physical_memory_required)
210 {
211  vlib_physmem_main_t *vpm = &vm->physmem_main;
213  clib_error_t *error = 0;
214 
215  /* Avoid multiple calls. */
216  if (vm->os_physmem_alloc_aligned)
217  return error;
218 
221  pm->mem = MAP_FAILED;
222 
223  if (pm->mem_size == 0)
224  pm->mem_size = 16 << 20;
225 
226  /* OK, Mr. App, you tell us */
227  if (vlib_app_physmem_init (vm, pm, physical_memory_required))
228  return 0;
229 
230  if (!pm->no_hugepages && htlb_init (vm))
231  {
232  fformat (stderr, "%s: use huge pages\n", __FUNCTION__);
233  return 0;
234  }
235 
236  pm->mem =
237  mmap (0, pm->mem_size, PROT_READ | PROT_WRITE,
238  MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
239  if (pm->mem == MAP_FAILED)
240  {
241  error = clib_error_return_unix (0, "mmap");
242  goto done;
243  }
244 
245  pm->heap = mheap_alloc (pm->mem, pm->mem_size);
246 
247  /* Identity map with a single page. */
249  vec_add1 (vpm->page_table, pointer_to_uword (pm->mem));
250 
252  vpm->virtual.start = pointer_to_uword (pm->mem);
253  vpm->virtual.size = pm->mem_size;
254  vpm->virtual.end = vpm->virtual.start + vpm->virtual.size;
255  vpm->is_fake = 1;
256 
257  fformat (stderr, "%s: use fake dma pages\n", __FUNCTION__);
258 
259 done:
260  if (error)
261  {
262  if (pm->mem != MAP_FAILED)
263  munmap (pm->mem, pm->mem_size);
264  }
265  return error;
266 }
267 
268 static clib_error_t *
270  unformat_input_t * input, vlib_cli_command_t * cmd)
271 {
274  {
275  vlib_cli_output (vm, "Not supported with external buffer management.");
276  return 0;
277  }
278 
279  if (pm->heap)
280  vlib_cli_output (vm, "%U", format_mheap, pm->heap, /* verbose */ 1);
281  else
282  vlib_cli_output (vm, "No physmem allocated.");
283  return 0;
284 }
285 
286 /* *INDENT-OFF* */
287 VLIB_CLI_COMMAND (show_physmem_command, static) = {
288  .path = "show physmem",
289  .short_help = "Show physical memory allocation",
290  .function = show_physmem,
291 };
292 /* *INDENT-ON* */
293 
294 static clib_error_t *
296  unformat_input_t * input, vlib_cli_command_t * cmd)
297 {
298  cpu_set_t set;
299  cpu_set_t *setp = &set;
300  int i, rv;
301  u8 *s = 0;
302  int first_set_bit_in_run = -1;
303  int last_set_bit_in_run = -1;
304  int output_done = 0;
305 
306  rv = sched_getaffinity (0 /* pid, 0 = this proc */ ,
307  sizeof (*setp), setp);
308  if (rv < 0)
309  {
310  vlib_cli_output (vm, "Couldn't get affinity mask: %s\n",
311  strerror (errno));
312  return 0;
313  }
314 
315  for (i = 0; i < 64; i++)
316  {
317  if (CPU_ISSET (i, setp))
318  {
319  if (first_set_bit_in_run == -1)
320  {
321  first_set_bit_in_run = i;
322  last_set_bit_in_run = i;
323  if (output_done)
324  s = format (s, ",");
325  s = format (s, "%d-", i);
326  output_done = 1;
327  }
328  else
329  {
330  if (i == (last_set_bit_in_run + 1))
331  last_set_bit_in_run = i;
332  }
333  }
334  else
335  {
336  if (first_set_bit_in_run != -1)
337  {
338  if (first_set_bit_in_run == (i - 1))
339  {
340  _vec_len (s) -= 2 + ((first_set_bit_in_run / 10));
341  }
342  s = format (s, "%d", last_set_bit_in_run);
343  first_set_bit_in_run = -1;
344  last_set_bit_in_run = -1;
345  }
346  }
347  }
348 
349  if (first_set_bit_in_run != -1)
350  s = format (s, "%d", first_set_bit_in_run);
351 
352  vlib_cli_output (vm, "Process runs on: %v", s);
353  return 0;
354 }
355 
356 /* *INDENT-OFF* */
357 VLIB_CLI_COMMAND (show_affinity_command, static) = {
358  .path = "show affinity",
359  .short_help = "Show process cpu affinity",
360  .function = show_affinity,
361 };
362 /* *INDENT-ON* */
363 
364 static clib_error_t *
366  unformat_input_t * input, vlib_cli_command_t * cmd)
367 {
368  cpu_set_t set;
369  cpu_set_t *setp = &set;
370  int i, rv;
371  int another_round;
372  u32 first, last;
373 
374  memset (setp, 0, sizeof (*setp));
375 
376  do
377  {
378  another_round = 0;
379  if (unformat (input, "%d-%d,", &first, &last))
380  {
381  if (first > 64 || last > 64)
382  {
383  barf1:
384  vlib_cli_output (vm, "range %d-%d invalid", first, last);
385  return 0;
386  }
387 
388  for (i = first; i <= last; i++)
389  CPU_SET (i, setp);
390  another_round = 1;
391  }
392  else if (unformat (input, "%d-%d", &first, &last))
393  {
394  if (first > 64 || last > 64)
395  goto barf1;
396 
397  for (i = first; i <= last; i++)
398  CPU_SET (i, setp);
399  }
400  else if (unformat (input, "%d,", &first))
401  {
402  if (first > 64)
403  {
404  barf2:
405  vlib_cli_output (vm, "cpu %d invalid", first);
406  return 0;
407  }
408  CPU_SET (first, setp);
409  another_round = 1;
410  }
411  else if (unformat (input, "%d", &first))
412  {
413  if (first > 64)
414  goto barf2;
415 
416  CPU_SET (first, setp);
417  }
418  }
419  while (another_round);
420 
421  rv = sched_setaffinity (0 /* pid, 0 = this proc */ ,
422  sizeof (*setp), setp);
423 
424  if (rv < 0)
425  {
426  vlib_cli_output (vm, "Couldn't get affinity mask: %s\n",
427  strerror (errno));
428  return 0;
429  }
430  return show_affinity (vm, input, cmd);
431 }
432 
433 /* *INDENT-OFF* */
434 VLIB_CLI_COMMAND (set_affinity_command, static) = {
435  .path = "set affinity",
436  .short_help = "Set process cpu affinity",
437  .function = set_affinity,
438 };
439 /* *INDENT-ON* */
440 
441 static clib_error_t *
443 {
445  u32 size_in_mb;
446 
448  {
449  if (unformat (input, "no-huge") || unformat (input, "no-huge-pages"))
450  pm->no_hugepages = 1;
451 
452  else if (unformat (input, "size-in-mb %d", &size_in_mb) ||
453  unformat (input, "size %d", &size_in_mb))
454  pm->mem_size = size_in_mb << 20;
455  else
456  return unformat_parse_error (input);
457  }
458 
459  unformat_free (input);
460  return 0;
461 }
462 
464 
465 /*
466  * fd.io coding-style-patch-verification: ON
467  *
468  * Local Variables:
469  * eval: (c-set-style "gnu")
470  * End:
471  */
sll srl srl sll sra u16x4 i
Definition: vector_sse2.h:343
static clib_error_t * set_affinity(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: physmem.c:365
vlib_physmem_main_t physmem_main
Definition: main.h:106
void * mheap_alloc(void *memory, uword size)
Definition: mheap.c:953
#define NULL
Definition: clib.h:55
uword mem_size
Definition: physmem.h:44
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static physmem_main_t physmem_main
Definition: physmem.c:42
static heap_elt_t * last(heap_header_t *h)
Definition: heap.c:53
vlib_buffer_main_t * buffer_main
Definition: main.h:104
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:418
u8 * format_mheap(u8 *s, va_list *va)
Definition: mheap.c:1168
int vlib_app_physmem_init(vlib_main_t *vm, physmem_main_t *pm, int)
Definition: physmem.c:203
static uword min_log2(uword x)
Definition: clib.h:189
static int htlb_init(vlib_main_t *vm)
Definition: physmem.c:115
static uword pow2_mask(uword x)
Definition: clib.h:257
#define MHEAP_FLAG_DISABLE_VM
void * mem
Definition: physmem.h:41
unsigned long u64
Definition: types.h:89
#define vec_resize(V, N)
Resize a vector (no header, unspecified alignment) Add N elements to end of given vector V...
Definition: vec.h:241
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
struct _unformat_input_t unformat_input_t
#define clib_error_return_unix(e, args...)
Definition: error.h:114
static clib_error_t * vlib_physmem_configure(vlib_main_t *vm, unformat_input_t *input)
Definition: physmem.c:442
static void * unix_physmem_alloc_aligned(vlib_physmem_main_t *vpm, uword n_bytes, uword alignment)
Definition: physmem.c:45
word fformat(FILE *f, char *fmt,...)
Definition: format.c:452
static void htlb_shutdown(void)
Definition: physmem.c:103
static clib_error_t * show_affinity(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: physmem.c:295
void * heap
Definition: physmem.h:47
#define VLIB_EARLY_CONFIG_FUNCTION(x, n,...)
Definition: init.h:140
#define UNFORMAT_END_OF_INPUT
Definition: format.h:143
void *(* os_physmem_alloc_aligned)(vlib_physmem_main_t *pm, uword n_bytes, uword alignment)
Definition: main.h:110
void * mheap_get_aligned(void *v, uword n_user_data_bytes, uword align, uword align_offset, uword *offset_return)
Definition: mheap.c:649
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:340
#define clib_warning(format, args...)
Definition: error.h:59
#define VLIB_CLI_COMMAND(x,...)
Definition: cli.h:154
void * mheap_alloc_with_flags(void *memory, uword memory_size, uword flags)
Definition: mheap.c:875
unsigned int u32
Definition: types.h:88
int no_hugepages
Definition: physmem.h:53
void(* os_physmem_free)(void *x)
Definition: main.h:112
#define clib_max(x, y)
Definition: clib.h:325
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
uword log2_n_bytes_per_page
Definition: physmem.h:52
u64 uword
Definition: types.h:112
static void unix_physmem_free(void *x)
Definition: physmem.c:94
#define unformat_parse_error(input)
Definition: format.h:267
static clib_error_t * show_physmem(vlib_main_t *vm, unformat_input_t *input, vlib_cli_command_t *cmd)
Definition: physmem.c:269
void mheap_put(void *v, uword uoffset)
Definition: mheap.c:761
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
unsigned char u8
Definition: types.h:56
clib_error_t * unix_physmem_init(vlib_main_t *vm, int physical_memory_required)
Definition: physmem.c:209
static void unformat_free(unformat_input_t *i)
Definition: format.h:161
#define clib_unix_warning(format, args...)
Definition: error.h:68
int extern_buffer_mgmt
Definition: buffer.h:408
vlib_physmem_region_t virtual
Definition: physmem.h:50
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:67
void vlib_cli_output(vlib_main_t *vm, char *fmt,...)
Definition: cli.c:577
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:971
static uword unformat_check_input(unformat_input_t *i)
Definition: format.h:169