FD.io VPP  v19.01.1-17-ge106252
Vector Packet Processing
main.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2017 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #define _GNU_SOURCE
19 #include <stdint.h>
20 #include <net/if.h>
21 #include <sys/types.h>
22 #include <fcntl.h>
23 #include <sys/ioctl.h>
24 #include <sys/socket.h>
25 #include <sys/un.h>
26 #include <sys/uio.h>
27 #include <sys/mman.h>
28 #include <sys/prctl.h>
29 #include <inttypes.h>
30 #include <string.h>
31 #include <stdio.h>
32 #include <netdb.h>
33 #include <linux/ip.h>
34 #include <linux/icmp.h>
35 #include <arpa/inet.h>
36 #include <stdlib.h>
37 #include <netinet/if_ether.h>
38 #include <net/if_arp.h>
39 #include <asm/byteorder.h>
40 #include <byteswap.h>
41 #include <string.h>
42 #include <errno.h>
43 #include <sys/stat.h>
44 #include <sys/eventfd.h>
45 #include <sys/timerfd.h>
46 #include <sys/epoll.h>
47 #include <signal.h>
48 #include <linux/memfd.h>
49 
50 /* memif protocol msg, ring and descriptor definitions */
51 #include <memif.h>
52 /* memif api */
53 #include <libmemif.h>
54 /* socket messaging functions */
55 #include <socket.h>
56 /* private structs and functions */
57 #include <memif_private.h>
58 
59 #define ERRLIST_LEN 40
60 #define MAX_ERRBUF_LEN 256
61 
62 #if __x86_x64__
63 #define MEMIF_MEMORY_BARRIER() __builtin_ia32_sfence ()
64 #else
65 #define MEMIF_MEMORY_BARRIER() __sync_synchronize ()
66 #endif /* __x86_x64__ */
67 
70 int poll_cancel_fd = -1;
71 
73 
74 const char *memif_errlist[ERRLIST_LEN] = { /* MEMIF_ERR_SUCCESS */
75  "Success.",
76  /* MEMIF_ERR_SYSCALL */
77  "Unspecified syscall error (build with -DMEMIF_DBG or make debug).",
78  /* MEMIF_ERR_CONNREFUSED */
79  "Connection refused",
80  /* MEMIF_ERR_ACCES */
81  "Permission to resoure denied.",
82  /* MEMIF_ERR_NO_FILE */
83  "Socket file does not exist",
84  /* MEMIF_ERR_FILE_LIMIT */
85  "System limit on total numer of open files reached.",
86  /* MEMIF_ERR_PROC_FILE_LIMIT */
87  "Per-process limit on total number of open files reached.",
88  /* MEMIF_ERR_ALREADY */
89  "Connection already requested.",
90  /* MEMIF_ERR_AGAIN */
91  "File descriptor refers to file other than socket, or operation would block.",
92  /* MEMIF_ERR_BAD_FD */
93  "Bad file descriptor.",
94  /* MEMIF_ERR_NOMEM */
95  "Out of memory.",
96  /* MEMIF_ERR_INVAL_ARG */
97  "Invalid argument.",
98  /* MEMIF_ERR_NOCONN */
99  "Memif connection handle does not point to existing conenction",
100  /* MEMIF_ERR_CONN */
101  "Memif connection handle points to existing connection",
102  /* MEMIF_ERR_CB_FDUPDATE */
103  "Callback memif_control_fd_update_t returned error",
104  /* MEMIF_ERR_FILE_NOT_SOCK */
105  "File specified by socket filename exists and is not socket.",
106  /* MEMIF_ERR_NO_SHMFD */
107  "Missing shared memory file descriptor. (internal error)",
108  /* MEMIF_ERR_COOKIE */
109  "Invalid cookie on ring. (internal error)",
110  /* MEMIF_ERR_NOBUF_RING */
111  "Ring buffer full.",
112  /* MEMIF_ERR_NOBUF */
113  "Not enough memif buffers. There are unreceived data in shared memory.",
114  /* MEMIF_ERR_NOBUF_DET */
115  "Not enough space for memif details in suplied buffer. String data might be malformed.",
116  /* MEMIF_ERR_INT_WRITE */
117  "Send interrupt error.",
118  /* MEMIF_ERR_MFMSG */
119  "Malformed message received on control channel.",
120  /* MEMIF_ERR_QID */
121  "Invalid queue id",
122  /* MEMIF_ERR_PROTO */
123  "Incompatible memory interface protocol version.",
124  /* MEMIF_ERR_ID */
125  "Unmatched interface id.",
126  /* MEMIF_ERR_ACCSLAVE */
127  "Slave cannot accept connection reqest.",
128  /* MEMIF_ERR_ALRCONN */
129  "Interface is already connected.",
130  /* MEMIF_ERR_MODE */
131  "Mode mismatch.",
132  /* MEMIF_ERR_SECRET */
133  "Secret mismatch.",
134  /* MEMIF_ERR_NOSECRET */
135  "Secret required.",
136  /* MEMIF_ERR_MAXREG */
137  "Limit on total number of regions reached.",
138  /* MEMIF_ERR_MAXRING */
139  "Limit on total number of ring reached.",
140  /* MEMIF_ERR_NO_INTFD */
141  "Missing interrupt file descriptor. (internal error)",
142  /* MEMIF_ERR_DISCONNECT */
143  "Interface received disconnect request.",
144  /* MEMIF_ERR_DISCONNECTED */
145  "Interface is disconnected.",
146  /* MEMIF_ERR_UNKNOWN_MSG */
147  "Unknown message type received on control channel. (internal error)",
148  /* MEMIF_ERR_POLL_CANCEL */
149  "Memif event polling was canceled.",
150  /* MEMIF_ERR_MAX_RING */
151  "Maximum log2 ring size is 15",
152  /* MEMIF_ERR_PRIVHDR */
153  "Private headers not supported."
154 };
155 
156 #define MEMIF_ERR_UNDEFINED "undefined error"
157 
158 char *
159 memif_strerror (int err_code)
160 {
161  if (err_code >= ERRLIST_LEN)
162  {
164  memif_buf[strlen (MEMIF_ERR_UNDEFINED)] = '\0';
165  }
166  else
167  {
168  strncpy (memif_buf, memif_errlist[err_code],
169  strlen (memif_errlist[err_code]));
170  memif_buf[strlen (memif_errlist[err_code])] = '\0';
171  }
172  return memif_buf;
173 }
174 
175 uint16_t
177 {
178  return MEMIF_VERSION;
179 }
180 
181 #define DBG_TX_BUF (0)
182 #define DBG_RX_BUF (1)
183 
184 #ifdef MEMIF_DBG_SHM
185 static void
186 print_bytes (void *data, uint16_t len, uint8_t q)
187 {
188  if (q == DBG_TX_BUF)
189  printf ("\nTX:\n\t");
190  else
191  printf ("\nRX:\n\t");
192  int i;
193  for (i = 0; i < len; i++)
194  {
195  if (i % 8 == 0)
196  printf ("\n%d:\t", i);
197  printf ("%02X ", ((uint8_t *) (data))[i]);
198  }
199  printf ("\n\n");
200 }
201 #endif /* MEMIF_DBG_SHM */
202 
203 int
205 {
206  DBG ("%s", strerror (err_code));
207 
208  if (err_code == 0)
209  return MEMIF_ERR_SUCCESS;
210  if (err_code == EACCES)
211  return MEMIF_ERR_ACCES;
212  if (err_code == ENFILE)
213  return MEMIF_ERR_FILE_LIMIT;
214  if (err_code == EMFILE)
216  if (err_code == ENOMEM)
217  return MEMIF_ERR_NOMEM;
218 /* connection refused if master does not exist
219  this error would spam the user until master was created */
220 /*
221  if (err_code == ECONNREFUSED)
222  return MEMIF_ERR_SUCCESS;
223 */
224  if (err_code == ECONNREFUSED)
225  return MEMIF_ERR_CONNREFUSED;
226  if (err_code == EALREADY)
227  return MEMIF_ERR_ALREADY;
228  if (err_code == EAGAIN)
229  return MEMIF_ERR_AGAIN;
230  if (err_code == EBADF)
231  return MEMIF_ERR_BAD_FD;
232  if (err_code == ENOENT)
233  return MEMIF_ERR_NO_FILE;
234 
235  /* other syscall errors */
236  return MEMIF_ERR_SYSCALL;
237 }
238 
239 static int
240 memif_add_epoll_fd (int fd, uint32_t events)
241 {
242  if (fd < 0)
243  {
244  DBG ("invalid fd %d", fd);
245  return -1;
246  }
247  struct epoll_event evt;
248  memset (&evt, 0, sizeof (evt));
249  evt.events = events;
250  evt.data.fd = fd;
251  if (epoll_ctl (memif_epfd, EPOLL_CTL_ADD, fd, &evt) < 0)
252  {
253  DBG ("epoll_ctl: %s fd %d", strerror (errno), fd);
254  return -1;
255  }
256  DBG ("fd %d added to epoll", fd);
257  return 0;
258 }
259 
260 static int
261 memif_mod_epoll_fd (int fd, uint32_t events)
262 {
263  if (fd < 0)
264  {
265  DBG ("invalid fd %d", fd);
266  return -1;
267  }
268  struct epoll_event evt;
269  memset (&evt, 0, sizeof (evt));
270  evt.events = events;
271  evt.data.fd = fd;
272  if (epoll_ctl (memif_epfd, EPOLL_CTL_MOD, fd, &evt) < 0)
273  {
274  DBG ("epoll_ctl: %s fd %d", strerror (errno), fd);
275  return -1;
276  }
277  DBG ("fd %d moddified on epoll", fd);
278  return 0;
279 }
280 
281 static int
283 {
284  if (fd < 0)
285  {
286  DBG ("invalid fd %d", fd);
287  return -1;
288  }
289  struct epoll_event evt;
290  memset (&evt, 0, sizeof (evt));
291  if (epoll_ctl (memif_epfd, EPOLL_CTL_DEL, fd, &evt) < 0)
292  {
293  DBG ("epoll_ctl: %s fd %d", strerror (errno), fd);
294  return -1;
295  }
296  DBG ("fd %d removed from epoll", fd);
297  return 0;
298 }
299 
300 int
301 memif_control_fd_update (int fd, uint8_t events)
302 {
303  if (events & MEMIF_FD_EVENT_DEL)
304  return memif_del_epoll_fd (fd);
305 
306  uint32_t evt = 0;
307  if (events & MEMIF_FD_EVENT_READ)
308  evt |= EPOLLIN;
309  if (events & MEMIF_FD_EVENT_WRITE)
310  evt |= EPOLLOUT;
311 
312  if (events & MEMIF_FD_EVENT_MOD)
313  return memif_mod_epoll_fd (fd, evt);
314 
315  return memif_add_epoll_fd (fd, evt);
316 }
317 
318 int
320 {
322 
323  int i;
324  for (i = 0; i < *len; i++)
325  {
326  if ((*list)[i].data_struct == NULL)
327  {
328  (*list)[i].key = e->key;
329  (*list)[i].data_struct = e->data_struct;
330  return i;
331  }
332  }
333  memif_list_elt_t *tmp;
334  tmp = lm->realloc (*list, sizeof (memif_list_elt_t) * *len * 2);
335  if (tmp == NULL)
336  return -1;
337 
338  for (i = *len; i < *len * 2; i++)
339  {
340  tmp[i].key = -1;
341  tmp[i].data_struct = NULL;
342  }
343 
344  tmp[*len].key = e->key;
345  tmp[*len].data_struct = e->data_struct;
346  i = *len;
347  *len = *len * 2;
348  *list = tmp;
349 
350  return i;
351 }
352 
353 int
355  int key)
356 {
357  if (key == -1)
358  {
359  *e = NULL;
360  return -1;
361  }
362  int i;
363  for (i = 0; i < len; i++)
364  {
365  if (list[i].key == key)
366  {
367  *e = &list[i];
368  return 0;
369  }
370  }
371  *e = NULL;
372  return -1;
373 }
374 
375 /* does not free memory, only marks element as free */
376 int
377 free_list_elt (memif_list_elt_t * list, uint16_t len, int key)
378 {
379  int i;
380  for (i = 0; i < len; i++)
381  {
382  if (list[i].key == key)
383  {
384  list[i].key = -1;
385  list[i].data_struct = NULL;
386  return 0;
387  }
388  }
389 
390  return -1;
391 }
392 
393 int
396 {
397  int i;
398  for (i = 0; i < len; i++)
399  {
400  if (list[i].key == -1)
401  {
402  if (list[i].data_struct == ctx)
403  {
404  list[i].data_struct = NULL;
405  return 0;
406  }
407  }
408  }
409 
410  return -1;
411 }
412 
413 static void
415 {
417  lm->control_fd_update = cb;
418 }
419 
420 void
425 {
427  lm->add_external_region = ar;
428  lm->get_external_region_addr = gr;
429  lm->del_external_region = dr;
431 }
432 
433 static void
435 {
437  lm->alloc = ma;
438 }
439 
440 static void
442 {
444  lm->realloc = mr;
445 }
446 
447 static void
449 {
451  lm->free = mf;
452 }
453 
454 int
455 memif_init (memif_control_fd_update_t * on_control_fd_update, char *app_name,
456  memif_alloc_t * memif_alloc, memif_realloc_t * memif_realloc,
457  memif_free_t * memif_free)
458 {
459  int err = MEMIF_ERR_SUCCESS; /* 0 */
461  memset (lm, 0, sizeof (libmemif_main_t));
462 
463  if (memif_alloc != NULL)
464  {
465  memif_alloc_register (memif_alloc);
466  }
467  else
468  memif_alloc_register (malloc);
469 
470  if (memif_realloc != NULL)
471  {
472  memif_realloc_register (memif_realloc);
473  }
474  else
475  memif_realloc_register (realloc);
476 
477  if (memif_free != NULL)
478  memif_free_register (memif_free);
479  else
480  memif_free_register (free);
481 
482  if (app_name != NULL)
483  {
484  uint8_t len = (strlen (app_name) > MEMIF_NAME_LEN)
485  ? strlen (app_name) : MEMIF_NAME_LEN;
486  strncpy ((char *) lm->app_name, app_name, len);
487  }
488  else
489  {
490  strncpy ((char *) lm->app_name, MEMIF_DEFAULT_APP_NAME,
491  strlen (MEMIF_DEFAULT_APP_NAME));
492  }
493 
494  /* register control fd update callback */
495  if (on_control_fd_update != NULL)
496  memif_control_fd_update_register (on_control_fd_update);
497  else
498  {
499  memif_epfd = epoll_create (1);
501  if ((poll_cancel_fd = eventfd (0, EFD_NONBLOCK)) < 0)
502  {
503  err = errno;
504  DBG ("eventfd: %s", strerror (err));
505  return memif_syscall_error_handler (err);
506  }
508  DBG ("libmemif event polling initialized");
509  }
510 
511  lm->control_list_len = 2;
512  lm->interrupt_list_len = 2;
513  lm->listener_list_len = 1;
514  lm->pending_list_len = 1;
515 
516  lm->control_list =
517  lm->alloc (sizeof (memif_list_elt_t) * lm->control_list_len);
518  if (lm->control_list == NULL)
519  {
520  err = MEMIF_ERR_NOMEM;
521  goto error;
522  }
523  lm->interrupt_list =
524  lm->alloc (sizeof (memif_list_elt_t) * lm->interrupt_list_len);
525  if (lm->interrupt_list == NULL)
526  {
527  err = MEMIF_ERR_NOMEM;
528  goto error;
529  }
530  lm->listener_list =
531  lm->alloc (sizeof (memif_list_elt_t) * lm->listener_list_len);
532  if (lm->listener_list == NULL)
533  {
534  err = MEMIF_ERR_NOMEM;
535  goto error;
536  }
537  lm->pending_list =
538  lm->alloc (sizeof (memif_list_elt_t) * lm->pending_list_len);
539  if (lm->pending_list == NULL)
540  {
541  err = MEMIF_ERR_NOMEM;
542  goto error;
543  }
544 
545  int i;
546  for (i = 0; i < lm->control_list_len; i++)
547  {
548  lm->control_list[i].key = -1;
550  }
551  for (i = 0; i < lm->interrupt_list_len; i++)
552  {
553  lm->interrupt_list[i].key = -1;
555  }
556  for (i = 0; i < lm->listener_list_len; i++)
557  {
558  lm->listener_list[i].key = -1;
560  }
561  for (i = 0; i < lm->pending_list_len; i++)
562  {
563  lm->pending_list[i].key = -1;
565  }
566 
567  lm->disconn_slaves = 0;
568 
569  lm->timerfd = timerfd_create (CLOCK_REALTIME, TFD_NONBLOCK);
570  if (lm->timerfd < 0)
571  {
572  err = memif_syscall_error_handler (errno);
573  goto error;
574  }
575 
576  lm->arm.it_value.tv_sec = 2;
577  lm->arm.it_value.tv_nsec = 0;
578  lm->arm.it_interval.tv_sec = 2;
579  lm->arm.it_interval.tv_nsec = 0;
580 
581  if (lm->control_fd_update (lm->timerfd, MEMIF_FD_EVENT_READ) < 0)
582  {
583  DBG ("callback type memif_control_fd_update_t error!");
584  err = MEMIF_ERR_CB_FDUPDATE;
585  goto error;
586  }
587 
588  return err;
589 
590 error:
591  memif_cleanup ();
592  return err;
593 }
594 
595 static inline memif_ring_t *
597  uint16_t ring_num)
598 {
599  if (&conn->regions[0] == NULL)
600  return NULL;
601  void *p = conn->regions[0].addr;
602  int ring_size =
603  sizeof (memif_ring_t) +
604  sizeof (memif_desc_t) * (1 << conn->run_args.log2_ring_size);
605  p += (ring_num + type * conn->run_args.num_s2m_rings) * ring_size;
606 
607  return (memif_ring_t *) p;
608 }
609 
610 int
612  uint16_t qid)
613 {
615  if (conn == NULL)
616  return MEMIF_ERR_NOCONN;
617  uint8_t num =
618  (conn->args.is_master) ? conn->run_args.num_s2m_rings : conn->
619  run_args.num_m2s_rings;
620  if (qid >= num)
621  return MEMIF_ERR_QID;
622 
623  conn->rx_queues[qid].ring->flags = rx_mode;
624  DBG ("rx_mode flag: %u", conn->rx_queues[qid].ring->flags);
625  return MEMIF_ERR_SUCCESS;
626 }
627 
628 int
632  memif_interrupt_t * on_interrupt, void *private_ctx)
633 {
635  int err, i, index = 0, sockfd = -1;
636  memif_list_elt_t list_elt;
637  memif_connection_t *conn = (memif_connection_t *) * c;
638  if (conn != NULL)
639  {
640  DBG ("This handle already points to existing memif.");
641  return MEMIF_ERR_CONN;
642  }
643  conn = (memif_connection_t *) lm->alloc (sizeof (memif_connection_t));
644  if (conn == NULL)
645  {
646  err = MEMIF_ERR_NOMEM;
647  goto error;
648  }
649  memset (conn, 0, sizeof (memif_connection_t));
650 
651  conn->args.interface_id = args->interface_id;
652 
653  if (args->log2_ring_size == 0)
655  else if (args->log2_ring_size > MEMIF_MAX_LOG2_RING_SIZE)
656  {
657  err = MEMIF_ERR_MAX_RING;
658  goto error;
659  }
660  if (args->buffer_size == 0)
662  if (args->num_s2m_rings == 0)
664  if (args->num_m2s_rings == 0)
666 
667  conn->args.num_s2m_rings = args->num_s2m_rings;
668  conn->args.num_m2s_rings = args->num_m2s_rings;
669  conn->args.buffer_size = args->buffer_size;
670  conn->args.log2_ring_size = args->log2_ring_size;
671  conn->args.is_master = args->is_master;
672  conn->args.mode = args->mode;
673  conn->msg_queue = NULL;
674  conn->regions = NULL;
675  conn->tx_queues = NULL;
676  conn->rx_queues = NULL;
677  conn->fd = -1;
678  conn->on_connect = on_connect;
679  conn->on_disconnect = on_disconnect;
680  conn->on_interrupt = on_interrupt;
681  conn->private_ctx = private_ctx;
682  memset (&conn->run_args, 0, sizeof (memif_conn_run_args_t));
683 
684  uint8_t l = strlen ((char *) args->interface_name);
685  strncpy ((char *) conn->args.interface_name, (char *) args->interface_name,
686  l);
687 
688  /* allocate and initialize socket_filename so it can be copyed to sun_path
689  without memory leaks */
690  conn->args.socket_filename = lm->alloc (sizeof (char *) * 108);
691  if (conn->args.socket_filename == NULL)
692  {
693  err = MEMIF_ERR_NOMEM;
694  goto error;
695  }
696  memset (conn->args.socket_filename, 0, 108 * sizeof (char *));
697 
698  if (args->socket_filename)
699  {
700  if (conn->args.socket_filename == NULL)
701  {
702  err = memif_syscall_error_handler (errno);
703  goto error;
704  }
705  strncpy ((char *) conn->args.socket_filename,
706  (char *) args->socket_filename,
707  strlen ((char *) args->socket_filename));
708  }
709  else
710  {
711  uint16_t sdl = strlen (MEMIF_DEFAULT_SOCKET_DIR);
712  uint16_t sfl = strlen (MEMIF_DEFAULT_SOCKET_FILENAME);
713  if (conn->args.socket_filename == NULL)
714  {
715  err = memif_syscall_error_handler (errno);
716  goto error;
717  }
718  strncpy ((char *) conn->args.socket_filename,
720  conn->args.socket_filename[sdl] = '/';
721  strncpy ((char *) (conn->args.socket_filename + 1 + sdl),
723  }
724 
725  if ((l = strlen ((char *) args->secret)) > 0)
726  {
727  strncpy ((char *) conn->args.secret, (char *) args->secret, l);
728  }
729 
730  if (conn->args.is_master)
731  {
732  conn->run_args.buffer_size = conn->args.buffer_size;
733  memif_socket_t *ms;
734  memif_list_elt_t elt;
735  for (i = 0; i < lm->listener_list_len; i++)
736  {
737  if ((ms =
739  {
740  if (strncmp
741  ((char *) ms->filename, (char *) conn->args.socket_filename,
742  strlen ((char *) ms->filename)) == 0)
743  {
744  /* add interface to listener socket */
745  elt.key = conn->args.interface_id;
746  *c = elt.data_struct = conn;
747  add_list_elt (&elt, &ms->interface_list,
748  &ms->interface_list_len);
749  ms->use_count++;
750  conn->listener_fd = ms->fd;
751  break;
752  }
753  }
754  else
755  {
756  struct stat file_stat;
757  if (stat ((char *) conn->args.socket_filename, &file_stat) == 0)
758  {
759  if (S_ISSOCK (file_stat.st_mode))
760  unlink ((char *) conn->args.socket_filename);
761  else
762  return memif_syscall_error_handler (errno);
763  }
764  DBG ("creating socket file");
765  ms = lm->alloc (sizeof (memif_socket_t));
766  if (ms == NULL)
767  {
768  err = MEMIF_ERR_NOMEM;
769  goto error;
770  }
771  ms->filename =
772  lm->alloc (strlen ((char *) conn->args.socket_filename) +
773  sizeof (char));
774  if (ms->filename == NULL)
775  {
776  err = MEMIF_ERR_NOMEM;
777  goto error;
778  }
779  memset (ms->filename, 0,
780  strlen ((char *) conn->args.socket_filename) +
781  sizeof (char));
782  strncpy ((char *) ms->filename,
783  (char *) conn->args.socket_filename,
784  strlen ((char *) conn->args.socket_filename));
785  ms->interface_list_len = 1;
786  ms->interface_list =
787  lm->alloc (sizeof (memif_list_elt_t) *
788  ms->interface_list_len);
789  if (ms->interface_list == NULL)
790  {
791  err = MEMIF_ERR_NOMEM;
792  goto error;
793  }
794  ms->interface_list[0].key = -1;
796  struct sockaddr_un un = { 0 };
797  int on = 1;
798 
799  ms->fd = socket (AF_UNIX, SOCK_SEQPACKET, 0);
800  if (ms->fd < 0)
801  {
802  err = memif_syscall_error_handler (errno);
803  goto error;
804  }
805  DBG ("socket %d created", ms->fd);
806  un.sun_family = AF_UNIX;
807  strncpy ((char *) un.sun_path, (char *) ms->filename,
808  sizeof (un.sun_path) - 1);
809  DBG ("sockopt");
810  if (setsockopt
811  (ms->fd, SOL_SOCKET, SO_PASSCRED, &on, sizeof (on)) < 0)
812  {
813  err = memif_syscall_error_handler (errno);
814  goto error;
815  }
816  DBG ("bind");
817  if (bind (ms->fd, (struct sockaddr *) &un, sizeof (un)) < 0)
818  {
819  err = memif_syscall_error_handler (errno);
820  goto error;
821  }
822  DBG ("listen");
823  if (listen (ms->fd, 1) < 0)
824  {
825  err = memif_syscall_error_handler (errno);
826  goto error;
827  }
828  DBG ("stat");
829  if (stat ((char *) ms->filename, &file_stat) < 0)
830  {
831  err = memif_syscall_error_handler (errno);
832  goto error;
833  }
834 
835  /* add interface to listener socket */
836  elt.key = conn->args.interface_id;
837  *c = elt.data_struct = conn;
838  add_list_elt (&elt, &ms->interface_list,
839  &ms->interface_list_len);
840  ms->use_count = 1;
841  conn->listener_fd = ms->fd;
842 
843  /* add listener socket to libmemif main */
844  elt.key = ms->fd;
845  elt.data_struct = ms;
846  add_list_elt (&elt, &lm->listener_list, &lm->listener_list_len);
848  break;
849  }
850  }
851  }
852  else
853  {
854  if (lm->disconn_slaves == 0)
855  {
856  if (timerfd_settime (lm->timerfd, 0, &lm->arm, NULL) < 0)
857  {
858  err = memif_syscall_error_handler (errno);
859  goto error;
860  }
861  }
862 
863  lm->disconn_slaves++;
864 
865  list_elt.key = -1;
866  *c = list_elt.data_struct = conn;
867  if ((index =
868  add_list_elt (&list_elt, &lm->control_list,
869  &lm->control_list_len)) < 0)
870  {
871  err = MEMIF_ERR_NOMEM;
872  goto error;
873  }
874  }
875 
876  conn->index = index;
877 
878  return 0;
879 
880 error:
881  if (sockfd > 0)
882  close (sockfd);
883  sockfd = -1;
884  if (conn->args.socket_filename)
885  lm->free (conn->args.socket_filename);
886  if (conn != NULL)
887  lm->free (conn);
888  *c = conn = NULL;
889  return err;
890 }
891 
892 int
893 memif_control_fd_handler (int fd, uint8_t events)
894 {
895  int i, sockfd = -1, err = MEMIF_ERR_SUCCESS; /* 0 */
896  uint16_t num;
897  memif_list_elt_t *e = NULL;
898  memif_connection_t *conn;
900  if (fd == lm->timerfd)
901  {
902  uint64_t b;
903  ssize_t size;
904  size = read (fd, &b, sizeof (b));
905 
906  if (size == -1)
907  goto error;
908 
909  for (i = 0; i < lm->control_list_len; i++)
910  {
911  if ((lm->control_list[i].key < 0)
912  && (lm->control_list[i].data_struct != NULL))
913  {
914  conn = lm->control_list[i].data_struct;
915  if (conn->args.is_master)
916  continue;
917 
918  struct sockaddr_un sun;
919  sockfd = socket (AF_UNIX, SOCK_SEQPACKET, 0);
920  if (sockfd < 0)
921  {
922  err = memif_syscall_error_handler (errno);
923  goto error;
924  }
925 
926  sun.sun_family = AF_UNIX;
927 
928  strncpy (sun.sun_path, (char *) conn->args.socket_filename,
929  sizeof (sun.sun_path) - 1);
930 
931  if (connect (sockfd, (struct sockaddr *) &sun,
932  sizeof (struct sockaddr_un)) == 0)
933  {
934  conn->fd = sockfd;
935  conn->read_fn = memif_conn_fd_read_ready;
936  conn->write_fn = memif_conn_fd_write_ready;
937  conn->error_fn = memif_conn_fd_error;
938 
939  lm->control_list[conn->index].key = conn->fd;
940 
941  lm->control_fd_update (sockfd,
944 
945  lm->disconn_slaves--;
946  if (lm->disconn_slaves == 0)
947  {
948  if (timerfd_settime (lm->timerfd, 0, &lm->disarm, NULL)
949  < 0)
950  {
951  err = memif_syscall_error_handler (errno);
952  goto error;
953  }
954  }
955  }
956  else
957  {
958  strcpy ((char *) conn->remote_disconnect_string,
960  (errno)));
961  }
962  }
963  }
964  }
965  else
966  {
968  if (e != NULL)
969  {
970  if (((memif_connection_t *) e->data_struct)->on_interrupt != NULL)
971  {
972  num =
973  (((memif_connection_t *) e->data_struct)->
974  args.is_master) ? ((memif_connection_t *) e->
975  data_struct)->run_args.
976  num_s2m_rings : ((memif_connection_t *) e->data_struct)->
977  run_args.num_m2s_rings;
978  for (i = 0; i < num; i++)
979  {
980  if (((memif_connection_t *) e->data_struct)->
981  rx_queues[i].int_fd == fd)
982  {
984  on_interrupt ((void *) e->data_struct,
985  ((memif_connection_t *) e->
986  data_struct)->private_ctx, i);
987  return MEMIF_ERR_SUCCESS;
988  }
989  }
990  }
991  return MEMIF_ERR_SUCCESS;
992  }
993  get_list_elt (&e, lm->listener_list, lm->listener_list_len, fd);
994  if (e != NULL)
995  {
997  return MEMIF_ERR_SUCCESS;
998  }
999 
1000  get_list_elt (&e, lm->pending_list, lm->pending_list_len, fd);
1001  if (e != NULL)
1002  {
1003  memif_read_ready (fd);
1004  return MEMIF_ERR_SUCCESS;
1005  }
1006 
1007  get_list_elt (&e, lm->control_list, lm->control_list_len, fd);
1008  if (e != NULL)
1009  {
1010  if (events & MEMIF_FD_EVENT_READ)
1011  {
1012  err =
1013  ((memif_connection_t *) e->data_struct)->
1014  read_fn (e->data_struct);
1015  if (err != MEMIF_ERR_SUCCESS)
1016  return err;
1017  }
1018  if (events & MEMIF_FD_EVENT_WRITE)
1019  {
1020  err =
1021  ((memif_connection_t *) e->data_struct)->
1022  write_fn (e->data_struct);
1023  if (err != MEMIF_ERR_SUCCESS)
1024  return err;
1025  }
1026  if (events & MEMIF_FD_EVENT_ERROR)
1027  {
1028  err =
1029  ((memif_connection_t *) e->data_struct)->
1030  error_fn (e->data_struct);
1031  if (err != MEMIF_ERR_SUCCESS)
1032  return err;
1033  }
1034  }
1035  }
1036 
1037  return MEMIF_ERR_SUCCESS; /* 0 */
1038 
1039 error:
1040  if (sockfd > 0)
1041  close (sockfd);
1042  sockfd = -1;
1043  return err;
1044 }
1045 
1046 int
1047 memif_poll_event (int timeout)
1048 {
1049  struct epoll_event evt;
1050  int en = 0, err = MEMIF_ERR_SUCCESS; /* 0 */
1051  uint32_t events = 0;
1052  uint64_t counter = 0;
1053  ssize_t r = 0;
1054  memset (&evt, 0, sizeof (evt));
1055  evt.events = EPOLLIN | EPOLLOUT;
1056  sigset_t sigset;
1057  sigemptyset (&sigset);
1058  en = epoll_pwait (memif_epfd, &evt, 1, timeout, &sigset);
1059  if (en < 0)
1060  {
1061  err = errno;
1062  DBG ("epoll_pwait: %s", strerror (err));
1063  return memif_syscall_error_handler (err);
1064  }
1065  if (en > 0)
1066  {
1067  if (evt.data.fd == poll_cancel_fd)
1068  {
1069  r = read (evt.data.fd, &counter, sizeof (counter));
1070  if (r == -1)
1071  return MEMIF_ERR_DISCONNECTED;
1072 
1073  return MEMIF_ERR_POLL_CANCEL;
1074  }
1075  if (evt.events & EPOLLIN)
1076  events |= MEMIF_FD_EVENT_READ;
1077  if (evt.events & EPOLLOUT)
1078  events |= MEMIF_FD_EVENT_WRITE;
1079  if (evt.events & EPOLLERR)
1080  events |= MEMIF_FD_EVENT_ERROR;
1081  err = memif_control_fd_handler (evt.data.fd, events);
1082  return err;
1083  }
1084  return 0;
1085 }
1086 
1087 int
1089 {
1090  uint64_t counter = 1;
1091  ssize_t w = 0;
1092 
1093  if (poll_cancel_fd == -1)
1094  return 0;
1095  w = write (poll_cancel_fd, &counter, sizeof (counter));
1096  if (w < sizeof (counter))
1097  return MEMIF_ERR_INT_WRITE;
1098 
1099  return 0;
1100 }
1101 
1102 static void
1104 {
1105  if (*e == NULL)
1106  return;
1107  memif_msg_queue_free (lm, &(*e)->next);
1108  lm->free (*e);
1109  *e = NULL;
1110  return;
1111 }
1112 
1113 /* send disconnect msg and close interface */
1114 int
1116 {
1117  if (c == NULL)
1118  {
1119  DBG ("no connection");
1120  return MEMIF_ERR_NOCONN;
1121  }
1122  uint16_t num;
1123  int err = MEMIF_ERR_SUCCESS, i; /* 0 */
1124  memif_queue_t *mq;
1126  memif_list_elt_t *e;
1127 
1128  c->on_disconnect ((void *) c, c->private_ctx);
1129 
1130  if (c->fd > 0)
1131  {
1132  memif_msg_send_disconnect (c->fd, (uint8_t *) "interface deleted", 0);
1134  close (c->fd);
1135  }
1136  get_list_elt (&e, lm->control_list, lm->control_list_len, c->fd);
1137  if (e != NULL)
1138  {
1139  if (c->args.is_master)
1140  free_list_elt (lm->control_list, lm->control_list_len, c->fd);
1141  e->key = c->fd = -1;
1142  }
1143 
1144  if (c->tx_queues != NULL)
1145  {
1146  num =
1147  (c->args.is_master) ? c->run_args.num_m2s_rings : c->
1148  run_args.num_s2m_rings;
1149  for (i = 0; i < num; i++)
1150  {
1151  mq = &c->tx_queues[i];
1152  if (mq != NULL)
1153  {
1154  if (mq->int_fd > 0)
1155  close (mq->int_fd);
1157  mq->int_fd);
1158  mq->int_fd = -1;
1159  }
1160  }
1161  lm->free (c->tx_queues);
1162  c->tx_queues = NULL;
1163  }
1164 
1165  if (c->rx_queues != NULL)
1166  {
1167  num =
1168  (c->args.is_master) ? c->run_args.num_s2m_rings : c->
1169  run_args.num_m2s_rings;
1170  for (i = 0; i < num; i++)
1171  {
1172  mq = &c->rx_queues[i];
1173  if (mq != NULL)
1174  {
1175  if (mq->int_fd > 0)
1176  {
1177  if (c->on_interrupt != NULL)
1179  close (mq->int_fd);
1180  }
1182  mq->int_fd);
1183  mq->int_fd = -1;
1184  }
1185  }
1186  lm->free (c->rx_queues);
1187  c->rx_queues = NULL;
1188  }
1189 
1190  for (i = 0; i < c->regions_num; i++)
1191  {
1192  if (&c->regions[i] == NULL)
1193  continue;
1194  if (c->regions[i].is_external != 0)
1195  {
1196  lm->del_external_region (c->regions[i].addr,
1197  c->regions[i].region_size,
1198  c->regions[i].fd, c->private_ctx);
1199  }
1200  else
1201  {
1202  if (munmap (c->regions[i].addr, c->regions[i].region_size) < 0)
1203  return memif_syscall_error_handler (errno);
1204  if (c->regions[i].fd > 0)
1205  close (c->regions[i].fd);
1206  c->regions[i].fd = -1;
1207  }
1208  }
1209  lm->free (c->regions);
1210  c->regions = NULL;
1211  c->regions_num = 0;
1212 
1213  memset (&c->run_args, 0, sizeof (memif_conn_run_args_t));
1214 
1215  memif_msg_queue_free (lm, &c->msg_queue);
1216 
1217  if (!(c->args.is_master))
1218  {
1219  if (lm->disconn_slaves == 0)
1220  {
1221  if (timerfd_settime (lm->timerfd, 0, &lm->arm, NULL) < 0)
1222  {
1223  err = memif_syscall_error_handler (errno);
1224  DBG ("timerfd_settime: arm");
1225  }
1226  }
1227  lm->disconn_slaves++;
1228  }
1229 
1230  return err;
1231 }
1232 
1233 int
1235 {
1236  memif_connection_t *c = (memif_connection_t *) * conn;
1237  if (c == NULL)
1238  {
1239  DBG ("no connection");
1240  return MEMIF_ERR_NOCONN;
1241  }
1243  memif_list_elt_t *e = NULL;
1244  memif_socket_t *ms = NULL;
1245 
1246  int err = MEMIF_ERR_SUCCESS;
1247 
1248  if (c->fd > 0)
1249  {
1250  DBG ("DISCONNECTING");
1251  err = memif_disconnect_internal (c);
1252  if (err == MEMIF_ERR_NOCONN)
1253  return err;
1254  }
1255 
1257 
1258  if (c->args.is_master)
1259  {
1261  c->listener_fd);
1262  if (e != NULL)
1263  {
1264  ms = (memif_socket_t *) e->data_struct;
1265  ms->use_count--;
1267  c->args.interface_id);
1268  if (ms->use_count <= 0)
1269  {
1270  lm->control_fd_update (c->listener_fd, MEMIF_FD_EVENT_DEL);
1272  c->listener_fd);
1273  close (c->listener_fd);
1274  c->listener_fd = ms->fd = -1;
1275  lm->free (ms->interface_list);
1276  ms->interface_list = NULL;
1277  lm->free (ms->filename);
1278  ms->filename = NULL;
1279  lm->free (ms);
1280  ms = NULL;
1281  }
1282  }
1283  }
1284  else
1285  {
1286  lm->disconn_slaves--;
1287  if (lm->disconn_slaves <= 0)
1288  {
1289  if (timerfd_settime (lm->timerfd, 0, &lm->disarm, NULL) < 0)
1290  {
1291  err = memif_syscall_error_handler (errno);
1292  DBG ("timerfd_settime: disarm");
1293  }
1294  }
1295  }
1296 
1297  if (c->args.socket_filename)
1298  lm->free (c->args.socket_filename);
1299  c->args.socket_filename = NULL;
1300 
1301  lm->free (c);
1302  c = NULL;
1303 
1304  *conn = c;
1305  return err;
1306 }
1307 
1308 int
1310 {
1312  memif_region_t *mr;
1313  memif_queue_t *mq;
1314  int i;
1315 
1316  for (i = 0; i < c->regions_num; i++)
1317  {
1318  mr = &c->regions[i];
1319  if (mr != NULL)
1320  {
1321  if (!mr->addr)
1322  {
1323  if (mr->is_external)
1324  {
1325  if (lm->get_external_region_addr == NULL)
1326  return 99; /* FIXME: proper error report */
1327  mr->addr =
1328  lm->get_external_region_addr (mr->region_size, mr->fd,
1329  c->private_ctx);
1330  }
1331  else
1332  {
1333  if (mr->fd < 0)
1334  return MEMIF_ERR_NO_SHMFD;
1335 
1336  if ((mr->addr =
1337  mmap (NULL, mr->region_size, PROT_READ | PROT_WRITE,
1338  MAP_SHARED, mr->fd, 0)) == MAP_FAILED)
1339  {
1340  return memif_syscall_error_handler (errno);
1341  }
1342  }
1343  }
1344  }
1345  }
1346 
1347  for (i = 0; i < c->rx_queues_num; i++)
1348  {
1349  mq = &c->rx_queues[i];
1350  if (mq != NULL)
1351  {
1352  mq->ring = c->regions[mq->region].addr + mq->offset;
1353  if (mq->ring->cookie != MEMIF_COOKIE)
1354  {
1355  DBG ("wrong cookie on rx ring %u", i);
1356  return MEMIF_ERR_COOKIE;
1357  }
1358  mq->ring->head = mq->ring->tail = mq->last_head = mq->alloc_bufs =
1359  0;
1360  }
1361  }
1362 
1363  for (i = 0; i < c->tx_queues_num; i++)
1364  {
1365  mq = &c->tx_queues[i];
1366  if (mq != NULL)
1367  {
1368  mq->ring = c->regions[mq->region].addr + mq->offset;
1369  if (mq->ring->cookie != MEMIF_COOKIE)
1370  {
1371  DBG ("wrong cookie on tx ring %u", i);
1372  return MEMIF_ERR_COOKIE;
1373  }
1374  mq->ring->head = mq->ring->tail = mq->last_head = mq->alloc_bufs =
1375  0;
1376  }
1377  }
1378 
1380 
1381  return 0;
1382 }
1383 
1384 static inline int
1386  uint8_t has_buffers)
1387 {
1388  memif_region_t *r;
1389 
1390  r =
1391  lm->realloc (conn->regions,
1392  sizeof (memif_region_t) * ++conn->regions_num);
1393  if (r == NULL)
1394  return MEMIF_ERR_NOMEM;
1395 
1396  conn->regions = r;
1397  r = &conn->regions[conn->regions_num - 1];
1398  memset (r, 0, sizeof (memif_region_t));
1399 
1400  if (has_buffers != 0)
1401  {
1402  r->buffer_offset = 0;
1403  }
1404  else
1405  {
1406  r->buffer_offset =
1407  (conn->run_args.num_s2m_rings +
1408  conn->run_args.num_m2s_rings) * (sizeof (memif_ring_t) +
1409  sizeof (memif_desc_t) *
1410  (1 << conn->
1411  run_args.log2_ring_size));
1412  }
1413 
1414  r->region_size = (has_buffers == 0) ? r->buffer_offset : r->buffer_offset +
1415  conn->run_args.buffer_size * (1 << conn->run_args.log2_ring_size) *
1416  (conn->run_args.num_s2m_rings + conn->run_args.num_m2s_rings);
1417 
1418  if ((r->fd = memfd_create ("memif region 0", MFD_ALLOW_SEALING)) == -1)
1419  return memif_syscall_error_handler (errno);
1420 
1421  if ((fcntl (r->fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1)
1422  return memif_syscall_error_handler (errno);
1423 
1424  if ((ftruncate (r->fd, r->region_size)) == -1)
1425  return memif_syscall_error_handler (errno);
1426 
1427  if ((r->addr = mmap (NULL, r->region_size, PROT_READ | PROT_WRITE,
1428  MAP_SHARED, r->fd, 0)) == MAP_FAILED)
1429  return memif_syscall_error_handler (errno);
1430 
1431  return MEMIF_ERR_SUCCESS;
1432 }
1433 
1434 static inline int
1436 {
1437  int i, j;
1438  memif_ring_t *ring;
1439 
1440  for (i = 0; i < conn->run_args.num_s2m_rings; i++)
1441  {
1442  ring = memif_get_ring (conn, MEMIF_RING_S2M, i);
1443  DBG ("RING: %p I: %d", ring, i);
1444  ring->head = ring->tail = 0;
1445  ring->cookie = MEMIF_COOKIE;
1446  ring->flags = 0;
1447  for (j = 0; j < (1 << conn->run_args.log2_ring_size); j++)
1448  {
1449  uint16_t slot = i * (1 << conn->run_args.log2_ring_size) + j;
1450  ring->desc[j].region = 1;
1451  ring->desc[j].offset =
1452  conn->regions[1].buffer_offset +
1453  (uint32_t) (slot * conn->run_args.buffer_size);
1454  ring->desc[j].length = conn->run_args.buffer_size;
1455  }
1456  }
1457  for (i = 0; i < conn->run_args.num_m2s_rings; i++)
1458  {
1459  ring = memif_get_ring (conn, MEMIF_RING_M2S, i);
1460  DBG ("RING: %p I: %d", ring, i);
1461  ring->head = ring->tail = 0;
1462  ring->cookie = MEMIF_COOKIE;
1463  ring->flags = 0;
1464  for (j = 0; j < (1 << conn->run_args.log2_ring_size); j++)
1465  {
1466  uint16_t slot = (i + conn->run_args.num_s2m_rings) *
1467  (1 << conn->run_args.log2_ring_size) + j;
1468  ring->desc[j].region = 1;
1469  ring->desc[j].offset =
1470  conn->regions[1].buffer_offset +
1471  (uint32_t) (slot * conn->run_args.buffer_size);
1472  ring->desc[j].length = conn->run_args.buffer_size;
1473  }
1474  }
1475  memif_queue_t *mq;
1476  DBG ("alloc: %p", lm->alloc);
1477  DBG ("size: %lu", sizeof (memif_queue_t) * conn->run_args.num_s2m_rings);
1478  mq =
1479  (memif_queue_t *) lm->alloc (sizeof (memif_queue_t) *
1480  conn->run_args.num_s2m_rings);
1481  if (mq == NULL)
1482  return MEMIF_ERR_NOMEM;
1483 
1484  int x;
1485  memif_list_elt_t e;
1486  for (x = 0; x < conn->run_args.num_s2m_rings; x++)
1487  {
1488  if ((mq[x].int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
1489  return memif_syscall_error_handler (errno);
1490  e.key = mq[x].int_fd;
1491  e.data_struct = conn;
1493 
1494  mq[x].ring = memif_get_ring (conn, MEMIF_RING_S2M, x);
1495  DBG ("RING: %p I: %d", mq[x].ring, x);
1496  mq[x].log2_ring_size = conn->run_args.log2_ring_size;
1497  mq[x].region = 0;
1498  mq[x].offset =
1499  (void *) mq[x].ring - (void *) conn->regions[mq->region].addr;
1500  mq[x].last_head = mq[x].last_tail = 0;
1501  mq[x].alloc_bufs = 0;
1502  }
1503  conn->tx_queues = mq;
1504 
1505  mq =
1506  (memif_queue_t *) lm->alloc (sizeof (memif_queue_t) *
1507  conn->run_args.num_m2s_rings);
1508  if (mq == NULL)
1509  return MEMIF_ERR_NOMEM;
1510 
1511  for (x = 0; x < conn->run_args.num_m2s_rings; x++)
1512  {
1513  if ((mq[x].int_fd = eventfd (0, EFD_NONBLOCK)) < 0)
1514  return memif_syscall_error_handler (errno);
1515  e.key = mq[x].int_fd;
1516  e.data_struct = conn;
1518 
1519  mq[x].ring = memif_get_ring (conn, MEMIF_RING_M2S, x);
1520  DBG ("RING: %p I: %d", mq[x].ring, x);
1521  mq[x].log2_ring_size = conn->run_args.log2_ring_size;
1522  mq[x].region = 0;
1523  mq[x].offset =
1524  (void *) mq[x].ring - (void *) conn->regions[mq->region].addr;
1525  mq[x].last_head = mq[x].last_tail = 0;
1526  mq[x].alloc_bufs = 0;
1527  }
1528  conn->rx_queues = mq;
1529 
1530  return MEMIF_ERR_SUCCESS;
1531 }
1532 
1533 int
1535 {
1536  memif_region_t *r;
1538 
1539  /* region 0. rings */
1540  memif_add_region (lm, conn, /* has_buffers */ 0);
1541 
1542  /* region 1. buffers */
1543  if (lm->add_external_region)
1544  {
1545  r =
1546  (memif_region_t *) lm->realloc (conn->regions,
1547  sizeof (memif_region_t) *
1548  ++conn->regions_num);
1549  if (r == NULL)
1550  return MEMIF_ERR_NOMEM;
1551  conn->regions = r;
1552 
1553  conn->regions[1].region_size =
1554  conn->run_args.buffer_size * (1 << conn->run_args.log2_ring_size) *
1555  (conn->run_args.num_s2m_rings + conn->run_args.num_m2s_rings);
1556  conn->regions[1].buffer_offset = 0;
1557  lm->add_external_region (&conn->regions[1].addr,
1558  conn->regions[1].region_size,
1559  &conn->regions[1].fd, conn->private_ctx);
1560  conn->regions[1].is_external = 1;
1561  }
1562  else
1563  {
1564  memif_add_region (lm, conn, 1);
1565  }
1566 
1567  memif_init_queues (lm, conn);
1568 
1569  return 0;
1570 }
1571 
1572 int
1574  memif_buffer_t * bufs, uint16_t count,
1575  uint16_t * count_out)
1576 {
1578  if (EXPECT_FALSE (c == NULL))
1579  return MEMIF_ERR_NOCONN;
1580  if (EXPECT_FALSE (c->fd < 0))
1581  return MEMIF_ERR_DISCONNECTED;
1582  uint8_t num =
1583  (c->args.is_master) ? c->run_args.num_m2s_rings : c->
1584  run_args.num_s2m_rings;
1585  if (EXPECT_FALSE (qid >= num))
1586  return MEMIF_ERR_QID;
1587  if (EXPECT_FALSE (!count_out))
1588  return MEMIF_ERR_INVAL_ARG;
1589  if (EXPECT_FALSE (c->args.is_master))
1590  return MEMIF_ERR_INVAL_ARG;
1591 
1592  memif_queue_t *mq = &c->tx_queues[qid];
1593  memif_ring_t *ring = mq->ring;
1594  memif_buffer_t *b0;
1595  uint16_t mask = (1 << mq->log2_ring_size) - 1;
1596  uint16_t ring_size;
1597  uint16_t slot, ns;
1598  int err = MEMIF_ERR_SUCCESS; /* 0 */
1599  *count_out = 0;
1600 
1601  ring_size = (1 << mq->log2_ring_size);
1602  slot = (c->args.is_master) ? ring->tail : ring->head;
1603  slot += mq->alloc_bufs;
1604 
1605  /* can only be called by slave */
1606  ns = ring_size - (ring->head + mq->alloc_bufs) + ring->tail;
1607 
1608  b0 = bufs;
1609 
1610  while (count && ns)
1611  {
1612  if (EXPECT_FALSE ((b0->flags & MEMIF_BUFFER_FLAG_RX) == 0))
1613  {
1614  /* not a valid buffer */
1615  count--;
1616  continue;
1617  }
1618  b0->flags &= ~MEMIF_BUFFER_FLAG_RX;
1619 
1620  ((memif_ring_t *) b0->ring)->desc[b0->desc_index & mask].offset = ring->desc[slot & mask].offset; /* put free buffer on rx ring */
1621 
1622  ring->desc[slot & mask].offset =
1623  (uint32_t) (b0->data -
1624  c->regions[ring->desc[slot & mask].region].addr);
1625  ring->desc[slot & mask].flags |=
1627 
1628  b0->desc_index = slot;
1629 
1630  mq->alloc_bufs++;
1631  slot++;
1632 
1633  count--;
1634  ns--;
1635  b0++;
1636  *count_out += 1;
1637  }
1638 
1639  DBG ("allocated: %u/%u bufs. Total %u allocated bufs", *count_out, count,
1640  mq->alloc_bufs);
1641 
1642  if (count)
1643  {
1644  DBG ("ring buffer full! qid: %u", qid);
1645  err = MEMIF_ERR_NOBUF_RING;
1646  }
1647 
1648  return err;
1649 }
1650 
1651 int
1653  memif_buffer_t * bufs, uint16_t count,
1654  uint16_t * count_out, uint16_t size)
1655 {
1657  if (EXPECT_FALSE (c == NULL))
1658  return MEMIF_ERR_NOCONN;
1659  if (EXPECT_FALSE (c->fd < 0))
1660  return MEMIF_ERR_DISCONNECTED;
1661  uint8_t num =
1662  (c->args.is_master) ? c->run_args.num_m2s_rings : c->
1663  run_args.num_s2m_rings;
1664  if (EXPECT_FALSE (qid >= num))
1665  return MEMIF_ERR_QID;
1666  if (EXPECT_FALSE (!count_out))
1667  return MEMIF_ERR_INVAL_ARG;
1668 
1670  memif_queue_t *mq = &c->tx_queues[qid];
1671  memif_ring_t *ring = mq->ring;
1672  memif_buffer_t *b0;
1673  uint16_t mask = (1 << mq->log2_ring_size) - 1;
1674  uint32_t offset_mask = c->run_args.buffer_size - 1;
1675  uint16_t ring_size;
1676  uint16_t slot, ns;
1677  int err = MEMIF_ERR_SUCCESS; /* 0 */
1678  uint16_t dst_left, src_left;
1679  uint16_t saved_count;
1680  memif_buffer_t *saved_b;
1681  *count_out = 0;
1682 
1683  ring_size = (1 << mq->log2_ring_size);
1684  slot = (c->args.is_master) ? ring->tail : ring->head;
1685  slot += mq->alloc_bufs;
1686 
1687  if (c->args.is_master)
1688  ns = ring->head - (ring->tail + mq->alloc_bufs);
1689  else
1690  ns = ring_size - (ring->head + mq->alloc_bufs) + ring->tail;
1691 
1692  while (count && ns)
1693  {
1694  b0 = (bufs + *count_out);
1695 
1696  saved_b = b0;
1697  saved_count = count;
1698 
1699  b0->desc_index = slot;
1700  ring->desc[slot & mask].flags = 0;
1701 
1702  /* slave can produce buffer with original length */
1703  dst_left = (c->args.is_master) ? ring->desc[slot & mask].length :
1704  c->run_args.buffer_size;
1705  src_left = size;
1706 
1707  while (src_left)
1708  {
1709  if (EXPECT_FALSE (dst_left == 0))
1710  {
1711  if (count && ns)
1712  {
1713  slot++;
1714  *count_out += 1;
1715  mq->alloc_bufs++;
1716  ns--;
1717 
1718  ring->desc[b0->desc_index & mask].flags |=
1721 
1722  b0 = (bufs + *count_out);
1723  b0->desc_index = slot;
1724  dst_left =
1725  (c->args.is_master) ? ring->desc[slot & mask].
1726  length : c->run_args.buffer_size;
1727  ring->desc[slot & mask].flags = 0;
1728  }
1729  else
1730  {
1731  /* rollback allocated chain buffers */
1732  memset (saved_b, 0, sizeof (memif_buffer_t)
1733  * (saved_count - count + 1));
1734  *count_out -= saved_count - count;
1735  mq->alloc_bufs = saved_count - count;
1736  goto no_ns;
1737  }
1738  }
1739  b0->len = memif_min (dst_left, src_left);
1740 
1741  /* slave resets buffer offset */
1742  if (c->args.is_master == 0)
1743  {
1744  memif_desc_t *d = &ring->desc[slot & mask];
1746  d->offset = lm->get_external_buffer_offset (c->private_ctx);
1747  else
1748  d->offset = d->offset - (d->offset & offset_mask);
1749  }
1750  b0->data = memif_get_buffer (c, ring, slot & mask);
1751 
1752  src_left -= b0->len;
1753  dst_left -= b0->len;
1754  }
1755 
1756  slot++;
1757  *count_out += 1;
1758  mq->alloc_bufs++;
1759  ns--;
1760  count--;
1761  }
1762 
1763 no_ns:
1764 
1765  DBG ("allocated: %u/%u bufs. Total %u allocated bufs", *count_out, count,
1766  mq->alloc_bufs);
1767 
1768  if (count)
1769  {
1770  DBG ("ring buffer full! qid: %u", qid);
1771  err = MEMIF_ERR_NOBUF_RING;
1772  }
1773 
1774  return err;
1775 }
1776 
1777 int
1778 memif_refill_queue (memif_conn_handle_t conn, uint16_t qid, uint16_t count,
1779  uint16_t headroom)
1780 {
1782  if (EXPECT_FALSE (c == NULL))
1783  return MEMIF_ERR_NOCONN;
1784  if (EXPECT_FALSE (c->fd < 0))
1785  return MEMIF_ERR_DISCONNECTED;
1786  uint8_t num =
1787  (c->args.is_master) ? c->run_args.num_s2m_rings : c->
1788  run_args.num_m2s_rings;
1789  if (EXPECT_FALSE (qid >= num))
1790  return MEMIF_ERR_QID;
1792  memif_queue_t *mq = &c->rx_queues[qid];
1793  memif_ring_t *ring = mq->ring;
1794  uint16_t mask = (1 << mq->log2_ring_size) - 1;
1795  uint32_t offset_mask = c->run_args.buffer_size - 1;
1796  uint16_t slot;
1797 
1798  if (c->args.is_master)
1799  {
1801  ring->tail =
1802  (ring->tail + count <=
1803  mq->last_head) ? ring->tail + count : mq->last_head;
1804  return MEMIF_ERR_SUCCESS;
1805  }
1806 
1807  uint16_t head = ring->head;
1808  uint16_t ns = (1 << mq->log2_ring_size) - head + mq->last_tail;
1809  head += (count < ns) ? count : ns;
1810 
1811  slot = ring->head;
1812  memif_desc_t *d;
1813  while (slot < head)
1814  {
1815  d = &ring->desc[slot & mask];
1816  d->region = 1;
1817  d->length = c->run_args.buffer_size - headroom;
1819  d->offset = lm->get_external_buffer_offset (c->private_ctx);
1820  else
1821  d->offset = d->offset - (d->offset & offset_mask) + headroom;
1822  slot++;
1823  }
1824 
1826  ring->head = head;
1827 
1828  return MEMIF_ERR_SUCCESS; /* 0 */
1829 }
1830 
1831 int
1833  memif_buffer_t * bufs, uint16_t count, uint16_t * tx)
1834 {
1836  if (EXPECT_FALSE (c == NULL))
1837  return MEMIF_ERR_NOCONN;
1838  if (EXPECT_FALSE (c->fd < 0))
1839  return MEMIF_ERR_DISCONNECTED;
1840  uint8_t num =
1841  (c->args.is_master) ? c->run_args.num_m2s_rings : c->
1842  run_args.num_s2m_rings;
1843  if (EXPECT_FALSE (qid >= num))
1844  return MEMIF_ERR_QID;
1845  if (EXPECT_FALSE (!tx))
1846  return MEMIF_ERR_INVAL_ARG;
1847 
1848  memif_queue_t *mq = &c->tx_queues[qid];
1849  memif_ring_t *ring = mq->ring;
1850  uint16_t mask = (1 << mq->log2_ring_size) - 1;
1851  memif_buffer_t *b0;
1852  *tx = 0;
1853 
1854  if (count > mq->alloc_bufs)
1855  count = mq->alloc_bufs;
1856 
1857  if (EXPECT_FALSE (count == 0))
1858  return MEMIF_ERR_SUCCESS;
1859 
1860  while (count)
1861  {
1862  b0 = (bufs + *tx);
1863  ring->desc[b0->desc_index & mask].length = b0->len;
1864 
1865 #ifdef MEMIF_DBG_SHM
1866  printf ("offset: %-6d\n", ring->desc[b0->desc_index & mask].offset);
1867  printf ("data: %p\n",
1868  memif_get_buffer (c, ring, b0->desc_index & mask));
1869  printf ("index: %u\n", b0->desc_index);
1870  print_bytes (memif_get_buffer (c, ring, b0->desc_index & mask),
1871  ring->desc[b0->desc_index & mask].length, DBG_TX_BUF);
1872 #endif /* MEMIF_DBG_SHM */
1873 
1874  *tx += 1;
1875  count--;
1876  }
1877 
1878 
1880  if (c->args.is_master)
1881  ring->tail = b0->desc_index + 1;
1882  else
1883  ring->head = b0->desc_index + 1;
1884 
1885  mq->alloc_bufs -= *tx;
1886 
1887  if ((ring->flags & MEMIF_RING_FLAG_MASK_INT) == 0)
1888  {
1889  uint64_t a = 1;
1890  int r = write (mq->int_fd, &a, sizeof (a));
1891  if (r < 0)
1892  return MEMIF_ERR_INT_WRITE;
1893  }
1894 
1895  return MEMIF_ERR_SUCCESS; /* 0 */
1896 }
1897 
1898 int
1900  memif_buffer_t * bufs, uint16_t count, uint16_t * rx)
1901 {
1903  if (EXPECT_FALSE (c == NULL))
1904  return MEMIF_ERR_NOCONN;
1905  if (EXPECT_FALSE (c->fd < 0))
1906  return MEMIF_ERR_DISCONNECTED;
1907  uint8_t num =
1908  (c->args.is_master) ? c->run_args.num_s2m_rings : c->
1909  run_args.num_m2s_rings;
1910  if (EXPECT_FALSE (qid >= num))
1911  return MEMIF_ERR_QID;
1912  if (EXPECT_FALSE (!rx))
1913  return MEMIF_ERR_INVAL_ARG;
1914 
1915  memif_queue_t *mq = &c->rx_queues[qid];
1916  memif_ring_t *ring = mq->ring;
1917  uint16_t cur_slot, last_slot;
1918  uint16_t ns;
1919  uint16_t mask = (1 << mq->log2_ring_size) - 1;
1920  memif_buffer_t *b0;
1921  *rx = 0;
1922 
1923  uint64_t b;
1924  ssize_t r = read (mq->int_fd, &b, sizeof (b));
1925  if (EXPECT_FALSE ((r == -1) && (errno != EAGAIN)))
1926  return memif_syscall_error_handler (errno);
1927 
1928  cur_slot = (c->args.is_master) ? mq->last_head : mq->last_tail;
1929  last_slot = (c->args.is_master) ? ring->head : ring->tail;
1930  if (cur_slot == last_slot)
1931  return MEMIF_ERR_SUCCESS;
1932 
1933  ns = last_slot - cur_slot;
1934 
1935  while (ns && count)
1936  {
1937  b0 = (bufs + *rx);
1938 
1939  b0->desc_index = cur_slot;
1940  b0->data = memif_get_buffer (c, ring, cur_slot & mask);
1941  b0->len = ring->desc[cur_slot & mask].length;
1942  /* slave resets buffer length */
1943  if (c->args.is_master == 0)
1944  {
1945  ring->desc[cur_slot & mask].length = c->run_args.buffer_size;
1946  }
1947 
1949  if (ring->desc[cur_slot & mask].flags & MEMIF_DESC_FLAG_NEXT)
1950  {
1952  ring->desc[cur_slot & mask].flags &= ~MEMIF_DESC_FLAG_NEXT;
1953  }
1954 /* b0->offset = ring->desc[cur_slot & mask].offset;*/
1955  b0->ring = ring;
1956 #ifdef MEMIF_DBG_SHM
1957  printf ("data: %p\n", b0->data);
1958  printf ("index: %u\n", b0->desc_index);
1959  printf ("ring: %p\n", b0->ring);
1960  print_bytes (b0->data, b0->len, DBG_RX_BUF);
1961 #endif /* MEMIF_DBG_SHM */
1962  ns--;
1963  *rx += 1;
1964 
1965  count--;
1966  cur_slot++;
1967  }
1968 
1969  if (c->args.is_master)
1970  mq->last_head = cur_slot;
1971  else
1972  mq->last_tail = cur_slot;
1973 
1974  if (ns)
1975  {
1976  DBG ("not enough buffers!");
1977  return MEMIF_ERR_NOBUF;
1978  }
1979 
1980  return MEMIF_ERR_SUCCESS; /* 0 */
1981 }
1982 
1983 int
1985  char *buf, ssize_t buflen)
1986 {
1989  if (c == NULL)
1990  return MEMIF_ERR_NOCONN;
1991 
1992  int err = MEMIF_ERR_SUCCESS, i;
1993  ssize_t l0, l1;
1994  l0 = 0;
1995 
1996  l1 = strlen ((char *) c->args.interface_name);
1997  if (l0 + l1 < buflen)
1998  {
1999  md->if_name =
2000  (uint8_t *) strcpy (buf + l0, (char *) c->args.interface_name);
2001  l0 += l1 + 1;
2002  }
2003  else
2004  err = MEMIF_ERR_NOBUF_DET;
2005 
2006  l1 = strlen ((char *) lm->app_name);
2007  if (l0 + l1 < buflen)
2008  {
2009  md->inst_name = (uint8_t *) strcpy (buf + l0, (char *) lm->app_name);
2010  l0 += l1 + 1;
2011  }
2012  else
2013  err = MEMIF_ERR_NOBUF_DET;
2014 
2015  l1 = strlen ((char *) c->remote_if_name);
2016  if (l0 + l1 < buflen)
2017  {
2018  md->remote_if_name =
2019  (uint8_t *) strcpy (buf + l0, (char *) c->remote_if_name);
2020  l0 += l1 + 1;
2021  }
2022  else
2023  err = MEMIF_ERR_NOBUF_DET;
2024 
2025  l1 = strlen ((char *) c->remote_name);
2026  if (l0 + l1 < buflen)
2027  {
2028  md->remote_inst_name =
2029  (uint8_t *) strcpy (buf + l0, (char *) c->remote_name);
2030  l0 += l1 + 1;
2031  }
2032  else
2033  err = MEMIF_ERR_NOBUF_DET;
2034 
2035  md->id = c->args.interface_id;
2036 
2037  if (strlen ((char *) c->args.secret) > 0)
2038  {
2039  l1 = strlen ((char *) c->args.secret);
2040  if (l0 + l1 < buflen)
2041  {
2042  md->secret = (uint8_t *) strcpy (buf + l0, (char *) c->args.secret);
2043  l0 += l1 + 1;
2044  }
2045  else
2046  err = MEMIF_ERR_NOBUF_DET;
2047  }
2048 
2049  md->role = (c->args.is_master) ? 0 : 1;
2050  md->mode = c->args.mode;
2051 
2052  l1 = strlen ((char *) c->args.socket_filename);
2053  if (l0 + l1 < buflen)
2054  {
2055  md->socket_filename =
2056  (uint8_t *) strcpy (buf + l0, (char *) c->args.socket_filename);
2057  l0 += l1 + 1;
2058  }
2059  else
2060  err = MEMIF_ERR_NOBUF_DET;
2061 
2062  l1 = strlen ((char *) c->remote_disconnect_string);
2063  if (l0 + l1 < buflen)
2064  {
2065  md->error =
2066  (uint8_t *) strcpy (buf + l0, (char *) c->remote_disconnect_string);
2067  l0 += l1 + 1;
2068  }
2069  else
2070  err = MEMIF_ERR_NOBUF_DET;
2071 
2072  md->regions_num = c->regions_num;
2073  l1 = sizeof (memif_region_details_t) * md->regions_num;
2074  if (l0 + l1 <= buflen)
2075  {
2076  md->regions = (memif_region_details_t *) (buf + l0);
2077  for (i = 0; i < md->regions_num; i++)
2078  {
2079  md->regions[i].index = i;
2080  md->regions[i].addr = c->regions[i].addr;
2081  md->regions[i].size = c->regions[i].region_size;
2082  md->regions[i].fd = c->regions[i].fd;
2083  md->regions[i].is_external = c->regions[i].is_external;
2084  }
2085  l0 += l1;
2086  }
2087  else
2088  err = MEMIF_ERR_NOBUF_DET;
2089 
2090  md->rx_queues_num =
2091  (c->args.is_master) ? c->run_args.num_s2m_rings : c->
2092  run_args.num_m2s_rings;
2093 
2094  l1 = sizeof (memif_queue_details_t) * md->rx_queues_num;
2095  if (l0 + l1 <= buflen)
2096  {
2097  md->rx_queues = (memif_queue_details_t *) (buf + l0);
2098  for (i = 0; i < md->rx_queues_num; i++)
2099  {
2100  md->rx_queues[i].region = c->rx_queues[i].region;
2101  md->rx_queues[i].qid = i;
2102  md->rx_queues[i].ring_size = (1 << c->rx_queues[i].log2_ring_size);
2103  md->rx_queues[i].flags = c->rx_queues[i].ring->flags;
2104  md->rx_queues[i].head = c->rx_queues[i].ring->head;
2105  md->rx_queues[i].tail = c->rx_queues[i].ring->tail;
2106  md->rx_queues[i].buffer_size = c->run_args.buffer_size;
2107  }
2108  l0 += l1;
2109  }
2110  else
2111  err = MEMIF_ERR_NOBUF_DET;
2112 
2113  md->tx_queues_num =
2114  (c->args.is_master) ? c->run_args.num_m2s_rings : c->
2115  run_args.num_s2m_rings;
2116 
2117  l1 = sizeof (memif_queue_details_t) * md->tx_queues_num;
2118  if (l0 + l1 <= buflen)
2119  {
2120  md->tx_queues = (memif_queue_details_t *) (buf + l0);
2121  for (i = 0; i < md->tx_queues_num; i++)
2122  {
2123  md->tx_queues[i].region = c->tx_queues[i].region;
2124  md->tx_queues[i].qid = i;
2125  md->tx_queues[i].ring_size = (1 << c->tx_queues[i].log2_ring_size);
2126  md->tx_queues[i].flags = c->tx_queues[i].ring->flags;
2127  md->tx_queues[i].head = c->tx_queues[i].ring->head;
2128  md->tx_queues[i].tail = c->tx_queues[i].ring->tail;
2129  md->tx_queues[i].buffer_size = c->run_args.buffer_size;
2130  }
2131  l0 += l1;
2132  }
2133  else
2134  err = MEMIF_ERR_NOBUF_DET;
2135 
2136  md->link_up_down = (c->fd > 0) ? 1 : 0;
2137 
2138  return err; /* 0 */
2139 }
2140 
2141 int
2142 memif_get_queue_efd (memif_conn_handle_t conn, uint16_t qid, int *efd)
2143 {
2145  *efd = -1;
2146  if (c == NULL)
2147  return MEMIF_ERR_NOCONN;
2148  if (c->fd < 0)
2149  return MEMIF_ERR_DISCONNECTED;
2150  uint8_t num =
2151  (c->args.is_master) ? c->run_args.num_s2m_rings : c->
2152  run_args.num_m2s_rings;
2153  if (qid >= num)
2154  return MEMIF_ERR_QID;
2155 
2156  *efd = c->rx_queues[qid].int_fd;
2157 
2158  return MEMIF_ERR_SUCCESS;
2159 }
2160 
2161 int
2163 {
2165  if (lm->control_list)
2166  lm->free (lm->control_list);
2167  lm->control_list = NULL;
2168  if (lm->interrupt_list)
2169  lm->free (lm->interrupt_list);
2170  lm->interrupt_list = NULL;
2171  if (lm->listener_list)
2172  lm->free (lm->listener_list);
2173  lm->listener_list = NULL;
2174  if (lm->pending_list)
2175  lm->free (lm->pending_list);
2176  lm->pending_list = NULL;
2177  if (poll_cancel_fd != -1)
2178  close (poll_cancel_fd);
2179 
2180  return MEMIF_ERR_SUCCESS; /* 0 */
2181 }
int poll_cancel_fd
Definition: main.c:70
#define MAX_ERRBUF_LEN
Definition: main.c:60
#define MEMIF_DEFAULT_LOG2_RING_SIZE
Definition: memif_private.h:41
static void memif_free_register(memif_free_t *mf)
Definition: main.c:448
uint8_t * filename
#define EXPECT_FALSE(x)
Definition: memif_private.h:56
a
Definition: bitmap.h:538
uint8_t * inst_name
Definition: libmemif.h:364
uint8_t * secret
Definition: libmemif.h:369
static char memif_buf[MAX_ERRBUF_LEN]
Definition: main.c:72
Memif region details.
Definition: libmemif.h:333
int on_disconnect(memif_conn_handle_t conn, void *private_ctx)
Definition: main.c:186
void * ring
Definition: libmemif.h:286
#define MEMIF_FD_EVENT_READ
user needs to set events that occured on fd and pass them to memif_control_fd_handler ...
Definition: libmemif.h:88
uint8_t num_m2s_rings
Definition: libmemif.h:259
Optimized string handling code, including c11-compliant "safe C library" variants.
#define NULL
Definition: clib.h:58
uint16_t buffer_size
Definition: libmemif.h:260
uint8_t secret[24]
Definition: libmemif.h:256
static void memif_msg_queue_free(libmemif_main_t *lm, memif_msg_queue_elt_t **e)
Definition: main.c:1103
uint32_t interface_id
Definition: libmemif.h:264
memif_list_elt_t * pending_list
int memif_conn_fd_read_ready(memif_connection_t *c)
Definition: socket.c:828
#define MEMIF_NAME_LEN
Definition: memif_private.h:34
uint32_t alloc_bufs
Definition: memif_private.h:90
for(i=1;i<=collision_buckets;i++)
int i
#define MEMIF_ERR_UNDEFINED
Definition: main.c:156
clib_error_t * memif_msg_send_disconnect(memif_if_t *mif, clib_error_t *err)
Definition: socket.c:190
int memif_disconnect_internal(memif_connection_t *c)
Definition: main.c:1115
int( memif_control_fd_update_t)(int fd, uint8_t events)
Memif control file descriptor update (callback function)
Definition: libmemif.h:140
memif_control_fd_update_t * control_fd_update
static void memif_control_fd_update_register(memif_control_fd_update_t *cb)
Definition: main.c:414
uint16_t desc_index
Definition: libmemif.h:285
#define MEMIF_MEMORY_BARRIER()
Definition: main.c:65
uint8_t * remote_inst_name
Definition: libmemif.h:366
static memif_ring_t * memif_get_ring(memif_connection_t *conn, memif_ring_type_t type, uint16_t ring_num)
Definition: main.c:596
memif_get_external_buffer_offset_t * get_external_buffer_offset
uint32_t length
Definition: memif.h:152
int memif_refill_queue(memif_conn_handle_t conn, uint16_t qid, uint16_t count, uint16_t headroom)
Memif refill ring.
Definition: main.c:1778
#define MEMIF_DEFAULT_APP_NAME
Default name of application using libmemif.
Definition: libmemif.h:28
#define MFD_ALLOW_SEALING
Definition: main.c:104
memif_list_elt_t * interface_list
int on_connect(memif_conn_handle_t conn, void *private_ctx)
Definition: main.c:177
uint8_t num_s2m_rings
Definition: libmemif.h:258
static void memif_realloc_register(memif_realloc_t *mr)
Definition: main.c:441
static void memif_alloc_register(memif_alloc_t *ma)
Definition: main.c:434
uint32_t len
Definition: libmemif.h:287
#define F_SEAL_SHRINK
Definition: main.c:112
memif_list_elt_t * listener_list
memif_realloc_t * realloc
int memif_get_details(memif_conn_handle_t conn, memif_details_t *md, char *buf, ssize_t buflen)
Memif get details.
Definition: main.c:1984
uint32_t cookie
Definition: memif.h:166
memif_interface_mode_t mode
Definition: libmemif.h:266
static clib_error_t * memif_conn_fd_write_ready(clib_file_t *uf, memif_if_t *mif)
Definition: socket.c:560
uint16_t disconn_slaves
memif_free_t * free
uint8_t mode
Definition: libmemif.h:371
#define DBG_TX_BUF
Definition: main.c:181
uint16_t flags
Definition: memif.h:149
uint8_t link_up_down
Definition: libmemif.h:381
struct itimerspec arm disarm
#define ERRLIST_LEN
Definition: main.c:59
char * memif_strerror(int err_code)
Memif strerror.
Definition: main.c:159
static int memfd_create(const char *name, unsigned int flags)
Definition: main.c:93
memif_region_offset_t offset
Definition: private.h:121
void *( memif_realloc_t)(void *ptr, size_t size)
Memif realloc.
Definition: libmemif.h:117
#define MEMIF_MAX_LOG2_RING_SIZE
Definition: private.h:30
int memif_epfd
Definition: main.c:69
#define MEMIF_BUFFER_FLAG_NEXT
next buffer present (chained buffers)
Definition: libmemif.h:289
int memif_syscall_error_handler(int err_code)
Definition: main.c:204
memif_list_elt_t * interrupt_list
memif_region_index_t region
Definition: memif.h:151
static int memif_del_epoll_fd(int fd)
Definition: main.c:282
u16 last_head
Definition: private.h:123
uint8_t interface_name[32]
Definition: libmemif.h:265
uint8_t * error
Definition: libmemif.h:380
uword size
void * data
Definition: libmemif.h:293
memif_desc_t desc[0]
Definition: memif.h:173
uint16_t buffer_size
Definition: libmemif.h:323
uint8_t * socket_filename
Definition: libmemif.h:372
#define MEMIF_FD_EVENT_DEL
if set, informs that fd is going to be closed (user may want to stop watching for events on this fd) ...
Definition: libmemif.h:93
int memif_set_rx_mode(memif_conn_handle_t c, memif_rx_mode_t rx_mode, uint16_t qid)
Memif set rx mode.
Definition: main.c:611
long ctx[MAX_CONNS]
Definition: main.c:144
int( memif_interrupt_t)(memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
Memif interrupt occured (callback function)
Definition: libmemif.h:159
int( memif_connection_update_t)(memif_conn_handle_t conn, void *private_ctx)
Memif connection status update (callback function)
Definition: libmemif.h:149
static int memif_add_epoll_fd(int fd, uint32_t events)
Definition: main.c:240
uint8_t is_master
Definition: libmemif.h:262
uint8_t rx_queues_num
Definition: libmemif.h:375
int memif_init(memif_control_fd_update_t *on_control_fd_update, char *app_name, memif_alloc_t *memif_alloc, memif_realloc_t *memif_realloc, memif_free_t *memif_free)
Memif initialization.
Definition: main.c:455
void *( memif_get_external_region_addr_t)(uint32_t size, int fd, void *private_ctx)
Get external region address.
Definition: libmemif.h:198
int memif_get_queue_efd(memif_conn_handle_t conn, uint16_t qid, int *efd)
Memif get queue event file descriptor
Definition: main.c:2142
#define MEMIF_BUFFER_FLAG_RX
states that buffer is from rx ring
Definition: libmemif.h:291
int memif_buffer_enq_tx(memif_conn_handle_t conn, uint16_t qid, memif_buffer_t *bufs, uint16_t count, uint16_t *count_out)
Memif buffer enq tx.
Definition: main.c:1573
#define MEMIF_DEFAULT_TX_QUEUES
Definition: private.h:24
uint32_t buffer_offset
Definition: memif_private.h:72
int add_list_elt(memif_list_elt_t *e, memif_list_elt_t **list, uint16_t *len)
Definition: main.c:319
int memif_init_regions_and_queues(memif_connection_t *conn)
Definition: main.c:1534
void( memif_free_t)(void *ptr)
Memif allocator free.
Definition: libmemif.h:124
u8 len
Definition: ip_types.api:49
uint16_t use_count
int( memif_del_external_region_t)(void *addr, uint32_t size, int fd, void *private_ctx)
Delete external region.
Definition: libmemif.h:209
uint8_t flags
Definition: libmemif.h:292
memif_add_external_region_t * add_external_region
int on_interrupt(memif_conn_handle_t conn, void *private_ctx, uint16_t qid)
Definition: main.c:287
uint8_t regions_num
Definition: libmemif.h:373
svmdb_client_t * c
int memif_tx_burst(memif_conn_handle_t conn, uint16_t qid, memif_buffer_t *bufs, uint16_t count, uint16_t *tx)
Memif transmit buffer burst.
Definition: main.c:1832
#define F_ADD_SEALS
Definition: main.c:108
int get_list_elt(memif_list_elt_t **e, memif_list_elt_t *list, uint16_t len, int key)
Definition: main.c:354
int memif_buffer_alloc(memif_conn_handle_t conn, uint16_t qid, memif_buffer_t *bufs, uint16_t count, uint16_t *count_out, uint16_t size)
Memif buffer alloc.
Definition: main.c:1652
static int memif_add_region(libmemif_main_t *lm, memif_connection_t *conn, uint8_t has_buffers)
Definition: main.c:1385
memif_alloc_t * alloc
int memif_poll_event(int timeout)
Memif poll event.
Definition: main.c:1047
memif_queue_details_t * rx_queues
Definition: libmemif.h:377
u16 last_tail
Definition: private.h:124
int memif_read_ready(int fd)
Definition: socket.c:903
uint8_t role
Definition: libmemif.h:370
#define MEMIF_DEFAULT_BUFFER_SIZE
Definition: private.h:25
int memif_cleanup()
Memif cleanup.
Definition: main.c:2162
#define memif_min(a, b)
Definition: memif_private.h:53
const char * memif_errlist[ERRLIST_LEN]
Definition: main.c:74
int memif_create(memif_conn_handle_t *c, memif_conn_args_t *args, memif_connection_update_t *on_connect, memif_connection_update_t *on_disconnect, memif_interrupt_t *on_interrupt, void *private_ctx)
Memory interface create function.
Definition: main.c:629
#define MEMIF_DESC_FLAG_NEXT
Definition: memif.h:150
int free_list_elt_ctx(memif_list_elt_t *list, uint16_t len, memif_connection_t *ctx)
Definition: main.c:394
static int memif_mod_epoll_fd(int fd, uint32_t events)
Definition: main.c:261
#define MEMIF_VERSION
Definition: memif.h:28
static_always_inline void * memif_get_buffer(memif_if_t *mif, memif_ring_t *ring, u16 slot)
Definition: private.h:289
memif_ring_t * ring
Definition: private.h:118
uint8_t tx_queues_num
Definition: libmemif.h:376
uint16_t interrupt_list_len
#define MEMIF_DEFAULT_RX_QUEUES
Definition: private.h:23
int( memif_add_external_region_t)(void **addr, uint32_t size, int *fd, void *private_ctx)
Add external region.
Definition: libmemif.h:186
size_t count
Definition: vapi.c:47
#define DBG(...)
Definition: main.c:61
#define MEMIF_DEFAULT_SOCKET_DIR
Definition: memif_private.h:38
#define MEMIF_RING_FLAG_MASK_INT
Definition: memif.h:168
void * memif_conn_handle_t
Memif connection handle pointer of type void, pointing to internal structure.
Definition: libmemif.h:101
memif_region_offset_t offset
Definition: memif.h:153
int memif_control_fd_handler(int fd, uint8_t events)
Memif control file descriptor handler.
Definition: main.c:893
memif_del_external_region_t * del_external_region
uint8_t app_name[MEMIF_NAME_LEN]
#define MEMIF_DEFAULT_SOCKET_FILENAME
Definition: private.h:21
uint8_t log2_ring_size
Definition: libmemif.h:261
uint32_t id
Definition: libmemif.h:368
int memif_rx_burst(memif_conn_handle_t conn, uint16_t qid, memif_buffer_t *bufs, uint16_t count, uint16_t *rx)
Memif receive buffer burst.
Definition: main.c:1899
uint16_t pending_list_len
memif_rx_mode_t
Definition: libmemif.h:270
memif_list_elt_t * control_list
int memif_delete(memif_conn_handle_t *conn)
Memif delete.
Definition: main.c:1234
static int memif_init_queues(libmemif_main_t *lm, memif_connection_t *conn)
Definition: main.c:1435
memif_log2_ring_size_t log2_ring_size
Definition: private.h:119
Memif queue details.
Definition: libmemif.h:313
memif_region_details_t * regions
Definition: libmemif.h:374
uint16_t flags
Definition: memif.h:167
#define MEMIF_FD_EVENT_ERROR
inform libmemif that error occured on fd
Definition: libmemif.h:91
int memif_cancel_poll_event()
Definition: main.c:1088
uint16_t control_list_len
#define MEMIF_FD_EVENT_MOD
update events
Definition: libmemif.h:95
#define DBG_RX_BUF
Definition: main.c:182
memif_ring_type_t
Definition: memif.h:47
memif_queue_details_t * tx_queues
Definition: libmemif.h:378
uint32_t( memif_get_external_buffer_offset_t)(void *private_ctx)
Get external buffer offset (optional)
Definition: libmemif.h:176
uint8_t * if_name
Definition: libmemif.h:363
volatile uint16_t head
Definition: memif.h:169
clib_error_t * memif_conn_fd_accept_ready(clib_file_t *uf)
Definition: socket.c:649
int memif_conn_fd_error(memif_connection_t *c)
Definition: socket.c:818
int memif_connect1(memif_connection_t *c)
Definition: main.c:1309
uint16_t index
Definition: main.c:74
libmemif_main_t libmemif_main
Definition: main.c:68
Memif buffer.
Definition: libmemif.h:283
int free_list_elt(memif_list_elt_t *list, uint16_t len, int key)
Definition: main.c:377
#define MEMIF_FD_EVENT_WRITE
Definition: libmemif.h:89
uint16_t memif_get_version()
Memif get version.
Definition: main.c:176
memif_get_external_region_addr_t * get_external_region_addr
memif_region_index_t region
Definition: private.h:120
uint16_t listener_list_len
#define MEMIF_COOKIE
Definition: memif.h:25
Memif connection arguments.
Definition: libmemif.h:253
uint16_t interface_list_len
Memif details.
Definition: libmemif.h:361
volatile uint16_t tail
Definition: memif.h:171
void memif_register_external_region(memif_add_external_region_t *ar, memif_get_external_region_addr_t *gr, memif_del_external_region_t *dr, memif_get_external_buffer_offset_t *go)
Register external region.
Definition: main.c:421
memif_region_size_t region_size
Definition: private.h:103
uint8_t * remote_if_name
Definition: libmemif.h:365
uint8_t * socket_filename
Definition: libmemif.h:255
void *( memif_alloc_t)(size_t size)
Memif allocator alloc.
Definition: libmemif.h:108
int memif_control_fd_update(int fd, uint8_t events)
Definition: main.c:301