FD.io VPP  v19.04.1-1-ge4a0f9f
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <unistd.h>
19 #include <fcntl.h>
20 #include <net/if.h>
21 #include <linux/if_link.h>
22 #include <linux/if_ether.h>
23 
24 #include <vppinfra/linux/sysfs.h>
25 #include <vlib/vlib.h>
26 #include <vlib/unix/unix.h>
27 #include <vlib/pci/pci.h>
28 #include <vnet/ethernet/ethernet.h>
29 
30 #include <rdma/rdma.h>
31 
32 /* Default RSS hash key (from DPDK MLX driver) */
33 static u8 rdma_rss_hash_key[] = {
34  0x2c, 0xc6, 0x81, 0xd1,
35  0x5b, 0xdb, 0xf4, 0xf7,
36  0xfc, 0xa2, 0x83, 0x19,
37  0xdb, 0x1a, 0x3e, 0x94,
38  0x6b, 0x9e, 0x38, 0xd9,
39  0x2c, 0x9c, 0x03, 0xd1,
40  0xad, 0x99, 0x44, 0xa7,
41  0xd9, 0x56, 0x3d, 0x59,
42  0x06, 0x3c, 0x25, 0xf3,
43  0xfc, 0x1f, 0xdc, 0x2a,
44 };
45 
47 
48 #define rdma_log_debug(dev, f, ...) \
49 { \
50  vlib_log(VLIB_LOG_LEVEL_DEBUG, rdma_main.log_class, "%U: " f, \
51  format_vlib_pci_addr, &rd->pci_addr, ##__VA_ARGS__); \
52 };
53 
54 static u32
56 {
57  rdma_main_t *rm = &rdma_main;
58  vlib_log_warn (rm->log_class, "TODO");
59  return 0;
60 }
61 
62 static void
64 {
65  struct ibv_port_attr attr;
66  u32 width = 0;
67  u32 speed = 0;
68 
69  if (ibv_query_port (rd->ctx, port, &attr))
70  {
73  return;
74  }
75 
76  /* update state */
77  switch (attr.state)
78  {
79  case IBV_PORT_ACTIVE: /* fallthrough */
80  case IBV_PORT_ACTIVE_DEFER:
81  rd->flags |= RDMA_DEVICE_F_LINK_UP;
84  break;
85  default:
86  rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
88  break;
89  }
90 
91  /* update speed */
92  switch (attr.active_width)
93  {
94  case 1:
95  width = 1;
96  break;
97  case 2:
98  width = 4;
99  break;
100  case 4:
101  width = 8;
102  break;
103  case 8:
104  width = 12;
105  break;
106  }
107  switch (attr.active_speed)
108  {
109  case 1:
110  speed = 2500000;
111  break;
112  case 2:
113  speed = 5000000;
114  break;
115  case 4: /* fallthrough */
116  case 8:
117  speed = 10000000;
118  break;
119  case 16:
120  speed = 14000000;
121  break;
122  case 32:
123  speed = 25000000;
124  break;
125  }
126  vnet_hw_interface_set_link_speed (vnm, rd->hw_if_index, width * speed);
127 }
128 
129 static clib_error_t *
131 {
132  rdma_main_t *rm = &rdma_main;
134  return clib_error_return (0, "RDMA async event error for device %U",
136 }
137 
138 static clib_error_t *
140 {
141  vnet_main_t *vnm = vnet_get_main ();
142  rdma_main_t *rm = &rdma_main;
144  int ret;
145  struct ibv_async_event event;
146  ret = ibv_get_async_event (rd->ctx, &event);
147  if (ret < 0)
148  {
149  return clib_error_return_unix (0, "ibv_get_async_event() failed");
150  }
151 
152  switch (event.event_type)
153  {
154  case IBV_EVENT_PORT_ACTIVE:
155  rdma_update_state (vnm, rd, event.element.port_num);
156  break;
157  case IBV_EVENT_PORT_ERR:
158  rdma_update_state (vnm, rd, event.element.port_num);
159  break;
160  case IBV_EVENT_DEVICE_FATAL:
161  rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
163  vlib_log_emerg (rm->log_class, "Fatal RDMA error for device %U",
165  break;
166  default:
168  "Unhandeld RDMA async event %i for device %U",
169  event.event_type, format_vlib_pci_addr, &rd->pci_addr);
170  break;
171  }
172 
173  ibv_ack_async_event (&event);
174  return 0;
175 }
176 
177 static clib_error_t *
179 {
180  clib_file_t t = { 0 };
181  int ret;
182 
183  /* make RDMA async event fd non-blocking */
184  ret = fcntl (rd->ctx->async_fd, F_GETFL);
185  if (ret < 0)
186  {
187  return clib_error_return_unix (0, "fcntl(F_GETFL) failed");
188  }
189  ret = fcntl (rd->ctx->async_fd, F_SETFL, ret | O_NONBLOCK);
190  if (ret < 0)
191  {
192  return clib_error_return_unix (0, "fcntl(F_SETFL, O_NONBLOCK) failed");
193  }
194 
195  /* register RDMA async event fd */
197  t.file_descriptor = rd->ctx->async_fd;
199  t.private_data = rd->dev_instance;
200  t.description =
201  format (0, "RMDA %U async event", format_vlib_pci_addr, &rd->pci_addr);
202 
204 
205  return 0;
206 }
207 
208 static void
210 {
212 }
213 
214 static clib_error_t *
216 {
218  rd->dev_instance, rd->hwaddr.bytes,
220 }
221 
222 static void
224 {
228 }
229 
230 static void
232 {
233  rdma_main_t *rm = &rdma_main;
234  rdma_rxq_t *rxq;
235  rdma_txq_t *txq;
236 
237 #define _(fn, arg) if (arg) \
238  { \
239  int rv; \
240  if ((rv = fn (arg))) \
241  rdma_log_debug (rd, #fn "() failed (rv = %d)", rv); \
242  }
243 
244  _(ibv_destroy_flow, rd->flow_mcast);
245  _(ibv_destroy_flow, rd->flow_ucast);
246  _(ibv_dereg_mr, rd->mr);
247  vec_foreach (txq, rd->txqs)
248  {
249  _(ibv_destroy_qp, txq->qp);
250  _(ibv_destroy_cq, txq->cq);
251  }
252  vec_foreach (rxq, rd->rxqs)
253  {
254  _(ibv_destroy_wq, rxq->wq);
255  _(ibv_destroy_cq, rxq->cq);
256  }
257  _(ibv_destroy_rwq_ind_table, rd->rx_rwq_ind_tbl);
258  _(ibv_destroy_qp, rd->rx_qp);
259  _(ibv_dealloc_pd, rd->pd);
260  _(ibv_close_device, rd->ctx);
261 #undef _
262 
263  clib_error_free (rd->error);
264 
265  vec_free (rd->rxqs);
266  vec_free (rd->txqs);
267  vec_free (rd->name);
268  pool_put (rm->devices, rd);
269 }
270 
271 static clib_error_t *
272 rdma_rxq_init_flow (struct ibv_flow **flow, struct ibv_qp *qp,
273  const mac_address_t * mac, const mac_address_t * mask,
274  u32 flags)
275 {
276  struct raw_eth_flow_attr
277  {
278  struct ibv_flow_attr attr;
279  struct ibv_flow_spec_eth spec_eth;
280  } __attribute__ ((packed)) fa;
281 
282  memset (&fa, 0, sizeof (fa));
283  fa.attr.num_of_specs = 1;
284  fa.attr.port = 1;
285  fa.attr.flags = flags;
286  fa.spec_eth.type = IBV_FLOW_SPEC_ETH;
287  fa.spec_eth.size = sizeof (struct ibv_flow_spec_eth);
288 
289  memcpy (fa.spec_eth.val.dst_mac, mac, sizeof (fa.spec_eth.val.dst_mac));
290  memcpy (fa.spec_eth.mask.dst_mac, mask, sizeof (fa.spec_eth.mask.dst_mac));
291 
292  if ((*flow = ibv_create_flow (qp, &fa.attr)) == 0)
293  return clib_error_return_unix (0, "create Flow Failed");
294 
295  return 0;
296 }
297 
298 static clib_error_t *
300 {
301  rdma_rxq_t *rxq;
302  struct ibv_wq_init_attr wqia;
303  struct ibv_wq_attr wqa;
304 
306  rxq = vec_elt_at_index (rd->rxqs, qid);
307  rxq->size = n_desc;
308 
309  if ((rxq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
310  return clib_error_return_unix (0, "Create CQ Failed");
311 
312  memset (&wqia, 0, sizeof (wqia));
313  wqia.wq_type = IBV_WQT_RQ;
314  wqia.max_wr = n_desc;
315  wqia.max_sge = 1;
316  wqia.pd = rd->pd;
317  wqia.cq = rxq->cq;
318  if ((rxq->wq = ibv_create_wq (rd->ctx, &wqia)) == 0)
319  return clib_error_return_unix (0, "Create WQ Failed");
320 
321  memset (&wqa, 0, sizeof (wqa));
322  wqa.attr_mask = IBV_WQ_ATTR_STATE;
323  wqa.wq_state = IBV_WQS_RDY;
324  if (ibv_modify_wq (rxq->wq, &wqa) != 0)
325  return clib_error_return_unix (0, "Modify WQ (RDY) Failed");
326 
327  return 0;
328 }
329 
330 static clib_error_t *
332 {
333  struct ibv_rwq_ind_table_init_attr rwqia;
334  struct ibv_qp_init_attr_ex qpia;
335  const mac_address_t ucast = {.bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
336  };
337  const mac_address_t mcast = {.bytes = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0} };
338  struct ibv_wq **ind_tbl;
339  clib_error_t *err;
340  u32 i;
341 
342  ASSERT (is_pow2 (vec_len (rd->rxqs))
343  && "rxq number should be a power of 2");
344 
345  ind_tbl = vec_new (struct ibv_wq *, vec_len (rd->rxqs));
346  vec_foreach_index (i, rd->rxqs)
347  ind_tbl[i] = vec_elt_at_index (rd->rxqs, i)->wq;
348  memset (&rwqia, 0, sizeof (rwqia));
349  rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl));
350  rwqia.ind_tbl = ind_tbl;
351  if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0)
352  return clib_error_return_unix (0, "RWQ indirection table create failed");
353  vec_free (ind_tbl);
354 
355  memset (&qpia, 0, sizeof (qpia));
356  qpia.qp_type = IBV_QPT_RAW_PACKET;
357  qpia.comp_mask =
358  IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_IND_TABLE |
359  IBV_QP_INIT_ATTR_RX_HASH;
360  qpia.pd = rd->pd;
361  qpia.rwq_ind_tbl = rd->rx_rwq_ind_tbl;
363  qpia.rx_hash_conf.rx_hash_key_len = sizeof (rdma_rss_hash_key);
364  qpia.rx_hash_conf.rx_hash_key = rdma_rss_hash_key;
365  qpia.rx_hash_conf.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ;
366  qpia.rx_hash_conf.rx_hash_fields_mask =
367  IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4;
368  if ((rd->rx_qp = ibv_create_qp_ex (rd->ctx, &qpia)) == 0)
369  return clib_error_return_unix (0, "Queue Pair create failed");
370 
371  /* receive only packets with src = our MAC */
372  if ((err =
373  rdma_rxq_init_flow (&rd->flow_ucast, rd->rx_qp, &rd->hwaddr, &ucast,
374  0)) != 0)
375  return err;
376  /* receive multicast packets */
377  return rdma_rxq_init_flow (&rd->flow_mcast, rd->rx_qp, &mcast, &mcast,
378  IBV_FLOW_ATTR_FLAGS_DONT_TRAP
379  /* let others receive mcast packet too (eg. Linux) */
380  );
381 }
382 
383 static clib_error_t *
385 {
386  rdma_txq_t *txq;
387  struct ibv_qp_init_attr qpia;
388  struct ibv_qp_attr qpa;
389  int qp_flags;
390 
392  txq = vec_elt_at_index (rd->txqs, qid);
393  txq->size = n_desc;
394 
395  if ((txq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
396  return clib_error_return_unix (0, "Create CQ Failed");
397 
398  memset (&qpia, 0, sizeof (qpia));
399  qpia.send_cq = txq->cq;
400  qpia.recv_cq = txq->cq;
401  qpia.cap.max_send_wr = n_desc;
402  qpia.cap.max_send_sge = 1;
403  qpia.qp_type = IBV_QPT_RAW_PACKET;
404  qpia.sq_sig_all = 1;
405 
406  if ((txq->qp = ibv_create_qp (rd->pd, &qpia)) == 0)
407  return clib_error_return_unix (0, "Queue Pair create failed");
408 
409  memset (&qpa, 0, sizeof (qpa));
410  qp_flags = IBV_QP_STATE | IBV_QP_PORT;
411  qpa.qp_state = IBV_QPS_INIT;
412  qpa.port_num = 1;
413  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
414  return clib_error_return_unix (0, "Modify QP (init) Failed");
415 
416  memset (&qpa, 0, sizeof (qpa));
417  qp_flags = IBV_QP_STATE;
418  qpa.qp_state = IBV_QPS_RTR;
419  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
420  return clib_error_return_unix (0, "Modify QP (receive) Failed");
421 
422  memset (&qpa, 0, sizeof (qpa));
423  qp_flags = IBV_QP_STATE;
424  qpa.qp_state = IBV_QPS_RTS;
425  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
426  return clib_error_return_unix (0, "Modify QP (send) Failed");
427  return 0;
428 }
429 
430 static clib_error_t *
432  u32 txq_size, u32 rxq_num)
433 {
434  clib_error_t *err;
437  u32 i;
438 
439  if (rd->ctx == 0)
440  return clib_error_return_unix (0, "Device Open Failed");
441 
442  if ((rd->pd = ibv_alloc_pd (rd->ctx)) == 0)
443  return clib_error_return_unix (0, "PD Alloc Failed");
444 
446 
447  for (i = 0; i < rxq_num; i++)
448  if ((err = rdma_rxq_init (vm, rd, i, rxq_size)))
449  return err;
450  if ((err = rdma_rxq_finalize (vm, rd)))
451  return err;
452 
453  for (i = 0; i < tm->n_vlib_mains; i++)
454  if ((err = rdma_txq_init (vm, rd, i, txq_size)))
455  return err;
456 
457  if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start,
458  bm->buffer_mem_size,
459  IBV_ACCESS_LOCAL_WRITE)) == 0)
460  return clib_error_return_unix (0, "Register MR Failed");
461 
462  return 0;
463 }
464 
465 static uword
466 sysfs_path_to_pci_addr (char *path, vlib_pci_addr_t * addr)
467 {
468  uword rv;
469  unformat_input_t in;
470  u8 *s;
471 
472  s = clib_sysfs_link_to_name (path);
473  unformat_init_string (&in, (char *) s, strlen ((char *) s));
474  rv = unformat (&in, "%U", unformat_vlib_pci_addr, addr);
475  unformat_free (&in);
476  vec_free (s);
477  return rv;
478 }
479 
480 void
482 {
483  vnet_main_t *vnm = vnet_get_main ();
484  rdma_main_t *rm = &rdma_main;
485  rdma_device_t *rd = 0;
486  struct ibv_device **dev_list = 0;
487  int n_devs;
488  u8 *s = 0, *s2 = 0;
489  u16 qid;
490 
491  args->rxq_size = args->rxq_size ? args->rxq_size : 2 * VLIB_FRAME_SIZE;
492  args->txq_size = args->txq_size ? args->txq_size : 2 * VLIB_FRAME_SIZE;
493  args->rxq_num = args->rxq_num ? args->rxq_num : 1;
494 
495  if (!is_pow2 (args->rxq_num))
496  {
497  args->rv = VNET_API_ERROR_INVALID_VALUE;
498  args->error =
499  clib_error_return (0, "rx queue number must be a power of two");
500  return;
501  }
502 
503  if (!is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size))
504  {
505  args->rv = VNET_API_ERROR_INVALID_VALUE;
506  args->error =
507  clib_error_return (0, "queue size must be a power of two");
508  return;
509  }
510 
511  pool_get_zero (rm->devices, rd);
512  rd->dev_instance = rd - rm->devices;
514  rd->name = vec_dup (args->name);
515 
516  /* check if device exist and if it is bound to mlx5_core */
517  s = format (s, "/sys/class/net/%s/device/driver/module%c", args->ifname, 0);
518  s2 = clib_sysfs_link_to_name ((char *) s);
519 
520  if (s2 == 0 || strncmp ((char *) s2, "mlx5_core", 9) != 0)
521  {
522  args->error =
524  "invalid interface (only mlx5 supported for now)");
525  goto err0;
526  }
527 
528  /* extract PCI address */
529  vec_reset_length (s);
530  s = format (s, "/sys/class/net/%s/device%c", args->ifname, 0);
531  if (sysfs_path_to_pci_addr ((char *) s, &rd->pci_addr) == 0)
532  {
533  args->error = clib_error_return (0, "cannot find PCI address");
534  goto err0;
535  }
536 
537  dev_list = ibv_get_device_list (&n_devs);
538  if (n_devs == 0)
539  {
540  args->error =
542  "no RDMA devices available, errno = %d. "
543  "Is the ib_uverbs module loaded?", errno);
544  goto err1;
545  }
546 
547  for (int i = 0; i < n_devs; i++)
548  {
549  vlib_pci_addr_t addr;
550 
551  vec_reset_length (s);
552  s = format (s, "%s/device%c", dev_list[i]->dev_path, 0);
553 
554  if (sysfs_path_to_pci_addr ((char *) s, &addr) == 0)
555  continue;
556 
557  if (addr.as_u32 != rd->pci_addr.as_u32)
558  continue;
559 
560  if ((rd->ctx = ibv_open_device (dev_list[i])))
561  break;
562  }
563 
564  if ((args->error =
565  rdma_dev_init (vm, rd, args->rxq_size, args->txq_size, args->rxq_num)))
566  goto err2;
567 
568  if ((args->error = rdma_register_interface (vnm, rd)))
569  goto err2;
570 
571  if ((args->error = rdma_async_event_init (rd)))
572  goto err3;
573 
574  rdma_update_state (vnm, rd, 1);
575 
577  args->sw_if_index = rd->sw_if_index = sw->sw_if_index;
578  /*
579  * FIXME: add support for interrupt mode
580  * vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, rd->hw_if_index);
581  * hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
582  */
584  rdma_input_node.index);
585  vec_foreach_index (qid, rd->rxqs)
587  return;
588 
589 err3:
590  rdma_unregister_interface (vnm, rd);
591 err2:
592  rdma_dev_cleanup (rd);
593 err1:
594  ibv_free_device_list (dev_list);
595 err0:
596  vec_free (s2);
597  vec_free (s);
598  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
599  vlib_log_err (rm->log_class, "%U", format_clib_error, args->error);
600 }
601 
602 void
604 {
607  rdma_dev_cleanup (rd);
608 }
609 
610 static clib_error_t *
612 {
613  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
614  rdma_main_t *rm = &rdma_main;
616  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
617 
618  if (rd->flags & RDMA_DEVICE_F_ERROR)
619  return clib_error_return (0, "device is in error state");
620 
621  if (is_up)
622  {
625  rd->flags |= RDMA_DEVICE_F_ADMIN_UP;
626  }
627  else
628  {
630  rd->flags &= ~RDMA_DEVICE_F_ADMIN_UP;
631  }
632  return 0;
633 }
634 
635 static void
637  u32 node_index)
638 {
639  rdma_main_t *rm = &rdma_main;
640  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
642 
643  /* Shut off redirection */
644  if (node_index == ~0)
645  {
646  rd->per_interface_next_index = node_index;
647  return;
648  }
649 
651  vlib_node_add_next (vlib_get_main (), rdma_input_node.index, node_index);
652 }
653 
654 static char *rdma_tx_func_error_strings[] = {
655 #define _(n,s) s,
657 #undef _
658 };
659 
660 /* *INDENT-OFF* */
662 {
663  .name = "RDMA interface",
664  .format_device = format_rdma_device,
665  .format_device_name = format_rdma_device_name,
666  .admin_up_down_function = rdma_interface_admin_up_down,
667  .rx_redirect_to_node = rdma_set_interface_next_node,
668  .tx_function_n_errors = RDMA_TX_N_ERROR,
669  .tx_function_error_strings = rdma_tx_func_error_strings,
670 };
671 /* *INDENT-ON* */
672 
673 clib_error_t *
675 {
676  rdma_main_t *rm = &rdma_main;
677 
678  rm->log_class = vlib_log_register_class ("rdma", 0);
679 
680  return 0;
681 }
682 
684 
685 /*
686  * fd.io coding-style-patch-verification: ON
687  *
688  * Local Variables:
689  * eval: (c-set-style "gnu")
690  * End:
691  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:227
struct ibv_mr * mr
Definition: rdma.h:80
vmrglw vmrglh hi
#define vec_foreach_index(var, v)
Iterate over vector indices.
u8 * format_clib_error(u8 *s, va_list *va)
Definition: error.c:191
u32 flags
Definition: vhost_user.h:115
#define vlib_log_warn(...)
Definition: log.h:49
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:324
static clib_error_t * rdma_rxq_finalize(vlib_main_t *vm, rdma_device_t *rd)
Definition: device.c:331
vnet_main_t * vnet_get_main(void)
Definition: misc.c:47
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
Definition: pool.h:239
format_function_t format_rdma_device
Definition: rdma.h:119
u32 size
Definition: rdma.h:43
#define NULL
Definition: clib.h:58
u32 dev_instance
Definition: rdma.h:65
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 file_descriptor
Definition: file.h:54
struct ibv_wq * wq
Definition: rdma.h:46
u32 size
Definition: rdma.h:52
u32 per_interface_next_index
Definition: rdma.h:63
int i
vlib_buffer_main_t * buffer_main
Definition: main.h:129
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static void rdma_unregister_interface(vnet_main_t *vnm, rdma_device_t *rd)
Definition: device.c:223
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
vhost_vring_addr_t addr
Definition: vhost_user.h:121
mac_address_t hwaddr
Definition: rdma.h:75
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1122
unsigned char u8
Definition: types.h:56
static uword min_log2(uword x)
Definition: clib.h:144
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
vlib_pci_addr_t pci_addr
Definition: rdma.h:76
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
vlib_log_class_t log_class
Definition: rdma.h:93
static clib_error_t * rdma_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:611
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:163
struct ibv_pd * pd
Definition: rdma.h:79
static void rdma_update_state(vnet_main_t *vnm, rdma_device_t *rd, int port)
Definition: device.c:63
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:311
rdma_device_t * devices
Definition: rdma.h:92
static clib_error_t * rdma_txq_init(vlib_main_t *vm, rdma_device_t *rd, u16 qid, u32 n_desc)
Definition: device.c:384
uword buffer_mem_size
Definition: buffer.h:412
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
struct ibv_cq * cq
Definition: rdma.h:54
#define clib_error_return(e, args...)
Definition: error.h:99
clib_file_main_t file_main
Definition: main.c:63
#define vlib_log_emerg(...)
Definition: log.h:45
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:376
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:1023
u32 flags
Definition: rdma.h:62
static clib_error_t * rdma_async_event_init(rdma_device_t *rd)
Definition: device.c:178
u8 * description
Definition: file.h:70
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
vlib_node_registration_t rdma_input_node
(constructor) VLIB_REGISTER_NODE (rdma_input_node)
Definition: input.c:358
#define clib_error_return_unix(e, args...)
Definition: error.h:102
struct ibv_cq * cq
Definition: rdma.h:45
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
unformat_function_t unformat_vlib_pci_addr
Definition: pci.h:323
#define vec_dup(V)
Return copy of vector (no header, no alignment)
Definition: vec.h:375
static uword sysfs_path_to_pci_addr(char *path, vlib_pci_addr_t *addr)
Definition: device.c:466
static void rdma_async_event_cleanup(rdma_device_t *rd)
Definition: device.c:209
static clib_error_t * rdma_dev_init(vlib_main_t *vm, rdma_device_t *rd, u32 rxq_size, u32 txq_size, u32 rxq_num)
Definition: device.c:431
struct ibv_qp * qp
Definition: rdma.h:55
static clib_error_t * rdma_rxq_init(vlib_main_t *vm, rdma_device_t *rd, u16 qid, u32 n_desc)
Definition: device.c:299
u32 hw_if_index
Definition: rdma.h:67
struct ibv_rwq_ind_table * rx_rwq_ind_tbl
Definition: rdma.h:82
clib_error_t * error
Definition: rdma.h:87
rdma_main_t rdma_main
Definition: device.c:46
vlib_main_t * vm
Definition: buffer.c:312
clib_error_t * error
Definition: rdma.h:109
static clib_error_t * rdma_rxq_init_flow(struct ibv_flow **flow, struct ibv_qp *qp, const mac_address_t *mac, const mac_address_t *mask, u32 flags)
Definition: device.c:272
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
static clib_error_t * rdma_register_interface(vnet_main_t *vnm, rdma_device_t *rd)
Definition: device.c:215
static clib_error_t * rdma_async_event_error_ready(clib_file_t *f)
Definition: device.c:130
void rdma_create_if(vlib_main_t *vm, rdma_create_if_args_t *args)
Definition: device.c:481
static u8 rdma_rss_hash_key[]
Definition: device.c:33
u32 sw_if_index
Definition: rdma.h:66
static clib_error_t * rdma_async_event_read_ready(clib_file_t *f)
Definition: device.c:139
#define ASSERT(truth)
format_function_t format_rdma_device_name
Definition: rdma.h:120
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
static void rdma_dev_cleanup(rdma_device_t *rd)
Definition: device.c:231
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
Definition: file.h:119
u8 * name
Definition: rdma.h:74
rdma_txq_t * txqs
Definition: rdma.h:72
VNET_DEVICE_CLASS(bond_dev_class)
rdma_rxq_t * rxqs
Definition: rdma.h:71
clib_error_t * rdma_init(vlib_main_t *vm)
Definition: device.c:674
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
vnet_device_class_t rdma_device_class
static uword is_pow2(uword x)
Definition: clib.h:235
u32 async_event_clib_file_index
Definition: rdma.h:69
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static void ethernet_mac_address_generate(u8 *mac)
Definition: mac_address.h:74
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:278
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:504
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:162
uword buffer_mem_start
Definition: buffer.h:411
#define foreach_rdma_tx_func_error
Definition: rdma.h:129
void rdma_delete_if(vlib_main_t *vm, rdma_device_t *rd)
Definition: device.c:603
#define clib_error_free(e)
Definition: error.h:86
clib_file_function_t * error_function
Definition: file.h:67
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
struct ibv_flow * flow_ucast
Definition: rdma.h:83
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static char * rdma_tx_func_error_strings[]
Definition: device.c:654
#define vec_foreach(var, vec)
Vector iterator.
uword private_data
Definition: file.h:64
#define vlib_log_err(...)
Definition: log.h:48
Definition: file.h:51
struct ibv_flow * flow_mcast
Definition: rdma.h:84
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
#define STATIC_ASSERT_SIZEOF(d, s)
vl_api_mac_address_t mac
Definition: gbp.api:120
static u32 rdma_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: device.c:55
static void vnet_hw_interface_set_link_speed(vnet_main_t *vnm, u32 hw_if_index, u32 link_speed)
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
struct ibv_context * ctx
Definition: rdma.h:78
icmpr_flow_t * flow
Definition: main.c:123
format_function_t format_vlib_pci_addr
Definition: pci.h:324
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:972
u8 * clib_sysfs_link_to_name(char *link)
Definition: sysfs.c:90
struct ibv_qp * rx_qp
Definition: rdma.h:81
static void rdma_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:636