FD.io VPP  v20.01-48-g3e0dafb74
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <unistd.h>
19 #include <fcntl.h>
20 #include <net/if.h>
21 #include <linux/if_link.h>
22 #include <linux/if_ether.h>
23 
24 #include <vppinfra/linux/sysfs.h>
25 #include <vlib/vlib.h>
26 #include <vlib/unix/unix.h>
27 #include <vlib/pci/pci.h>
28 #include <vnet/ethernet/ethernet.h>
29 
30 #include <rdma/rdma.h>
31 
32 /* Default RSS hash key (from DPDK MLX driver) */
33 static u8 rdma_rss_hash_key[] = {
34  0x2c, 0xc6, 0x81, 0xd1,
35  0x5b, 0xdb, 0xf4, 0xf7,
36  0xfc, 0xa2, 0x83, 0x19,
37  0xdb, 0x1a, 0x3e, 0x94,
38  0x6b, 0x9e, 0x38, 0xd9,
39  0x2c, 0x9c, 0x03, 0xd1,
40  0xad, 0x99, 0x44, 0xa7,
41  0xd9, 0x56, 0x3d, 0x59,
42  0x06, 0x3c, 0x25, 0xf3,
43  0xfc, 0x1f, 0xdc, 0x2a,
44 };
45 
47 
48 #define rdma_log__(lvl, dev, f, ...) \
49  do { \
50  vlib_log((lvl), rdma_main.log_class, "%s: " f, \
51  &(dev)->name, ##__VA_ARGS__); \
52  } while (0)
53 
54 #define rdma_log(lvl, dev, f, ...) \
55  rdma_log__((lvl), (dev), "%s (%d): " f, strerror(errno), errno, ##__VA_ARGS__)
56 
57 static struct ibv_flow *
58 rdma_rxq_init_flow (const rdma_device_t * rd, struct ibv_qp *qp,
59  const mac_address_t * mac, const mac_address_t * mask,
60  u32 flags)
61 {
62  struct ibv_flow *flow;
63  struct raw_eth_flow_attr
64  {
65  struct ibv_flow_attr attr;
66  struct ibv_flow_spec_eth spec_eth;
67  } __attribute__ ((packed)) fa;
68 
69  memset (&fa, 0, sizeof (fa));
70  fa.attr.num_of_specs = 1;
71  fa.attr.port = 1;
72  fa.attr.flags = flags;
73  fa.spec_eth.type = IBV_FLOW_SPEC_ETH;
74  fa.spec_eth.size = sizeof (struct ibv_flow_spec_eth);
75 
76  memcpy (fa.spec_eth.val.dst_mac, mac, sizeof (fa.spec_eth.val.dst_mac));
77  memcpy (fa.spec_eth.mask.dst_mac, mask, sizeof (fa.spec_eth.mask.dst_mac));
78 
79  flow = ibv_create_flow (qp, &fa.attr);
80  if (!flow)
81  rdma_log (VLIB_LOG_LEVEL_ERR, rd, "ibv_create_flow() failed");
82  return flow;
83 }
84 
85 static u32
86 rdma_rxq_destroy_flow (const rdma_device_t * rd, struct ibv_flow **flow)
87 {
88  if (!*flow)
89  return 0;
90 
91  if (ibv_destroy_flow (*flow))
92  {
93  rdma_log (VLIB_LOG_LEVEL_ERR, rd, "ibv_destroy_flow() failed");
94  return ~0;
95  }
96 
97  *flow = 0;
98  return 0;
99 }
100 
101 static u32
103 {
104  const mac_address_t all = {.bytes = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0} };
105  int err;
106 
107  err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast);
108  if (err)
109  return ~0;
110 
111  err = rdma_rxq_destroy_flow (rd, &rd->flow_ucast);
112  if (err)
113  return ~0;
114 
115  rd->flow_ucast = rdma_rxq_init_flow (rd, rd->rx_qp, &all, &all, 0);
116  if (!rd->flow_ucast)
117  return ~0;
118 
119  rd->flags |= RDMA_DEVICE_F_PROMISC;
120  return 0;
121 }
122 
123 static u32
125 {
126  const mac_address_t ucast = {.bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
127  };
128  const mac_address_t mcast = {.bytes = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0} };
129  int err;
130 
131  err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast);
132  if (err)
133  return ~0;
134 
135  err = rdma_rxq_destroy_flow (rd, &rd->flow_ucast);
136  if (err)
137  return ~0;
138 
139  /* receive only packets with src = our MAC */
140  rd->flow_ucast = rdma_rxq_init_flow (rd, rd->rx_qp, &rd->hwaddr, &ucast, 0);
141  if (!rd->flow_ucast)
142  return ~0;
143 
144  /* receive multicast packets */
145  rd->flow_mcast = rdma_rxq_init_flow (rd, rd->rx_qp, &mcast, &mcast,
146  IBV_FLOW_ATTR_FLAGS_DONT_TRAP
147  /* let others receive mcast packet too (eg. Linux) */
148  );
149  if (!rd->flow_mcast)
150  return ~0;
151 
152  rd->flags &= ~RDMA_DEVICE_F_PROMISC;
153  return 0;
154 }
155 
156 static clib_error_t *
157 rdma_mac_change (vnet_hw_interface_t * hw, const u8 * old, const u8 * new)
158 {
159  rdma_main_t *rm = &rdma_main;
161  mac_address_from_bytes (&rd->hwaddr, new);
162  if (!(rd->flags & RDMA_DEVICE_F_PROMISC) && rdma_dev_set_ucast (rd))
163  {
164  mac_address_from_bytes (&rd->hwaddr, old);
165  return clib_error_return_unix (0, "MAC update failed");
166  }
167  return 0;
168 }
169 
170 static u32
172 {
173  rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "MTU change not supported");
174  return ~0;
175 }
176 
177 static u32
179 {
180  rdma_main_t *rm = &rdma_main;
182 
183  switch (flags)
184  {
185  case 0:
186  return rdma_dev_set_ucast (rd);
188  return rdma_dev_set_promisc (rd);
190  return rdma_dev_change_mtu (rd);
191  }
192 
193  rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "unknown flag %x requested", flags);
194  return ~0;
195 }
196 
197 static void
199 {
200  struct ibv_port_attr attr;
201  u32 width = 0;
202  u32 speed = 0;
203 
204  if (ibv_query_port (rd->ctx, port, &attr))
205  {
208  return;
209  }
210 
211  /* update state */
212  switch (attr.state)
213  {
214  case IBV_PORT_ACTIVE: /* fallthrough */
215  case IBV_PORT_ACTIVE_DEFER:
216  rd->flags |= RDMA_DEVICE_F_LINK_UP;
219  break;
220  default:
221  rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
223  break;
224  }
225 
226  /* update speed */
227  switch (attr.active_width)
228  {
229  case 1:
230  width = 1;
231  break;
232  case 2:
233  width = 4;
234  break;
235  case 4:
236  width = 8;
237  break;
238  case 8:
239  width = 12;
240  break;
241  }
242  switch (attr.active_speed)
243  {
244  case 1:
245  speed = 2500000;
246  break;
247  case 2:
248  speed = 5000000;
249  break;
250  case 4: /* fallthrough */
251  case 8:
252  speed = 10000000;
253  break;
254  case 16:
255  speed = 14000000;
256  break;
257  case 32:
258  speed = 25000000;
259  break;
260  }
261  vnet_hw_interface_set_link_speed (vnm, rd->hw_if_index, width * speed);
262 }
263 
264 static clib_error_t *
266 {
267  rdma_main_t *rm = &rdma_main;
269  return clib_error_return (0, "RDMA: %s: async event error", rd->name);
270 }
271 
272 static clib_error_t *
274 {
275  vnet_main_t *vnm = vnet_get_main ();
276  rdma_main_t *rm = &rdma_main;
278  int ret;
279  struct ibv_async_event event;
280  ret = ibv_get_async_event (rd->ctx, &event);
281  if (ret < 0)
282  return clib_error_return_unix (0, "ibv_get_async_event() failed");
283 
284  switch (event.event_type)
285  {
286  case IBV_EVENT_PORT_ACTIVE:
287  rdma_update_state (vnm, rd, event.element.port_num);
288  break;
289  case IBV_EVENT_PORT_ERR:
290  rdma_update_state (vnm, rd, event.element.port_num);
291  break;
292  case IBV_EVENT_DEVICE_FATAL:
293  rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
295  vlib_log_emerg (rm->log_class, "%s: fatal error", rd->name);
296  break;
297  default:
298  rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "unhandeld RDMA async event %i",
299  event.event_type);
300  break;
301  }
302 
303  ibv_ack_async_event (&event);
304  return 0;
305 }
306 
307 static clib_error_t *
309 {
310  clib_file_t t = { 0 };
311  int ret;
312 
313  /* make RDMA async event fd non-blocking */
314  ret = fcntl (rd->ctx->async_fd, F_GETFL);
315  if (ret < 0)
316  return clib_error_return_unix (0, "fcntl(F_GETFL) failed");
317 
318  ret = fcntl (rd->ctx->async_fd, F_SETFL, ret | O_NONBLOCK);
319  if (ret < 0)
320  return clib_error_return_unix (0, "fcntl(F_SETFL, O_NONBLOCK) failed");
321 
322  /* register RDMA async event fd */
324  t.file_descriptor = rd->ctx->async_fd;
326  t.private_data = rd->dev_instance;
327  t.description = format (0, "%v async event", rd->name);
328 
330  return 0;
331 }
332 
333 static void
335 {
337 }
338 
339 static clib_error_t *
341 {
343  rd->dev_instance, rd->hwaddr.bytes,
345 }
346 
347 static void
349 {
353 }
354 
355 static void
357 {
358  rdma_main_t *rm = &rdma_main;
359  rdma_rxq_t *rxq;
360  rdma_txq_t *txq;
361 
362 #define _(fn, arg) if (arg) \
363  { \
364  int rv; \
365  if ((rv = fn (arg))) \
366  rdma_log (VLIB_LOG_LEVEL_DEBUG, rd, #fn "() failed (rv = %d)", rv); \
367  }
368 
369  _(ibv_destroy_flow, rd->flow_mcast);
370  _(ibv_destroy_flow, rd->flow_ucast);
371  _(ibv_dereg_mr, rd->mr);
372  vec_foreach (txq, rd->txqs)
373  {
374  _(ibv_destroy_qp, txq->qp);
375  _(ibv_destroy_cq, txq->cq);
376  }
377  vec_foreach (rxq, rd->rxqs)
378  {
379  _(ibv_destroy_wq, rxq->wq);
380  _(ibv_destroy_cq, rxq->cq);
381  }
382  _(ibv_destroy_rwq_ind_table, rd->rx_rwq_ind_tbl);
383  _(ibv_destroy_qp, rd->rx_qp);
384  _(ibv_dealloc_pd, rd->pd);
385  _(ibv_close_device, rd->ctx);
386 #undef _
387 
388  clib_error_free (rd->error);
389 
390  vec_free (rd->rxqs);
391  vec_free (rd->txqs);
392  vec_free (rd->name);
394  pool_put (rm->devices, rd);
395 }
396 
397 static clib_error_t *
399 {
400  rdma_rxq_t *rxq;
401  struct ibv_wq_init_attr wqia;
402  struct ibv_wq_attr wqa;
403 
405  rxq = vec_elt_at_index (rd->rxqs, qid);
406  rxq->size = n_desc;
407  vec_validate_aligned (rxq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES);
408 
409  if ((rxq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
410  return clib_error_return_unix (0, "Create CQ Failed");
411 
412  memset (&wqia, 0, sizeof (wqia));
413  wqia.wq_type = IBV_WQT_RQ;
414  wqia.max_wr = n_desc;
415  wqia.max_sge = 1;
416  wqia.pd = rd->pd;
417  wqia.cq = rxq->cq;
418  if ((rxq->wq = ibv_create_wq (rd->ctx, &wqia)) == 0)
419  return clib_error_return_unix (0, "Create WQ Failed");
420 
421  memset (&wqa, 0, sizeof (wqa));
422  wqa.attr_mask = IBV_WQ_ATTR_STATE;
423  wqa.wq_state = IBV_WQS_RDY;
424  if (ibv_modify_wq (rxq->wq, &wqa) != 0)
425  return clib_error_return_unix (0, "Modify WQ (RDY) Failed");
426 
427  return 0;
428 }
429 
430 static clib_error_t *
432 {
433  struct ibv_rwq_ind_table_init_attr rwqia;
434  struct ibv_qp_init_attr_ex qpia;
435  struct ibv_wq **ind_tbl;
436  u32 i;
437 
438  ASSERT (is_pow2 (vec_len (rd->rxqs))
439  && "rxq number should be a power of 2");
440 
441  ind_tbl = vec_new (struct ibv_wq *, vec_len (rd->rxqs));
442  vec_foreach_index (i, rd->rxqs)
443  ind_tbl[i] = vec_elt_at_index (rd->rxqs, i)->wq;
444  memset (&rwqia, 0, sizeof (rwqia));
445  rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl));
446  rwqia.ind_tbl = ind_tbl;
447  if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0)
448  return clib_error_return_unix (0, "RWQ indirection table create failed");
449  vec_free (ind_tbl);
450 
451  memset (&qpia, 0, sizeof (qpia));
452  qpia.qp_type = IBV_QPT_RAW_PACKET;
453  qpia.comp_mask =
454  IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_IND_TABLE |
455  IBV_QP_INIT_ATTR_RX_HASH;
456  qpia.pd = rd->pd;
457  qpia.rwq_ind_tbl = rd->rx_rwq_ind_tbl;
459  qpia.rx_hash_conf.rx_hash_key_len = sizeof (rdma_rss_hash_key);
460  qpia.rx_hash_conf.rx_hash_key = rdma_rss_hash_key;
461  qpia.rx_hash_conf.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ;
462  qpia.rx_hash_conf.rx_hash_fields_mask =
463  IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4;
464  if ((rd->rx_qp = ibv_create_qp_ex (rd->ctx, &qpia)) == 0)
465  return clib_error_return_unix (0, "Queue Pair create failed");
466 
467  if (rdma_dev_set_ucast (rd))
468  return clib_error_return_unix (0, "Set unicast mode failed");
469 
470  return 0;
471 }
472 
473 static clib_error_t *
475 {
476  rdma_txq_t *txq;
477  struct ibv_qp_init_attr qpia;
478  struct ibv_qp_attr qpa;
479  int qp_flags;
480 
482  txq = vec_elt_at_index (rd->txqs, qid);
483  txq->size = n_desc;
484  vec_validate_aligned (txq->bufs, n_desc - 1, CLIB_CACHE_LINE_BYTES);
485 
486  if ((txq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
487  return clib_error_return_unix (0, "Create CQ Failed");
488 
489  memset (&qpia, 0, sizeof (qpia));
490  qpia.send_cq = txq->cq;
491  qpia.recv_cq = txq->cq;
492  qpia.cap.max_send_wr = n_desc;
493  qpia.cap.max_send_sge = 1;
494  qpia.qp_type = IBV_QPT_RAW_PACKET;
495 
496  if ((txq->qp = ibv_create_qp (rd->pd, &qpia)) == 0)
497  return clib_error_return_unix (0, "Queue Pair create failed");
498 
499  memset (&qpa, 0, sizeof (qpa));
500  qp_flags = IBV_QP_STATE | IBV_QP_PORT;
501  qpa.qp_state = IBV_QPS_INIT;
502  qpa.port_num = 1;
503  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
504  return clib_error_return_unix (0, "Modify QP (init) Failed");
505 
506  memset (&qpa, 0, sizeof (qpa));
507  qp_flags = IBV_QP_STATE;
508  qpa.qp_state = IBV_QPS_RTR;
509  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
510  return clib_error_return_unix (0, "Modify QP (receive) Failed");
511 
512  memset (&qpa, 0, sizeof (qpa));
513  qp_flags = IBV_QP_STATE;
514  qpa.qp_state = IBV_QPS_RTS;
515  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
516  return clib_error_return_unix (0, "Modify QP (send) Failed");
517  return 0;
518 }
519 
520 static clib_error_t *
522  u32 txq_size, u32 rxq_num)
523 {
524  clib_error_t *err;
527  u32 i;
528 
529  if (rd->ctx == 0)
530  return clib_error_return_unix (0, "Device Open Failed");
531 
532  if ((rd->pd = ibv_alloc_pd (rd->ctx)) == 0)
533  return clib_error_return_unix (0, "PD Alloc Failed");
534 
536 
537  /*
538  * /!\ WARNING /!\ creation order is important
539  * We *must* create TX queues *before* RX queues, otherwise we will receive
540  * the broacast packets we sent
541  */
542  for (i = 0; i < tm->n_vlib_mains; i++)
543  if ((err = rdma_txq_init (vm, rd, i, txq_size)))
544  return err;
545 
546  for (i = 0; i < rxq_num; i++)
547  if ((err = rdma_rxq_init (vm, rd, i, rxq_size)))
548  return err;
549  if ((err = rdma_rxq_finalize (vm, rd)))
550  return err;
551 
552  if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start,
553  bm->buffer_mem_size,
554  IBV_ACCESS_LOCAL_WRITE)) == 0)
555  return clib_error_return_unix (0, "Register MR Failed");
556  rd->lkey = rd->mr->lkey; /* avoid indirection in datapath */
557 
558  return 0;
559 }
560 
561 static uword
562 sysfs_path_to_pci_addr (char *path, vlib_pci_addr_t * addr)
563 {
564  uword rv;
565  unformat_input_t in;
566  u8 *s;
567 
568  s = clib_sysfs_link_to_name (path);
569  if (!s)
570  return 0;
571 
572  unformat_init_string (&in, (char *) s, strlen ((char *) s));
573  rv = unformat (&in, "%U", unformat_vlib_pci_addr, addr);
574  unformat_free (&in);
575  vec_free (s);
576  return rv;
577 }
578 
579 void
581 {
582  vnet_main_t *vnm = vnet_get_main ();
583  rdma_main_t *rm = &rdma_main;
584  rdma_device_t *rd;
585  vlib_pci_addr_t pci_addr;
586  struct ibv_device **dev_list;
587  int n_devs;
588  u8 *s;
589  u16 qid;
590  int i;
591 
592  args->rxq_size = args->rxq_size ? args->rxq_size : 1024;
593  args->txq_size = args->txq_size ? args->txq_size : 1024;
594  args->rxq_num = args->rxq_num ? args->rxq_num : 1;
595 
596  if (!is_pow2 (args->rxq_num))
597  {
598  args->rv = VNET_API_ERROR_INVALID_VALUE;
599  args->error =
600  clib_error_return (0, "rx queue number must be a power of two");
601  goto err0;
602  }
603 
604  if (args->rxq_size < VLIB_FRAME_SIZE || args->txq_size < VLIB_FRAME_SIZE ||
605  !is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size))
606  {
607  args->rv = VNET_API_ERROR_INVALID_VALUE;
608  args->error =
609  clib_error_return (0, "queue size must be a power of two >= %i",
611  goto err0;
612  }
613 
614  switch (args->mode)
615  {
616  case RDMA_MODE_AUTO:
617  break;
618  case RDMA_MODE_IBV:
619  break;
620  case RDMA_MODE_DV:
621  args->rv = VNET_API_ERROR_INVALID_VALUE;
622  args->error = clib_error_return (0, "unsupported mode");
623  goto err0;
624  break;
625  }
626 
627  dev_list = ibv_get_device_list (&n_devs);
628  if (n_devs == 0)
629  {
630  args->error =
632  "no RDMA devices available. Is the ib_uverbs module loaded?");
633  goto err0;
634  }
635 
636  /* get PCI address */
637  s = format (0, "/sys/class/net/%s/device%c", args->ifname, 0);
638  if (sysfs_path_to_pci_addr ((char *) s, &pci_addr) == 0)
639  {
640  args->error =
641  clib_error_return (0, "cannot find PCI address for device ");
642  goto err1;
643  }
644 
645  pool_get_zero (rm->devices, rd);
646  rd->dev_instance = rd - rm->devices;
648  rd->linux_ifname = format (0, "%s", args->ifname);
649 
650  if (!args->name || 0 == args->name[0])
651  rd->name = format (0, "%s/%d", args->ifname, rd->dev_instance);
652  else
653  rd->name = format (0, "%s", args->name);
654 
655  rd->pci = vlib_pci_get_device_info (vm, &pci_addr, &args->error);
656  if (!rd->pci)
657  goto err2;
658 
659  /* if we failed to parse NUMA node, default to 0 */
660  if (-1 == rd->pci->numa_node)
661  rd->pci->numa_node = 0;
662 
664 
665  if (strncmp ((char *) rd->pci->driver_name, "mlx5_core", 9))
666  {
667  args->error =
669  "invalid interface (only mlx5 supported for now)");
670  goto err2;
671  }
672 
673  for (i = 0; i < n_devs; i++)
674  {
675  vlib_pci_addr_t addr;
676 
677  vec_reset_length (s);
678  s = format (s, "%s/device%c", dev_list[i]->dev_path, 0);
679 
680  if (sysfs_path_to_pci_addr ((char *) s, &addr) == 0)
681  continue;
682 
683  if (addr.as_u32 != rd->pci->addr.as_u32)
684  continue;
685 
686  if ((rd->ctx = ibv_open_device (dev_list[i])))
687  break;
688  }
689 
690  if ((args->error =
691  rdma_dev_init (vm, rd, args->rxq_size, args->txq_size, args->rxq_num)))
692  goto err2;
693 
694  if ((args->error = rdma_register_interface (vnm, rd)))
695  goto err2;
696 
697  if ((args->error = rdma_async_event_init (rd)))
698  goto err3;
699 
700  rdma_update_state (vnm, rd, 1);
701 
703  args->sw_if_index = rd->sw_if_index = sw->sw_if_index;
704  /*
705  * FIXME: add support for interrupt mode
706  * vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, rd->hw_if_index);
707  * hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
708  */
710  rdma_input_node.index);
711  vec_foreach_index (qid, rd->rxqs)
713 
714  vec_free (s);
715  return;
716 
717 err3:
718  rdma_unregister_interface (vnm, rd);
719 err2:
720  rdma_dev_cleanup (rd);
721 err1:
722  ibv_free_device_list (dev_list);
723  vec_free (s);
724  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
725 err0:
726  vlib_log_err (rm->log_class, "%U", format_clib_error, args->error);
727 }
728 
729 void
731 {
734  rdma_dev_cleanup (rd);
735 }
736 
737 static clib_error_t *
739 {
740  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
741  rdma_main_t *rm = &rdma_main;
743  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
744 
745  if (rd->flags & RDMA_DEVICE_F_ERROR)
746  return clib_error_return (0, "device is in error state");
747 
748  if (is_up)
749  {
752  rd->flags |= RDMA_DEVICE_F_ADMIN_UP;
753  }
754  else
755  {
757  rd->flags &= ~RDMA_DEVICE_F_ADMIN_UP;
758  }
759  return 0;
760 }
761 
762 static void
764  u32 node_index)
765 {
766  rdma_main_t *rm = &rdma_main;
767  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
770  ~0 ==
772  vlib_node_add_next (vlib_get_main (), rdma_input_node.index, node_index);
773 }
774 
775 static char *rdma_tx_func_error_strings[] = {
776 #define _(n,s) s,
778 #undef _
779 };
780 
781 /* *INDENT-OFF* */
783 {
784  .name = "RDMA interface",
785  .format_device = format_rdma_device,
786  .format_device_name = format_rdma_device_name,
787  .admin_up_down_function = rdma_interface_admin_up_down,
788  .rx_redirect_to_node = rdma_set_interface_next_node,
789  .tx_function_n_errors = RDMA_TX_N_ERROR,
790  .tx_function_error_strings = rdma_tx_func_error_strings,
791  .mac_addr_change_function = rdma_mac_change,
792 };
793 /* *INDENT-ON* */
794 
795 clib_error_t *
797 {
798  rdma_main_t *rm = &rdma_main;
799 
800  rm->log_class = vlib_log_register_class ("rdma", 0);
801 
802  return 0;
803 }
804 
805 /* *INDENT-OFF* */
807 {
808  .runs_after = VLIB_INITS ("pci_bus_init"),
809 };
810 /* *INDENT-OFF* */
811 
812 /*
813  * fd.io coding-style-patch-verification: ON
814  *
815  * Local Variables:
816  * eval: (c-set-style "gnu")
817  * End:
818  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:176
struct ibv_mr * mr
Definition: rdma.h:87
#define vec_foreach_index(var, v)
Iterate over vector indices.
u8 * format_clib_error(u8 *s, va_list *va)
Definition: error.c:191
u8 * linux_ifname
Definition: rdma.h:80
vl_api_mac_address_t mac
Definition: l2.api:491
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:378
static clib_error_t * rdma_rxq_finalize(vlib_main_t *vm, rdma_device_t *rd)
Definition: device.c:431
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
Definition: pool.h:240
format_function_t format_rdma_device
Definition: rdma.h:133
u32 size
Definition: rdma.h:46
#define NULL
Definition: clib.h:58
vlib_pci_device_info_t * pci
Definition: rdma.h:78
u32 dev_instance
Definition: rdma.h:83
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 file_descriptor
Definition: file.h:54
struct ibv_wq * wq
Definition: rdma.h:44
u32 size
Definition: rdma.h:58
u32 per_interface_next_index
Definition: rdma.h:71
static void vlib_pci_free_device_info(vlib_pci_device_info_t *di)
Definition: pci.h:114
int i
vlib_buffer_main_t * buffer_main
Definition: main.h:152
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
vl_api_fib_path_t path
Definition: mfib_types.api:34
static void rdma_unregister_interface(vnet_main_t *vnm, rdma_device_t *rd)
Definition: device.c:348
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:451
vhost_vring_addr_t addr
Definition: vhost_user.h:147
mac_address_t hwaddr
Definition: rdma.h:81
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1092
unsigned char u8
Definition: types.h:56
static uword min_log2(uword x)
Definition: clib.h:144
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
vlib_log_class_t log_class
Definition: rdma.h:99
static clib_error_t * rdma_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:738
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
static u32 rdma_dev_change_mtu(rdma_device_t *rd)
Definition: device.c:171
struct ibv_pd * pd
Definition: rdma.h:86
static void rdma_update_state(vnet_main_t *vnm, rdma_device_t *rd, int port)
Definition: device.c:198
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:312
rdma_device_t * devices
Definition: rdma.h:98
static clib_error_t * rdma_txq_init(vlib_main_t *vm, rdma_device_t *rd, u16 qid, u32 n_desc)
Definition: device.c:474
uword buffer_mem_size
Definition: buffer.h:453
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
struct ibv_cq * cq
Definition: rdma.h:55
#define clib_error_return(e, args...)
Definition: error.h:99
clib_file_main_t file_main
Definition: main.c:63
#define vlib_log_emerg(...)
Definition: log.h:99
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:378
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:1029
u32 flags
Definition: rdma.h:70
static clib_error_t * rdma_async_event_init(rdma_device_t *rd)
Definition: device.c:308
u32 * bufs
Definition: rdma.h:45
vlib_pci_device_info_t * vlib_pci_get_device_info(vlib_main_t *vm, vlib_pci_addr_t *addr, clib_error_t **error)
Definition: pci.c:202
u8 * description
Definition: file.h:70
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:519
static_always_inline void mac_address_from_bytes(mac_address_t *mac, const u8 *bytes)
Definition: mac_address.h:92
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
u8 pool
Definition: rdma.h:75
vlib_node_registration_t rdma_input_node
(constructor) VLIB_REGISTER_NODE (rdma_input_node)
Definition: input.c:361
#define clib_error_return_unix(e, args...)
Definition: error.h:102
struct ibv_cq * cq
Definition: rdma.h:43
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:287
unformat_function_t unformat_vlib_pci_addr
Definition: pci.h:323
#define rdma_log(lvl, dev, f,...)
Definition: device.c:54
static uword sysfs_path_to_pci_addr(char *path, vlib_pci_addr_t *addr)
Definition: device.c:562
static void rdma_async_event_cleanup(rdma_device_t *rd)
Definition: device.c:334
static clib_error_t * rdma_dev_init(vlib_main_t *vm, rdma_device_t *rd, u32 rxq_size, u32 txq_size, u32 rxq_num)
Definition: device.c:521
struct ibv_qp * qp
Definition: rdma.h:56
static clib_error_t * rdma_rxq_init(vlib_main_t *vm, rdma_device_t *rd, u16 qid, u32 n_desc)
Definition: device.c:398
vlib_main_t * vm
Definition: in2out_ed.c:1810
static clib_error_t * rdma_mac_change(vnet_hw_interface_t *hw, const u8 *old, const u8 *new)
Definition: device.c:157
u32 hw_if_index
Definition: rdma.h:73
static u32 rdma_dev_set_promisc(rdma_device_t *rd)
Definition: device.c:102
u32 flags
Definition: vhost_user.h:141
struct ibv_rwq_ind_table * rx_rwq_ind_tbl
Definition: rdma.h:89
clib_error_t * error
Definition: rdma.h:93
rdma_main_t rdma_main
Definition: device.c:46
clib_error_t * error
Definition: rdma.h:124
rdma_mode_t mode
Definition: rdma.h:119
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:342
u8 * driver_name
Definition: pci.h:82
static clib_error_t * rdma_register_interface(vnet_main_t *vnm, rdma_device_t *rd)
Definition: device.c:340
static clib_error_t * rdma_async_event_error_ready(clib_file_t *f)
Definition: device.c:265
void rdma_create_if(vlib_main_t *vm, rdma_create_if_args_t *args)
Definition: device.c:580
u32 lkey
Definition: rdma.h:74
#define ETHERNET_INTERFACE_FLAG_MTU
Definition: ethernet.h:146
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:141
static u8 rdma_rss_hash_key[]
Definition: device.c:33
u32 sw_if_index
Definition: rdma.h:72
static clib_error_t * rdma_async_event_read_ready(clib_file_t *f)
Definition: device.c:273
#define ASSERT(truth)
format_function_t format_rdma_device_name
Definition: rdma.h:134
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
static void rdma_dev_cleanup(rdma_device_t *rd)
Definition: device.c:356
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
Definition: file.h:119
u8 * name
Definition: rdma.h:79
rdma_txq_t * txqs
Definition: rdma.h:69
static struct ibv_flow * rdma_rxq_init_flow(const rdma_device_t *rd, struct ibv_qp *qp, const mac_address_t *mac, const mac_address_t *mask, u32 flags)
Definition: device.c:58
#define rdma_log__(lvl, dev, f,...)
Definition: device.c:48
vlib_pci_addr_t addr
Definition: pci.h:66
rdma_rxq_t * rxqs
Definition: rdma.h:68
clib_error_t * rdma_init(vlib_main_t *vm)
Definition: device.c:796
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
vl_api_ip4_address_t hi
Definition: arp.api:37
vnet_device_class_t rdma_device_class
static uword is_pow2(uword x)
Definition: clib.h:235
u32 async_event_clib_file_index
Definition: rdma.h:82
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static void ethernet_mac_address_generate(u8 *mac)
Definition: mac_address.h:74
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:331
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:492
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:163
static u32 rdma_dev_set_ucast(rdma_device_t *rd)
Definition: device.c:124
uword buffer_mem_start
Definition: buffer.h:452
u32 * bufs
Definition: rdma.h:57
#define foreach_rdma_tx_func_error
Definition: rdma.h:144
void rdma_delete_if(vlib_main_t *vm, rdma_device_t *rd)
Definition: device.c:730
#define clib_error_free(e)
Definition: error.h:86
u16 port
Definition: lb_types.api:72
clib_file_function_t * error_function
Definition: file.h:67
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
struct ibv_flow * flow_ucast
Definition: rdma.h:90
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static char * rdma_tx_func_error_strings[]
Definition: device.c:775
#define vec_foreach(var, vec)
Vector iterator.
uword private_data
Definition: file.h:64
#define vlib_log_err(...)
Definition: log.h:102
Definition: file.h:51
struct ibv_flow * flow_mcast
Definition: rdma.h:91
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static u32 rdma_rxq_destroy_flow(const rdma_device_t *rd, struct ibv_flow **flow)
Definition: device.c:86
#define STATIC_ASSERT_SIZEOF(d, s)
static u8 vlib_buffer_pool_get_default_for_numa(vlib_main_t *vm, u32 numa_node)
Definition: buffer_funcs.h:163
static u32 rdma_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: device.c:178
#define VLIB_INITS(...)
Definition: init.h:344
static void vnet_hw_interface_set_link_speed(vnet_main_t *vnm, u32 hw_if_index, u32 link_speed)
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
struct ibv_context * ctx
Definition: rdma.h:85
icmpr_flow_t * flow
Definition: main.c:123
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
u8 * clib_sysfs_link_to_name(char *link)
Definition: sysfs.c:90
struct ibv_qp * rx_qp
Definition: rdma.h:88
static void rdma_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:763
VNET_DEVICE_CLASS(avf_device_class,)