FD.io VPP  v19.08-27-gf4dcae4
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <unistd.h>
19 #include <fcntl.h>
20 #include <net/if.h>
21 #include <linux/if_link.h>
22 #include <linux/if_ether.h>
23 
24 #include <vppinfra/linux/sysfs.h>
25 #include <vlib/vlib.h>
26 #include <vlib/unix/unix.h>
27 #include <vlib/pci/pci.h>
28 #include <vnet/ethernet/ethernet.h>
29 
30 #include <rdma/rdma.h>
31 
32 /* Default RSS hash key (from DPDK MLX driver) */
33 static u8 rdma_rss_hash_key[] = {
34  0x2c, 0xc6, 0x81, 0xd1,
35  0x5b, 0xdb, 0xf4, 0xf7,
36  0xfc, 0xa2, 0x83, 0x19,
37  0xdb, 0x1a, 0x3e, 0x94,
38  0x6b, 0x9e, 0x38, 0xd9,
39  0x2c, 0x9c, 0x03, 0xd1,
40  0xad, 0x99, 0x44, 0xa7,
41  0xd9, 0x56, 0x3d, 0x59,
42  0x06, 0x3c, 0x25, 0xf3,
43  0xfc, 0x1f, 0xdc, 0x2a,
44 };
45 
47 
48 #define rdma_log__(lvl, dev, f, ...) \
49  do { \
50  vlib_log((lvl), rdma_main.log_class, "%s: " f, \
51  &(dev)->name, ##__VA_ARGS__); \
52  } while (0)
53 
54 #define rdma_log(lvl, dev, f, ...) \
55  rdma_log__((lvl), (dev), "%s (%d): " f, strerror(errno), errno, ##__VA_ARGS__)
56 
57 static struct ibv_flow *
58 rdma_rxq_init_flow (const rdma_device_t * rd, struct ibv_qp *qp,
59  const mac_address_t * mac, const mac_address_t * mask,
60  u32 flags)
61 {
62  struct ibv_flow *flow;
63  struct raw_eth_flow_attr
64  {
65  struct ibv_flow_attr attr;
66  struct ibv_flow_spec_eth spec_eth;
67  } __attribute__ ((packed)) fa;
68 
69  memset (&fa, 0, sizeof (fa));
70  fa.attr.num_of_specs = 1;
71  fa.attr.port = 1;
72  fa.attr.flags = flags;
73  fa.spec_eth.type = IBV_FLOW_SPEC_ETH;
74  fa.spec_eth.size = sizeof (struct ibv_flow_spec_eth);
75 
76  memcpy (fa.spec_eth.val.dst_mac, mac, sizeof (fa.spec_eth.val.dst_mac));
77  memcpy (fa.spec_eth.mask.dst_mac, mask, sizeof (fa.spec_eth.mask.dst_mac));
78 
79  flow = ibv_create_flow (qp, &fa.attr);
80  if (!flow)
81  rdma_log (VLIB_LOG_LEVEL_ERR, rd, "ibv_create_flow() failed");
82  return flow;
83 }
84 
85 static u32
86 rdma_rxq_destroy_flow (const rdma_device_t * rd, struct ibv_flow **flow)
87 {
88  if (!*flow)
89  return 0;
90 
91  if (ibv_destroy_flow (*flow))
92  {
93  rdma_log (VLIB_LOG_LEVEL_ERR, rd, "ibv_destroy_flow() failed");
94  return ~0;
95  }
96 
97  *flow = 0;
98  return 0;
99 }
100 
101 static u32
103 {
104  const mac_address_t all = {.bytes = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0} };
105  int err;
106 
107  err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast);
108  if (err)
109  return ~0;
110 
111  err = rdma_rxq_destroy_flow (rd, &rd->flow_ucast);
112  if (err)
113  return ~0;
114 
115  rd->flow_ucast = rdma_rxq_init_flow (rd, rd->rx_qp, &all, &all, 0);
116  if (!rd->flow_ucast)
117  return ~0;
118 
119  rd->flags |= RDMA_DEVICE_F_PROMISC;
120  return 0;
121 }
122 
123 static u32
125 {
126  const mac_address_t ucast = {.bytes = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
127  };
128  const mac_address_t mcast = {.bytes = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0} };
129  int err;
130 
131  err = rdma_rxq_destroy_flow (rd, &rd->flow_mcast);
132  if (err)
133  return ~0;
134 
135  err = rdma_rxq_destroy_flow (rd, &rd->flow_ucast);
136  if (err)
137  return ~0;
138 
139  /* receive only packets with src = our MAC */
140  rd->flow_ucast = rdma_rxq_init_flow (rd, rd->rx_qp, &rd->hwaddr, &ucast, 0);
141  if (!rd->flow_ucast)
142  return ~0;
143 
144  /* receive multicast packets */
145  rd->flow_mcast = rdma_rxq_init_flow (rd, rd->rx_qp, &mcast, &mcast,
146  IBV_FLOW_ATTR_FLAGS_DONT_TRAP
147  /* let others receive mcast packet too (eg. Linux) */
148  );
149  if (!rd->flow_mcast)
150  return ~0;
151 
152  rd->flags &= ~RDMA_DEVICE_F_PROMISC;
153  return 0;
154 }
155 
156 static u32
158 {
159  rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "MTU change not supported");
160  return ~0;
161 }
162 
163 static u32
165 {
166  rdma_main_t *rm = &rdma_main;
168 
169  switch (flags)
170  {
171  case 0:
172  return rdma_dev_set_ucast (rd);
174  return rdma_dev_set_promisc (rd);
176  return rdma_dev_change_mtu (rd);
177  }
178 
179  rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "unknown flag %x requested", flags);
180  return ~0;
181 }
182 
183 static void
185 {
186  struct ibv_port_attr attr;
187  u32 width = 0;
188  u32 speed = 0;
189 
190  if (ibv_query_port (rd->ctx, port, &attr))
191  {
194  return;
195  }
196 
197  /* update state */
198  switch (attr.state)
199  {
200  case IBV_PORT_ACTIVE: /* fallthrough */
201  case IBV_PORT_ACTIVE_DEFER:
202  rd->flags |= RDMA_DEVICE_F_LINK_UP;
205  break;
206  default:
207  rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
209  break;
210  }
211 
212  /* update speed */
213  switch (attr.active_width)
214  {
215  case 1:
216  width = 1;
217  break;
218  case 2:
219  width = 4;
220  break;
221  case 4:
222  width = 8;
223  break;
224  case 8:
225  width = 12;
226  break;
227  }
228  switch (attr.active_speed)
229  {
230  case 1:
231  speed = 2500000;
232  break;
233  case 2:
234  speed = 5000000;
235  break;
236  case 4: /* fallthrough */
237  case 8:
238  speed = 10000000;
239  break;
240  case 16:
241  speed = 14000000;
242  break;
243  case 32:
244  speed = 25000000;
245  break;
246  }
247  vnet_hw_interface_set_link_speed (vnm, rd->hw_if_index, width * speed);
248 }
249 
250 static clib_error_t *
252 {
253  rdma_main_t *rm = &rdma_main;
255  return clib_error_return (0, "RDMA async event error for device %U",
257 }
258 
259 static clib_error_t *
261 {
262  vnet_main_t *vnm = vnet_get_main ();
263  rdma_main_t *rm = &rdma_main;
265  int ret;
266  struct ibv_async_event event;
267  ret = ibv_get_async_event (rd->ctx, &event);
268  if (ret < 0)
269  return clib_error_return_unix (0, "ibv_get_async_event() failed");
270 
271  switch (event.event_type)
272  {
273  case IBV_EVENT_PORT_ACTIVE:
274  rdma_update_state (vnm, rd, event.element.port_num);
275  break;
276  case IBV_EVENT_PORT_ERR:
277  rdma_update_state (vnm, rd, event.element.port_num);
278  break;
279  case IBV_EVENT_DEVICE_FATAL:
280  rd->flags &= ~RDMA_DEVICE_F_LINK_UP;
282  vlib_log_emerg (rm->log_class, "Fatal RDMA error for device %U",
284  break;
285  default:
286  rdma_log__ (VLIB_LOG_LEVEL_ERR, rd, "unhandeld RDMA async event %i",
287  event.event_type);
288  break;
289  }
290 
291  ibv_ack_async_event (&event);
292  return 0;
293 }
294 
295 static clib_error_t *
297 {
298  clib_file_t t = { 0 };
299  int ret;
300 
301  /* make RDMA async event fd non-blocking */
302  ret = fcntl (rd->ctx->async_fd, F_GETFL);
303  if (ret < 0)
304  return clib_error_return_unix (0, "fcntl(F_GETFL) failed");
305 
306  ret = fcntl (rd->ctx->async_fd, F_SETFL, ret | O_NONBLOCK);
307  if (ret < 0)
308  return clib_error_return_unix (0, "fcntl(F_SETFL, O_NONBLOCK) failed");
309 
310  /* register RDMA async event fd */
312  t.file_descriptor = rd->ctx->async_fd;
314  t.private_data = rd->dev_instance;
315  t.description =
316  format (0, "RMDA %U async event", format_vlib_pci_addr, &rd->pci_addr);
317 
319  return 0;
320 }
321 
322 static void
324 {
326 }
327 
328 static clib_error_t *
330 {
332  rd->dev_instance, rd->hwaddr.bytes,
334 }
335 
336 static void
338 {
342 }
343 
344 static void
346 {
347  rdma_main_t *rm = &rdma_main;
348  rdma_rxq_t *rxq;
349  rdma_txq_t *txq;
350 
351 #define _(fn, arg) if (arg) \
352  { \
353  int rv; \
354  if ((rv = fn (arg))) \
355  rdma_log (VLIB_LOG_LEVEL_DEBUG, rd, #fn "() failed (rv = %d)", rv); \
356  }
357 
358  _(ibv_destroy_flow, rd->flow_mcast);
359  _(ibv_destroy_flow, rd->flow_ucast);
360  _(ibv_dereg_mr, rd->mr);
361  vec_foreach (txq, rd->txqs)
362  {
363  _(ibv_destroy_qp, txq->qp);
364  _(ibv_destroy_cq, txq->cq);
365  }
366  vec_foreach (rxq, rd->rxqs)
367  {
368  _(ibv_destroy_wq, rxq->wq);
369  _(ibv_destroy_cq, rxq->cq);
370  }
371  _(ibv_destroy_rwq_ind_table, rd->rx_rwq_ind_tbl);
372  _(ibv_destroy_qp, rd->rx_qp);
373  _(ibv_dealloc_pd, rd->pd);
374  _(ibv_close_device, rd->ctx);
375 #undef _
376 
377  clib_error_free (rd->error);
378 
379  vec_free (rd->rxqs);
380  vec_free (rd->txqs);
381  vec_free (rd->name);
382  pool_put (rm->devices, rd);
383 }
384 
385 static clib_error_t *
387 {
388  rdma_rxq_t *rxq;
389  struct ibv_wq_init_attr wqia;
390  struct ibv_wq_attr wqa;
391 
393  rxq = vec_elt_at_index (rd->rxqs, qid);
394  rxq->size = n_desc;
395 
396  if ((rxq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
397  return clib_error_return_unix (0, "Create CQ Failed");
398 
399  memset (&wqia, 0, sizeof (wqia));
400  wqia.wq_type = IBV_WQT_RQ;
401  wqia.max_wr = n_desc;
402  wqia.max_sge = 1;
403  wqia.pd = rd->pd;
404  wqia.cq = rxq->cq;
405  if ((rxq->wq = ibv_create_wq (rd->ctx, &wqia)) == 0)
406  return clib_error_return_unix (0, "Create WQ Failed");
407 
408  memset (&wqa, 0, sizeof (wqa));
409  wqa.attr_mask = IBV_WQ_ATTR_STATE;
410  wqa.wq_state = IBV_WQS_RDY;
411  if (ibv_modify_wq (rxq->wq, &wqa) != 0)
412  return clib_error_return_unix (0, "Modify WQ (RDY) Failed");
413 
414  return 0;
415 }
416 
417 static clib_error_t *
419 {
420  struct ibv_rwq_ind_table_init_attr rwqia;
421  struct ibv_qp_init_attr_ex qpia;
422  struct ibv_wq **ind_tbl;
423  u32 i;
424 
425  ASSERT (is_pow2 (vec_len (rd->rxqs))
426  && "rxq number should be a power of 2");
427 
428  ind_tbl = vec_new (struct ibv_wq *, vec_len (rd->rxqs));
429  vec_foreach_index (i, rd->rxqs)
430  ind_tbl[i] = vec_elt_at_index (rd->rxqs, i)->wq;
431  memset (&rwqia, 0, sizeof (rwqia));
432  rwqia.log_ind_tbl_size = min_log2 (vec_len (ind_tbl));
433  rwqia.ind_tbl = ind_tbl;
434  if ((rd->rx_rwq_ind_tbl = ibv_create_rwq_ind_table (rd->ctx, &rwqia)) == 0)
435  return clib_error_return_unix (0, "RWQ indirection table create failed");
436  vec_free (ind_tbl);
437 
438  memset (&qpia, 0, sizeof (qpia));
439  qpia.qp_type = IBV_QPT_RAW_PACKET;
440  qpia.comp_mask =
441  IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_IND_TABLE |
442  IBV_QP_INIT_ATTR_RX_HASH;
443  qpia.pd = rd->pd;
444  qpia.rwq_ind_tbl = rd->rx_rwq_ind_tbl;
446  qpia.rx_hash_conf.rx_hash_key_len = sizeof (rdma_rss_hash_key);
447  qpia.rx_hash_conf.rx_hash_key = rdma_rss_hash_key;
448  qpia.rx_hash_conf.rx_hash_function = IBV_RX_HASH_FUNC_TOEPLITZ;
449  qpia.rx_hash_conf.rx_hash_fields_mask =
450  IBV_RX_HASH_SRC_IPV4 | IBV_RX_HASH_DST_IPV4;
451  if ((rd->rx_qp = ibv_create_qp_ex (rd->ctx, &qpia)) == 0)
452  return clib_error_return_unix (0, "Queue Pair create failed");
453 
454  if (rdma_dev_set_ucast (rd))
455  return clib_error_return_unix (0, "Set unicast mode failed");
456 
457  return 0;
458 }
459 
460 static clib_error_t *
462 {
463  rdma_txq_t *txq;
464  struct ibv_qp_init_attr qpia;
465  struct ibv_qp_attr qpa;
466  int qp_flags;
467 
469  txq = vec_elt_at_index (rd->txqs, qid);
470  txq->size = n_desc;
471 
472  if ((txq->cq = ibv_create_cq (rd->ctx, n_desc, NULL, NULL, 0)) == 0)
473  return clib_error_return_unix (0, "Create CQ Failed");
474 
475  memset (&qpia, 0, sizeof (qpia));
476  qpia.send_cq = txq->cq;
477  qpia.recv_cq = txq->cq;
478  qpia.cap.max_send_wr = n_desc;
479  qpia.cap.max_send_sge = 1;
480  qpia.qp_type = IBV_QPT_RAW_PACKET;
481  qpia.sq_sig_all = 1;
482 
483  if ((txq->qp = ibv_create_qp (rd->pd, &qpia)) == 0)
484  return clib_error_return_unix (0, "Queue Pair create failed");
485 
486  memset (&qpa, 0, sizeof (qpa));
487  qp_flags = IBV_QP_STATE | IBV_QP_PORT;
488  qpa.qp_state = IBV_QPS_INIT;
489  qpa.port_num = 1;
490  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
491  return clib_error_return_unix (0, "Modify QP (init) Failed");
492 
493  memset (&qpa, 0, sizeof (qpa));
494  qp_flags = IBV_QP_STATE;
495  qpa.qp_state = IBV_QPS_RTR;
496  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
497  return clib_error_return_unix (0, "Modify QP (receive) Failed");
498 
499  memset (&qpa, 0, sizeof (qpa));
500  qp_flags = IBV_QP_STATE;
501  qpa.qp_state = IBV_QPS_RTS;
502  if (ibv_modify_qp (txq->qp, &qpa, qp_flags) != 0)
503  return clib_error_return_unix (0, "Modify QP (send) Failed");
504  return 0;
505 }
506 
507 static clib_error_t *
509  u32 txq_size, u32 rxq_num)
510 {
511  clib_error_t *err;
514  u32 i;
515 
516  if (rd->ctx == 0)
517  return clib_error_return_unix (0, "Device Open Failed");
518 
519  if ((rd->pd = ibv_alloc_pd (rd->ctx)) == 0)
520  return clib_error_return_unix (0, "PD Alloc Failed");
521 
523 
524  for (i = 0; i < rxq_num; i++)
525  if ((err = rdma_rxq_init (vm, rd, i, rxq_size)))
526  return err;
527  if ((err = rdma_rxq_finalize (vm, rd)))
528  return err;
529 
530  for (i = 0; i < tm->n_vlib_mains; i++)
531  if ((err = rdma_txq_init (vm, rd, i, txq_size)))
532  return err;
533 
534  if ((rd->mr = ibv_reg_mr (rd->pd, (void *) bm->buffer_mem_start,
535  bm->buffer_mem_size,
536  IBV_ACCESS_LOCAL_WRITE)) == 0)
537  return clib_error_return_unix (0, "Register MR Failed");
538 
539  return 0;
540 }
541 
542 static uword
543 sysfs_path_to_pci_addr (char *path, vlib_pci_addr_t * addr)
544 {
545  uword rv;
546  unformat_input_t in;
547  u8 *s;
548 
549  s = clib_sysfs_link_to_name (path);
550  unformat_init_string (&in, (char *) s, strlen ((char *) s));
551  rv = unformat (&in, "%U", unformat_vlib_pci_addr, addr);
552  unformat_free (&in);
553  vec_free (s);
554  return rv;
555 }
556 
557 void
559 {
560  vnet_main_t *vnm = vnet_get_main ();
561  rdma_main_t *rm = &rdma_main;
562  rdma_device_t *rd = 0;
563  struct ibv_device **dev_list = 0;
564  int n_devs;
565  u8 *s = 0, *s2 = 0;
566  u16 qid;
567 
568  args->rxq_size = args->rxq_size ? args->rxq_size : 2 * VLIB_FRAME_SIZE;
569  args->txq_size = args->txq_size ? args->txq_size : 2 * VLIB_FRAME_SIZE;
570  args->rxq_num = args->rxq_num ? args->rxq_num : 1;
571 
572  if (!is_pow2 (args->rxq_num))
573  {
574  args->rv = VNET_API_ERROR_INVALID_VALUE;
575  args->error =
576  clib_error_return (0, "rx queue number must be a power of two");
577  return;
578  }
579 
580  if (!is_pow2 (args->rxq_size) || !is_pow2 (args->txq_size))
581  {
582  args->rv = VNET_API_ERROR_INVALID_VALUE;
583  args->error =
584  clib_error_return (0, "queue size must be a power of two");
585  return;
586  }
587 
588  pool_get_zero (rm->devices, rd);
589  rd->dev_instance = rd - rm->devices;
591  rd->name = vec_dup (args->name);
592 
593  /* check if device exist and if it is bound to mlx5_core */
594  s = format (s, "/sys/class/net/%s/device/driver/module%c", args->ifname, 0);
595  s2 = clib_sysfs_link_to_name ((char *) s);
596 
597  if (s2 == 0 || strncmp ((char *) s2, "mlx5_core", 9) != 0)
598  {
599  args->error =
601  "invalid interface (only mlx5 supported for now)");
602  goto err0;
603  }
604 
605  /* extract PCI address */
606  vec_reset_length (s);
607  s = format (s, "/sys/class/net/%s/device%c", args->ifname, 0);
608  if (sysfs_path_to_pci_addr ((char *) s, &rd->pci_addr) == 0)
609  {
610  args->error = clib_error_return (0, "cannot find PCI address");
611  goto err0;
612  }
613 
614  dev_list = ibv_get_device_list (&n_devs);
615  if (n_devs == 0)
616  {
617  args->error =
619  "no RDMA devices available, errno = %d. "
620  "Is the ib_uverbs module loaded?", errno);
621  goto err0;
622  }
623 
624  for (int i = 0; i < n_devs; i++)
625  {
626  vlib_pci_addr_t addr;
627 
628  vec_reset_length (s);
629  s = format (s, "%s/device%c", dev_list[i]->dev_path, 0);
630 
631  if (sysfs_path_to_pci_addr ((char *) s, &addr) == 0)
632  continue;
633 
634  if (addr.as_u32 != rd->pci_addr.as_u32)
635  continue;
636 
637  if ((rd->ctx = ibv_open_device (dev_list[i])))
638  break;
639  }
640 
641  if ((args->error =
642  rdma_dev_init (vm, rd, args->rxq_size, args->txq_size, args->rxq_num)))
643  goto err1;
644 
645  if ((args->error = rdma_register_interface (vnm, rd)))
646  goto err2;
647 
648  if ((args->error = rdma_async_event_init (rd)))
649  goto err3;
650 
651  rdma_update_state (vnm, rd, 1);
652 
654  args->sw_if_index = rd->sw_if_index = sw->sw_if_index;
655  /*
656  * FIXME: add support for interrupt mode
657  * vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, rd->hw_if_index);
658  * hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
659  */
661  rdma_input_node.index);
662  vec_foreach_index (qid, rd->rxqs)
664  return;
665 
666 err3:
667  rdma_unregister_interface (vnm, rd);
668 err2:
669  rdma_dev_cleanup (rd);
670 err1:
671  ibv_free_device_list (dev_list);
672 err0:
673  vec_free (s2);
674  vec_free (s);
675  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
676  vlib_log_err (rm->log_class, "%U", format_clib_error, args->error);
677 }
678 
679 void
681 {
684  rdma_dev_cleanup (rd);
685 }
686 
687 static clib_error_t *
689 {
690  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
691  rdma_main_t *rm = &rdma_main;
693  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
694 
695  if (rd->flags & RDMA_DEVICE_F_ERROR)
696  return clib_error_return (0, "device is in error state");
697 
698  if (is_up)
699  {
702  rd->flags |= RDMA_DEVICE_F_ADMIN_UP;
703  }
704  else
705  {
707  rd->flags &= ~RDMA_DEVICE_F_ADMIN_UP;
708  }
709  return 0;
710 }
711 
712 static void
714  u32 node_index)
715 {
716  rdma_main_t *rm = &rdma_main;
717  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
719 
720  /* Shut off redirection */
721  if (node_index == ~0)
722  {
723  rd->per_interface_next_index = node_index;
724  return;
725  }
726 
728  vlib_node_add_next (vlib_get_main (), rdma_input_node.index, node_index);
729 }
730 
731 static char *rdma_tx_func_error_strings[] = {
732 #define _(n,s) s,
734 #undef _
735 };
736 
737 /* *INDENT-OFF* */
739 {
740  .name = "RDMA interface",
741  .format_device = format_rdma_device,
742  .format_device_name = format_rdma_device_name,
743  .admin_up_down_function = rdma_interface_admin_up_down,
744  .rx_redirect_to_node = rdma_set_interface_next_node,
745  .tx_function_n_errors = RDMA_TX_N_ERROR,
746  .tx_function_error_strings = rdma_tx_func_error_strings,
747 };
748 /* *INDENT-ON* */
749 
750 clib_error_t *
752 {
753  rdma_main_t *rm = &rdma_main;
754 
755  rm->log_class = vlib_log_register_class ("rdma", 0);
756 
757  return 0;
758 }
759 
761 
762 /*
763  * fd.io coding-style-patch-verification: ON
764  *
765  * Local Variables:
766  * eval: (c-set-style "gnu")
767  * End:
768  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:176
struct ibv_mr * mr
Definition: rdma.h:77
vmrglw vmrglh hi
#define vec_foreach_index(var, v)
Iterate over vector indices.
u8 * format_clib_error(u8 *s, va_list *va)
Definition: error.c:191
u32 flags
Definition: vhost_user.h:141
vl_api_mac_address_t mac
Definition: l2.api:490
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:324
static clib_error_t * rdma_rxq_finalize(vlib_main_t *vm, rdma_device_t *rd)
Definition: device.c:418
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
#define pool_get_zero(P, E)
Allocate an object E from a pool P and zero it.
Definition: pool.h:239
format_function_t format_rdma_device
Definition: rdma.h:116
u32 size
Definition: rdma.h:40
#define NULL
Definition: clib.h:58
u32 dev_instance
Definition: rdma.h:62
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
u32 file_descriptor
Definition: file.h:54
struct ibv_wq * wq
Definition: rdma.h:43
u32 size
Definition: rdma.h:49
u32 per_interface_next_index
Definition: rdma.h:60
int i
vlib_buffer_main_t * buffer_main
Definition: main.h:134
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
static void rdma_unregister_interface(vnet_main_t *vnm, rdma_device_t *rd)
Definition: device.c:337
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
vhost_vring_addr_t addr
Definition: vhost_user.h:147
mac_address_t hwaddr
Definition: rdma.h:72
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1092
unsigned char u8
Definition: types.h:56
static uword min_log2(uword x)
Definition: clib.h:144
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
clib_file_function_t * read_function
Definition: file.h:67
vlib_pci_addr_t pci_addr
Definition: rdma.h:73
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
vlib_log_class_t log_class
Definition: rdma.h:90
static clib_error_t * rdma_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:688
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
static u32 rdma_dev_change_mtu(rdma_device_t *rd)
Definition: device.c:157
struct ibv_pd * pd
Definition: rdma.h:76
static void rdma_update_state(vnet_main_t *vnm, rdma_device_t *rd, int port)
Definition: device.c:184
#define vec_new(T, N)
Create new vector of given type and length (unspecified alignment, no header).
Definition: vec.h:311
rdma_device_t * devices
Definition: rdma.h:89
static clib_error_t * rdma_txq_init(vlib_main_t *vm, rdma_device_t *rd, u16 qid, u32 n_desc)
Definition: device.c:461
uword buffer_mem_size
Definition: buffer.h:450
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
struct ibv_cq * cq
Definition: rdma.h:51
#define clib_error_return(e, args...)
Definition: error.h:99
clib_file_main_t file_main
Definition: main.c:63
#define vlib_log_emerg(...)
Definition: log.h:99
unsigned int u32
Definition: types.h:88
#define VLIB_FRAME_SIZE
Definition: node.h:376
void unformat_init_string(unformat_input_t *input, char *string, int string_len)
Definition: unformat.c:1029
u32 flags
Definition: rdma.h:59
static clib_error_t * rdma_async_event_init(rdma_device_t *rd)
Definition: device.c:296
u8 * description
Definition: file.h:70
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
struct _unformat_input_t unformat_input_t
unsigned short u16
Definition: types.h:57
vlib_node_registration_t rdma_input_node
(constructor) VLIB_REGISTER_NODE (rdma_input_node)
Definition: input.c:358
#define clib_error_return_unix(e, args...)
Definition: error.h:102
struct ibv_cq * cq
Definition: rdma.h:42
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:286
unformat_function_t unformat_vlib_pci_addr
Definition: pci.h:323
#define vec_dup(V)
Return copy of vector (no header, no alignment)
Definition: vec.h:375
#define rdma_log(lvl, dev, f,...)
Definition: device.c:54
static uword sysfs_path_to_pci_addr(char *path, vlib_pci_addr_t *addr)
Definition: device.c:543
static void rdma_async_event_cleanup(rdma_device_t *rd)
Definition: device.c:323
static clib_error_t * rdma_dev_init(vlib_main_t *vm, rdma_device_t *rd, u32 rxq_size, u32 txq_size, u32 rxq_num)
Definition: device.c:508
u16 port
Definition: punt.api:40
struct ibv_qp * qp
Definition: rdma.h:52
static clib_error_t * rdma_rxq_init(vlib_main_t *vm, rdma_device_t *rd, u16 qid, u32 n_desc)
Definition: device.c:386
u32 hw_if_index
Definition: rdma.h:64
static u32 rdma_dev_set_promisc(rdma_device_t *rd)
Definition: device.c:102
struct ibv_rwq_ind_table * rx_rwq_ind_tbl
Definition: rdma.h:79
clib_error_t * error
Definition: rdma.h:84
rdma_main_t rdma_main
Definition: device.c:46
vlib_main_t * vm
Definition: buffer.c:312
clib_error_t * error
Definition: rdma.h:106
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
static clib_error_t * rdma_register_interface(vnet_main_t *vnm, rdma_device_t *rd)
Definition: device.c:329
static clib_error_t * rdma_async_event_error_ready(clib_file_t *f)
Definition: device.c:251
void rdma_create_if(vlib_main_t *vm, rdma_create_if_args_t *args)
Definition: device.c:558
#define ETHERNET_INTERFACE_FLAG_MTU
Definition: ethernet.h:146
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:141
static u8 rdma_rss_hash_key[]
Definition: device.c:33
u32 sw_if_index
Definition: rdma.h:63
static clib_error_t * rdma_async_event_read_ready(clib_file_t *f)
Definition: device.c:260
#define ASSERT(truth)
format_function_t format_rdma_device_name
Definition: rdma.h:117
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
static void rdma_dev_cleanup(rdma_device_t *rd)
Definition: device.c:345
static uword clib_file_add(clib_file_main_t *um, clib_file_t *template)
Definition: file.h:96
static void clib_file_del_by_index(clib_file_main_t *um, uword index)
Definition: file.h:119
u8 * name
Definition: rdma.h:71
rdma_txq_t * txqs
Definition: rdma.h:69
static struct ibv_flow * rdma_rxq_init_flow(const rdma_device_t *rd, struct ibv_qp *qp, const mac_address_t *mac, const mac_address_t *mask, u32 flags)
Definition: device.c:58
#define rdma_log__(lvl, dev, f,...)
Definition: device.c:48
VNET_DEVICE_CLASS(bond_dev_class)
rdma_rxq_t * rxqs
Definition: rdma.h:68
clib_error_t * rdma_init(vlib_main_t *vm)
Definition: device.c:751
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
vnet_device_class_t rdma_device_class
static uword is_pow2(uword x)
Definition: clib.h:235
u32 async_event_clib_file_index
Definition: rdma.h:66
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
static void ethernet_mac_address_generate(u8 *mac)
Definition: mac_address.h:74
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:278
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:501
u64 uword
Definition: types.h:112
static void unformat_free(unformat_input_t *i)
Definition: format.h:163
static u32 rdma_dev_set_ucast(rdma_device_t *rd)
Definition: device.c:124
uword buffer_mem_start
Definition: buffer.h:449
#define foreach_rdma_tx_func_error
Definition: rdma.h:126
void rdma_delete_if(vlib_main_t *vm, rdma_device_t *rd)
Definition: device.c:680
#define clib_error_free(e)
Definition: error.h:86
clib_file_function_t * error_function
Definition: file.h:67
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
struct ibv_flow * flow_ucast
Definition: rdma.h:80
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static char * rdma_tx_func_error_strings[]
Definition: device.c:731
#define vec_foreach(var, vec)
Vector iterator.
uword private_data
Definition: file.h:64
#define vlib_log_err(...)
Definition: log.h:102
Definition: file.h:51
struct ibv_flow * flow_mcast
Definition: rdma.h:81
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static u32 rdma_rxq_destroy_flow(const rdma_device_t *rd, struct ibv_flow **flow)
Definition: device.c:86
#define STATIC_ASSERT_SIZEOF(d, s)
static u32 rdma_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: device.c:164
static void vnet_hw_interface_set_link_speed(vnet_main_t *vnm, u32 hw_if_index, u32 link_speed)
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
struct ibv_context * ctx
Definition: rdma.h:75
icmpr_flow_t * flow
Definition: main.c:123
format_function_t format_vlib_pci_addr
Definition: pci.h:324
uword unformat(unformat_input_t *i, const char *fmt,...)
Definition: unformat.c:978
u8 * clib_sysfs_link_to_name(char *link)
Definition: sysfs.c:90
struct ibv_qp * rx_qp
Definition: rdma.h:78
static void rdma_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:713