FD.io VPP  v20.09-64-g4f7b92f0a
Vector Packet Processing
device.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2018 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vppinfra/ring.h>
20 #include <vlib/unix/unix.h>
21 #include <vlib/pci/pci.h>
22 #include <vnet/ethernet/ethernet.h>
23 
24 #include <avf/avf.h>
25 
26 #define AVF_MBOX_LEN 64
27 #define AVF_MBOX_BUF_SZ 512
28 #define AVF_RXQ_SZ 512
29 #define AVF_TXQ_SZ 512
30 #define AVF_ITR_INT 250
31 
32 #define PCI_VENDOR_ID_INTEL 0x8086
33 #define PCI_DEVICE_ID_INTEL_AVF 0x1889
34 #define PCI_DEVICE_ID_INTEL_X710_VF 0x154c
35 #define PCI_DEVICE_ID_INTEL_X722_VF 0x37cd
36 
38 void avf_delete_if (vlib_main_t * vm, avf_device_t * ad, int with_barrier);
39 
40 static pci_device_id_t avf_pci_device_ids[] = {
42  {.vendor_id = PCI_VENDOR_ID_INTEL,.device_id = PCI_DEVICE_ID_INTEL_X710_VF},
43  {.vendor_id = PCI_VENDOR_ID_INTEL,.device_id = PCI_DEVICE_ID_INTEL_X722_VF},
44  {0},
45 };
46 
47 const static char *virtchnl_event_names[] = {
48 #define _(v, n) [v] = #n,
50 #undef _
51 };
52 
53 typedef enum
54 {
59 
60 static inline void
62 {
63  u32 dyn_ctl0 = 0, icr0_ena = 0;
64 
65  dyn_ctl0 |= (3 << 3); /* 11b = No ITR update */
66 
67  avf_reg_write (ad, AVFINT_ICR0_ENA1, icr0_ena);
68  avf_reg_write (ad, AVFINT_DYN_CTL0, dyn_ctl0);
69  avf_reg_flush (ad);
70 
71  if (state == AVF_IRQ_STATE_DISABLED)
72  return;
73 
74  dyn_ctl0 = 0;
75  icr0_ena = 0;
76 
77  icr0_ena |= (1 << 30); /* [30] Admin Queue Enable */
78 
79  dyn_ctl0 |= (1 << 0); /* [0] Interrupt Enable */
80  dyn_ctl0 |= (1 << 1); /* [1] Clear PBA */
81  dyn_ctl0 |= (2 << 3); /* [4:3] ITR Index, 11b = No ITR update */
82  dyn_ctl0 |= ((AVF_ITR_INT / 2) << 5); /* [16:5] ITR Interval in 2us steps */
83 
84  avf_reg_write (ad, AVFINT_ICR0_ENA1, icr0_ena);
85  avf_reg_write (ad, AVFINT_DYN_CTL0, dyn_ctl0);
86  avf_reg_flush (ad);
87 }
88 
89 static inline void
91 {
92  u32 dyn_ctln = 0;
93 
94  /* disable */
95  avf_reg_write (ad, AVFINT_DYN_CTLN (line), dyn_ctln);
96  avf_reg_flush (ad);
97 
98  if (state == AVF_IRQ_STATE_DISABLED)
99  return;
100 
101  dyn_ctln |= (1 << 1); /* [1] Clear PBA */
102  if (state == AVF_IRQ_STATE_WB_ON_ITR)
103  {
104  /* minimal ITR interval, use ITR1 */
105  dyn_ctln |= (1 << 3); /* [4:3] ITR Index */
106  dyn_ctln |= ((32 / 2) << 5); /* [16:5] ITR Interval in 2us steps */
107  dyn_ctln |= (1 << 30); /* [30] Writeback on ITR */
108  }
109  else
110  {
111  /* configured ITR interval, use ITR0 */
112  dyn_ctln |= (1 << 0); /* [0] Interrupt Enable */
113  dyn_ctln |= ((AVF_ITR_INT / 2) << 5); /* [16:5] ITR Interval in 2us steps */
114  }
115 
116  avf_reg_write (ad, AVFINT_DYN_CTLN (line), dyn_ctln);
117  avf_reg_flush (ad);
118 }
119 
120 
121 clib_error_t *
123  void *data, int len)
124 {
125  clib_error_t *err = 0;
126  avf_aq_desc_t *d, dc;
127  f64 t0, suspend_time = AVF_AQ_ENQ_SUSPEND_TIME;
128 
129  d = &ad->atq[ad->atq_next_slot];
130  clib_memcpy_fast (d, dt, sizeof (avf_aq_desc_t));
131  d->flags |= AVF_AQ_F_RD | AVF_AQ_F_SI;
132  if (len)
133  d->datalen = len;
134  if (len)
135  {
136  u64 pa;
137  pa = ad->atq_bufs_pa + ad->atq_next_slot * AVF_MBOX_BUF_SZ;
138  d->addr_hi = (u32) (pa >> 32);
139  d->addr_lo = (u32) pa;
141  data, len);
142  d->flags |= AVF_AQ_F_BUF;
143  }
144 
145  if (ad->flags & AVF_DEVICE_F_ELOG)
146  clib_memcpy_fast (&dc, d, sizeof (avf_aq_desc_t));
147 
149  ad->atq_next_slot = (ad->atq_next_slot + 1) % AVF_MBOX_LEN;
151  avf_reg_flush (ad);
152 
153  t0 = vlib_time_now (vm);
154 retry:
155  vlib_process_suspend (vm, suspend_time);
156 
157  if (((d->flags & AVF_AQ_F_DD) == 0) || ((d->flags & AVF_AQ_F_CMP) == 0))
158  {
159  f64 t = vlib_time_now (vm) - t0;
160  if (t > AVF_AQ_ENQ_MAX_WAIT_TIME)
161  {
162  avf_log_err (ad, "aq_desc_enq failed (timeout %.3fs)", t);
163  err = clib_error_return (0, "adminq enqueue timeout [opcode 0x%x]",
164  d->opcode);
165  goto done;
166  }
167  suspend_time *= 2;
168  goto retry;
169  }
170 
171  clib_memcpy_fast (dt, d, sizeof (avf_aq_desc_t));
172  if (d->flags & AVF_AQ_F_ERR)
173  return clib_error_return (0, "adminq enqueue error [opcode 0x%x, retval "
174  "%d]", d->opcode, d->retval);
175 
176 done:
177  if (ad->flags & AVF_DEVICE_F_ELOG)
178  {
179  /* *INDENT-OFF* */
180  ELOG_TYPE_DECLARE (el) =
181  {
182  .format = "avf[%d] aq enq: s_flags 0x%x r_flags 0x%x opcode 0x%x "
183  "datalen %d retval %d",
184  .format_args = "i4i2i2i2i2i2",
185  };
186  struct
187  {
188  u32 dev_instance;
189  u16 s_flags;
190  u16 r_flags;
191  u16 opcode;
192  u16 datalen;
193  u16 retval;
194  } *ed;
195  ed = ELOG_DATA (&vm->elog_main, el);
196  ed->dev_instance = ad->dev_instance;
197  ed->s_flags = dc.flags;
198  ed->r_flags = d->flags;
199  ed->opcode = dc.opcode;
200  ed->datalen = dc.datalen;
201  ed->retval = d->retval;
202  /* *INDENT-ON* */
203  }
204 
205  return err;
206 }
207 
208 clib_error_t *
210  u32 val)
211 {
212  clib_error_t *err;
213  avf_aq_desc_t d = {.opcode = 0x207,.param1 = reg,.param3 = val };
214  err = avf_aq_desc_enq (vm, ad, &d, 0, 0);
215 
216  if (ad->flags & AVF_DEVICE_F_ELOG)
217  {
218  /* *INDENT-OFF* */
219  ELOG_TYPE_DECLARE (el) =
220  {
221  .format = "avf[%d] rx ctl reg write: reg 0x%x val 0x%x ",
222  .format_args = "i4i4i4",
223  };
224  struct
225  {
226  u32 dev_instance;
227  u32 reg;
228  u32 val;
229  } *ed;
230  ed = ELOG_DATA (&vm->elog_main, el);
231  ed->dev_instance = ad->dev_instance;
232  ed->reg = reg;
233  ed->val = val;
234  /* *INDENT-ON* */
235  }
236  return err;
237 }
238 
239 clib_error_t *
240 avf_rxq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 rxq_size)
241 {
242  clib_error_t *err;
243  avf_rxq_t *rxq;
244  u32 n_alloc, i;
245 
247  rxq = vec_elt_at_index (ad->rxqs, qid);
248  rxq->size = rxq_size;
249  rxq->next = 0;
251  sizeof (avf_rx_desc_t),
253  ad->numa_node);
254 
255  rxq->buffer_pool_index =
257 
258  if (rxq->descs == 0)
259  return vlib_physmem_last_error (vm);
260 
261  if ((err = vlib_pci_map_dma (vm, ad->pci_dev_handle, (void *) rxq->descs)))
262  return err;
263 
264  clib_memset ((void *) rxq->descs, 0, rxq->size * sizeof (avf_rx_desc_t));
266  rxq->qrx_tail = ad->bar0 + AVF_QRX_TAIL (qid);
267 
268  n_alloc = vlib_buffer_alloc_from_pool (vm, rxq->bufs, rxq->size - 8,
269  rxq->buffer_pool_index);
270 
271  if (n_alloc == 0)
272  return clib_error_return (0, "buffer allocation error");
273 
274  rxq->n_enqueued = n_alloc;
275  avf_rx_desc_t *d = rxq->descs;
276  for (i = 0; i < n_alloc; i++)
277  {
278  vlib_buffer_t *b = vlib_get_buffer (vm, rxq->bufs[i]);
279  if (ad->flags & AVF_DEVICE_F_VA_DMA)
280  d->qword[0] = vlib_buffer_get_va (b);
281  else
282  d->qword[0] = vlib_buffer_get_pa (vm, b);
283  d++;
284  }
285 
286  ad->n_rx_queues = clib_min (ad->num_queue_pairs, qid + 1);
287  return 0;
288 }
289 
290 clib_error_t *
291 avf_txq_init (vlib_main_t * vm, avf_device_t * ad, u16 qid, u16 txq_size)
292 {
293  clib_error_t *err;
294  avf_txq_t *txq;
295 
296  if (qid >= ad->num_queue_pairs)
297  {
298  qid = qid % ad->num_queue_pairs;
299  txq = vec_elt_at_index (ad->txqs, qid);
300  if (txq->lock == 0)
301  clib_spinlock_init (&txq->lock);
302  ad->flags |= AVF_DEVICE_F_SHARED_TXQ_LOCK;
303  return 0;
304  }
305 
307  txq = vec_elt_at_index (ad->txqs, qid);
308  txq->size = txq_size;
309  txq->next = 0;
311  sizeof (avf_tx_desc_t),
313  ad->numa_node);
314  if (txq->descs == 0)
315  return vlib_physmem_last_error (vm);
316 
317  if ((err = vlib_pci_map_dma (vm, ad->pci_dev_handle, (void *) txq->descs)))
318  return err;
319 
321  txq->qtx_tail = ad->bar0 + AVF_QTX_TAIL (qid);
322 
323  /* initialize ring of pending RS slots */
325 
326  ad->n_tx_queues = clib_min (ad->num_queue_pairs, qid + 1);
327  return 0;
328 }
329 
330 typedef struct
331 {
335 
336 void
338 {
339  avf_aq_desc_t *d;
340  u64 pa = ad->arq_bufs_pa + slot * AVF_MBOX_BUF_SZ;
341  d = &ad->arq[slot];
342  clib_memset (d, 0, sizeof (avf_aq_desc_t));
343  d->flags = AVF_AQ_F_BUF;
345  d->addr_hi = (u32) (pa >> 32);
346  d->addr_lo = (u32) pa;
347 }
348 
349 static inline uword
351 {
352  return (ad->flags & AVF_DEVICE_F_VA_DMA) ?
354 }
355 
356 static void
358 {
359  u64 pa;
360  int i;
361 
362  /* VF MailBox Transmit */
363  clib_memset (ad->atq, 0, sizeof (avf_aq_desc_t) * AVF_MBOX_LEN);
364  ad->atq_bufs_pa = avf_dma_addr (vm, ad, ad->atq_bufs);
365 
366  pa = avf_dma_addr (vm, ad, ad->atq);
367  avf_reg_write (ad, AVF_ATQT, 0); /* Tail */
368  avf_reg_write (ad, AVF_ATQH, 0); /* Head */
369  avf_reg_write (ad, AVF_ATQLEN, AVF_MBOX_LEN | (1ULL << 31)); /* len & ena */
370  avf_reg_write (ad, AVF_ATQBAL, (u32) pa); /* Base Address Low */
371  avf_reg_write (ad, AVF_ATQBAH, (u32) (pa >> 32)); /* Base Address High */
372 
373  /* VF MailBox Receive */
374  clib_memset (ad->arq, 0, sizeof (avf_aq_desc_t) * AVF_MBOX_LEN);
375  ad->arq_bufs_pa = avf_dma_addr (vm, ad, ad->arq_bufs);
376 
377  for (i = 0; i < AVF_MBOX_LEN; i++)
378  avf_arq_slot_init (ad, i);
379 
380  pa = avf_dma_addr (vm, ad, ad->arq);
381 
382  avf_reg_write (ad, AVF_ARQH, 0); /* Head */
383  avf_reg_write (ad, AVF_ARQT, 0); /* Head */
384  avf_reg_write (ad, AVF_ARQLEN, AVF_MBOX_LEN | (1ULL << 31)); /* len & ena */
385  avf_reg_write (ad, AVF_ARQBAL, (u32) pa); /* Base Address Low */
386  avf_reg_write (ad, AVF_ARQBAH, (u32) (pa >> 32)); /* Base Address High */
387  avf_reg_write (ad, AVF_ARQT, AVF_MBOX_LEN - 1); /* Tail */
388 
389  ad->atq_next_slot = 0;
390  ad->arq_next_slot = 0;
391 }
392 
393 clib_error_t *
395  void *in, int in_len, void *out, int out_len)
396 {
397  clib_error_t *err;
398  avf_aq_desc_t *d, dt = {.opcode = 0x801,.v_opcode = op };
399  u32 head;
400  f64 t0, suspend_time = AVF_SEND_TO_PF_SUSPEND_TIME;
401 
402  /* adminq operations should be only done from process node after device
403  * is initialized */
404  ASSERT ((ad->flags & AVF_DEVICE_F_INITIALIZED) == 0 ||
406 
407  /* suppress interrupt in the next adminq receive slot
408  as we are going to wait for response
409  we only need interrupts when event is received */
410  d = &ad->arq[ad->arq_next_slot];
411  d->flags |= AVF_AQ_F_SI;
412 
413  if ((err = avf_aq_desc_enq (vm, ad, &dt, in, in_len)))
414  return err;
415 
416  t0 = vlib_time_now (vm);
417 retry:
418  head = avf_get_u32 (ad->bar0, AVF_ARQH);
419 
420  if (ad->arq_next_slot == head)
421  {
422  f64 t = vlib_time_now (vm) - t0;
424  {
425  avf_log_err (ad, "send_to_pf failed (timeout %.3fs)", t);
426  return clib_error_return (0, "timeout");
427  }
428  vlib_process_suspend (vm, suspend_time);
429  suspend_time *= 2;
430  goto retry;
431  }
432 
433  d = &ad->arq[ad->arq_next_slot];
434 
435  if (d->v_opcode == VIRTCHNL_OP_EVENT)
436  {
437  void *buf = ad->arq_bufs + ad->arq_next_slot * AVF_MBOX_BUF_SZ;
439 
440  if ((d->datalen != sizeof (virtchnl_pf_event_t)) ||
441  ((d->flags & AVF_AQ_F_BUF) == 0))
442  return clib_error_return (0, "event message error");
443 
444  vec_add2 (ad->events, e, 1);
445  clib_memcpy_fast (e, buf, sizeof (virtchnl_pf_event_t));
447  ad->arq_next_slot++;
448  /* reset timer */
449  t0 = vlib_time_now (vm);
450  suspend_time = AVF_SEND_TO_PF_SUSPEND_TIME;
451  goto retry;
452  }
453 
454  if (d->v_opcode != op)
455  {
456  err =
458  "unexpected message receiver [v_opcode = %u, "
459  "expected %u, v_retval %d]", d->v_opcode, op,
460  d->v_retval);
461  goto done;
462  }
463 
464  if (d->v_retval)
465  {
466  err = clib_error_return (0, "error [v_opcode = %u, v_retval %d]",
467  d->v_opcode, d->v_retval);
468  goto done;
469  }
470 
471  if (d->flags & AVF_AQ_F_BUF)
472  {
473  void *buf = ad->arq_bufs + ad->arq_next_slot * AVF_MBOX_BUF_SZ;
474  clib_memcpy_fast (out, buf, out_len);
475  }
476 
479  avf_reg_flush (ad);
480  ad->arq_next_slot = (ad->arq_next_slot + 1) % AVF_MBOX_LEN;
481 
482 done:
483 
484  if (ad->flags & AVF_DEVICE_F_ELOG)
485  {
486  /* *INDENT-OFF* */
487  ELOG_TYPE_DECLARE (el) =
488  {
489  .format = "avf[%d] send to pf: v_opcode %s (%d) v_retval 0x%x",
490  .format_args = "i4t4i4i4",
491  .n_enum_strings = VIRTCHNL_N_OPS,
492  .enum_strings = {
493 #define _(v, n) [v] = #n,
495 #undef _
496  },
497  };
498  struct
499  {
500  u32 dev_instance;
501  u32 v_opcode;
502  u32 v_opcode_val;
503  u32 v_retval;
504  } *ed;
505  ed = ELOG_DATA (&vm->elog_main, el);
506  ed->dev_instance = ad->dev_instance;
507  ed->v_opcode = op;
508  ed->v_opcode_val = op;
509  ed->v_retval = d->v_retval;
510  /* *INDENT-ON* */
511  }
512  return err;
513 }
514 
515 clib_error_t *
518 {
519  clib_error_t *err = 0;
520  virtchnl_version_info_t myver = {
522  .minor = VIRTCHNL_VERSION_MINOR,
523  };
524 
525  avf_log_debug (ad, "version: major %u minor %u", myver.major, myver.minor);
526 
527  err = avf_send_to_pf (vm, ad, VIRTCHNL_OP_VERSION, &myver,
528  sizeof (virtchnl_version_info_t), ver,
529  sizeof (virtchnl_version_info_t));
530 
531  if (err)
532  return err;
533 
534  return err;
535 }
536 
537 clib_error_t *
540 {
541  clib_error_t *err = 0;
542  u32 bitmap = (VIRTCHNL_VF_OFFLOAD_L2 | VIRTCHNL_VF_OFFLOAD_RSS_PF |
543  VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_VLAN |
544  VIRTCHNL_VF_OFFLOAD_RX_POLLING |
545  VIRTCHNL_VF_CAP_ADV_LINK_SPEED);
546 
547  avf_log_debug (ad, "get_vf_reqources: bitmap 0x%x", bitmap);
548  err = avf_send_to_pf (vm, ad, VIRTCHNL_OP_GET_VF_RESOURCES, &bitmap,
549  sizeof (u32), res, sizeof (virtchnl_vf_resource_t));
550 
551  if (err == 0)
552  {
553  int i;
554  avf_log_debug (ad, "get_vf_reqources: num_vsis %u num_queue_pairs %u "
555  "max_vectors %u max_mtu %u vf_offload_flags 0x%04x "
556  "rss_key_size %u rss_lut_size %u",
557  res->num_vsis, res->num_queue_pairs, res->max_vectors,
558  res->max_mtu, res->vf_offload_flags, res->rss_key_size,
559  res->rss_lut_size);
560  for (i = 0; i < res->num_vsis; i++)
561  avf_log_debug (ad, "get_vf_reqources_vsi[%u]: vsi_id %u "
562  "num_queue_pairs %u vsi_type %u qset_handle %u "
563  "default_mac_addr %U", i,
564  res->vsi_res[i].vsi_id,
565  res->vsi_res[i].num_queue_pairs,
566  res->vsi_res[i].vsi_type,
567  res->vsi_res[i].qset_handle,
569  res->vsi_res[i].default_mac_addr);
570  }
571 
572  return err;
573 }
574 
575 clib_error_t *
577 {
578  int msg_len = sizeof (virtchnl_rss_lut_t) + ad->rss_lut_size - 1;
579  int i;
580  u8 msg[msg_len];
581  virtchnl_rss_lut_t *rl;
582 
583  clib_memset (msg, 0, msg_len);
584  rl = (virtchnl_rss_lut_t *) msg;
585  rl->vsi_id = ad->vsi_id;
586  rl->lut_entries = ad->rss_lut_size;
587  for (i = 0; i < ad->rss_lut_size; i++)
588  rl->lut[i] = i % ad->n_rx_queues;
589 
590  avf_log_debug (ad, "config_rss_lut: vsi_id %u rss_lut_size %u lut 0x%U",
591  rl->vsi_id, rl->lut_entries, format_hex_bytes_no_wrap,
592  rl->lut, rl->lut_entries);
593 
594  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_CONFIG_RSS_LUT, msg, msg_len, 0,
595  0);
596 }
597 
598 clib_error_t *
600 {
601  int msg_len = sizeof (virtchnl_rss_key_t) + ad->rss_key_size - 1;
602  int i;
603  u8 msg[msg_len];
604  virtchnl_rss_key_t *rk;
605 
606  clib_memset (msg, 0, msg_len);
607  rk = (virtchnl_rss_key_t *) msg;
608  rk->vsi_id = ad->vsi_id;
609  rk->key_len = ad->rss_key_size;
610  u32 seed = random_default_seed ();
611  for (i = 0; i < ad->rss_key_size; i++)
612  rk->key[i] = (u8) random_u32 (&seed);
613 
614  avf_log_debug (ad, "config_rss_key: vsi_id %u rss_key_size %u key 0x%U",
615  rk->vsi_id, rk->key_len, format_hex_bytes_no_wrap, rk->key,
616  rk->key_len);
617 
618  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_CONFIG_RSS_KEY, msg, msg_len, 0,
619  0);
620 }
621 
622 clib_error_t *
624 {
625  avf_log_debug (ad, "disable_vlan_stripping");
626 
627  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_DISABLE_VLAN_STRIPPING, 0, 0, 0,
628  0);
629 }
630 
631 clib_error_t *
633 {
634  virtchnl_promisc_info_t pi = { 0 };
635 
636  pi.vsi_id = ad->vsi_id;
637 
638  if (is_enable)
639  pi.flags = FLAG_VF_UNICAST_PROMISC | FLAG_VF_MULTICAST_PROMISC;
640 
641  avf_log_debug (ad, "config_promisc_mode: unicast %s multicast %s",
642  pi.flags & FLAG_VF_UNICAST_PROMISC ? "on" : "off",
643  pi.flags & FLAG_VF_MULTICAST_PROMISC ? "on" : "off");
644 
645  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE, &pi,
646  sizeof (virtchnl_promisc_info_t), 0, 0);
647 }
648 
649 
650 clib_error_t *
652 {
653  int i;
654  int n_qp = clib_max (vec_len (ad->rxqs), vec_len (ad->txqs));
655  int msg_len = sizeof (virtchnl_vsi_queue_config_info_t) + n_qp *
657  u8 msg[msg_len];
659 
660  clib_memset (msg, 0, msg_len);
662  ci->vsi_id = ad->vsi_id;
663  ci->num_queue_pairs = n_qp;
664 
665  avf_log_debug (ad, "config_vsi_queues: vsi_id %u num_queue_pairs %u",
666  ad->vsi_id, ci->num_queue_pairs);
667 
668  for (i = 0; i < n_qp; i++)
669  {
670  virtchnl_txq_info_t *txq = &ci->qpair[i].txq;
671  virtchnl_rxq_info_t *rxq = &ci->qpair[i].rxq;
672 
673  rxq->vsi_id = ad->vsi_id;
674  rxq->queue_id = i;
676  if (i < vec_len (ad->rxqs))
677  {
678  avf_rxq_t *q = vec_elt_at_index (ad->rxqs, i);
679  rxq->ring_len = q->size;
681  rxq->dma_ring_addr = avf_dma_addr (vm, ad, (void *) q->descs);
682  avf_reg_write (ad, AVF_QRX_TAIL (i), q->size - 1);
683  }
684  avf_log_debug (ad, "config_vsi_queues_rx[%u]: max_pkt_size %u "
685  "ring_len %u databuffer_size %u dma_ring_addr 0x%llx",
686  i, rxq->max_pkt_size, rxq->ring_len,
687  rxq->databuffer_size, rxq->dma_ring_addr);
688 
689  txq->vsi_id = ad->vsi_id;
690  txq->queue_id = i;
691  if (i < vec_len (ad->txqs))
692  {
693  avf_txq_t *q = vec_elt_at_index (ad->txqs, i);
694  txq->ring_len = q->size;
695  txq->dma_ring_addr = avf_dma_addr (vm, ad, (void *) q->descs);
696  }
697  avf_log_debug (ad, "config_vsi_queues_tx[%u]: ring_len %u "
698  "dma_ring_addr 0x%llx", i, txq->ring_len,
699  txq->dma_ring_addr);
700  }
701 
702  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_CONFIG_VSI_QUEUES, msg, msg_len,
703  0, 0);
704 }
705 
706 clib_error_t *
708 {
709  int msg_len = sizeof (virtchnl_irq_map_info_t) +
710  (ad->n_rx_irqs) * sizeof (virtchnl_vector_map_t);
711  u8 msg[msg_len];
713 
714  clib_memset (msg, 0, msg_len);
715  imi = (virtchnl_irq_map_info_t *) msg;
716  imi->num_vectors = ad->n_rx_irqs;
717 
718  for (int i = 0; i < ad->n_rx_irqs; i++)
719  {
720  imi->vecmap[i].vector_id = i + 1;
721  imi->vecmap[i].vsi_id = ad->vsi_id;
722  if (ad->n_rx_irqs == ad->n_rx_queues)
723  imi->vecmap[i].rxq_map = 1 << i;
724  else
725  imi->vecmap[i].rxq_map = pow2_mask (ad->n_rx_queues);;
726 
727  avf_log_debug (ad, "config_irq_map[%u/%u]: vsi_id %u vector_id %u "
728  "rxq_map %u", i, ad->n_rx_irqs - 1, ad->vsi_id,
729  imi->vecmap[i].vector_id, imi->vecmap[i].rxq_map);
730  }
731 
732 
733  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_CONFIG_IRQ_MAP, msg, msg_len, 0,
734  0);
735 }
736 
737 clib_error_t *
739 {
740  int msg_len =
741  sizeof (virtchnl_ether_addr_list_t) +
742  count * sizeof (virtchnl_ether_addr_t);
743  u8 msg[msg_len];
745  int i;
746 
747  clib_memset (msg, 0, msg_len);
748  al = (virtchnl_ether_addr_list_t *) msg;
749  al->vsi_id = ad->vsi_id;
750  al->num_elements = count;
751 
752  avf_log_debug (ad, "add_eth_addr: vsi_id %u num_elements %u",
753  ad->vsi_id, al->num_elements);
754 
755  for (i = 0; i < count; i++)
756  {
757  clib_memcpy_fast (&al->list[i].addr, macs + i * 6, 6);
758  avf_log_debug (ad, "add_eth_addr[%u]: %U", i,
760  }
761  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_ADD_ETH_ADDR, msg, msg_len, 0,
762  0);
763 }
764 
765 clib_error_t *
767 {
768  virtchnl_queue_select_t qs = { 0 };
769  int i = 0;
770  qs.vsi_id = ad->vsi_id;
771  qs.rx_queues = rx;
772  qs.tx_queues = tx;
773 
774  avf_log_debug (ad, "enable_queues: vsi_id %u rx_queues %u tx_queues %u",
775  ad->vsi_id, qs.rx_queues, qs.tx_queues);
776 
777  while (rx)
778  {
779  if (rx & (1 << i))
780  {
781  avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
782  avf_reg_write (ad, AVF_QRX_TAIL (i), rxq->n_enqueued);
783  rx &= ~(1 << i);
784  }
785  i++;
786  }
787  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_ENABLE_QUEUES, &qs,
788  sizeof (virtchnl_queue_select_t), 0, 0);
789 }
790 
791 clib_error_t *
794 {
795  virtchnl_queue_select_t qs = { 0 };
796  qs.vsi_id = ad->vsi_id;
797 
798  avf_log_debug (ad, "get_stats: vsi_id %u", ad->vsi_id);
799 
800  return avf_send_to_pf (vm, ad, VIRTCHNL_OP_GET_STATS,
801  &qs, sizeof (virtchnl_queue_select_t),
802  es, sizeof (virtchnl_eth_stats_t));
803 }
804 
805 clib_error_t *
807 {
808  avf_aq_desc_t d = { 0 };
809  clib_error_t *error;
810  u32 rstat;
811  f64 t0, t = 0, suspend_time = AVF_RESET_SUSPEND_TIME;
812 
813  avf_log_debug (ad, "reset");
814 
815  d.opcode = 0x801;
816  d.v_opcode = VIRTCHNL_OP_RESET_VF;
817  if ((error = avf_aq_desc_enq (vm, ad, &d, 0, 0)))
818  return error;
819 
820  t0 = vlib_time_now (vm);
821 retry:
822  vlib_process_suspend (vm, suspend_time);
823 
824  rstat = avf_get_u32 (ad->bar0, AVFGEN_RSTAT);
825 
826  if (rstat == 2 || rstat == 3)
827  {
828  avf_log_debug (ad, "reset completed in %.3fs", t);
829  return 0;
830  }
831 
832  t = vlib_time_now (vm) - t0;
833  if (t > AVF_RESET_MAX_WAIT_TIME)
834  {
835  avf_log_err (ad, "reset failed (timeout %.3fs)", t);
836  return clib_error_return (0, "reset failed (timeout)");
837  }
838 
839  suspend_time *= 2;
840  goto retry;
841 }
842 
843 clib_error_t *
845 {
846  virtchnl_vf_res_request_t res_req = { 0 };
847  clib_error_t *error;
848  u32 rstat;
849  f64 t0, t, suspend_time = AVF_RESET_SUSPEND_TIME;
850 
851  res_req.num_queue_pairs = num_queue_pairs;
852 
853  avf_log_debug (ad, "request_queues: num_queue_pairs %u", num_queue_pairs);
854 
855  error = avf_send_to_pf (vm, ad, VIRTCHNL_OP_REQUEST_QUEUES, &res_req,
856  sizeof (virtchnl_vf_res_request_t), &res_req,
857  sizeof (virtchnl_vf_res_request_t));
858 
859  /*
860  * if PF responds, the request failed
861  * else PF initializes restart and avf_send_to_pf returns an error
862  */
863  if (!error)
864  {
865  return clib_error_return (0, "requested more than %u queue pairs",
866  res_req.num_queue_pairs);
867  }
868 
869  t0 = vlib_time_now (vm);
870 retry:
871  vlib_process_suspend (vm, suspend_time);
872  t = vlib_time_now (vm) - t0;
873 
874  rstat = avf_get_u32 (ad->bar0, AVFGEN_RSTAT);
875 
876  if ((rstat == VIRTCHNL_VFR_COMPLETED) || (rstat == VIRTCHNL_VFR_VFACTIVE))
877  goto done;
878 
879  if (t > AVF_RESET_MAX_WAIT_TIME)
880  {
881  avf_log_err (ad, "request queues failed (timeout %.3f seconds)", t);
882  return clib_error_return (0, "request queues failed (timeout)");
883  }
884 
885  suspend_time *= 2;
886  goto retry;
887 
888 done:
889  return NULL;
890 }
891 
892 clib_error_t *
894  avf_create_if_args_t * args)
895 {
896  virtchnl_version_info_t ver = { 0 };
897  virtchnl_vf_resource_t res = { 0 };
898  clib_error_t *error;
900  int i, wb_on_itr;
901 
902  avf_adminq_init (vm, ad);
903 
904  if ((error = avf_request_queues (vm, ad, clib_max (tm->n_vlib_mains,
905  args->rxq_num))))
906  {
907  /* we failed to get more queues, but still we want to proceed */
908  clib_error_free (error);
909 
910  if ((error = avf_device_reset (vm, ad)))
911  return error;
912  }
913 
914  avf_adminq_init (vm, ad);
915 
916  /*
917  * OP_VERSION
918  */
919  if ((error = avf_op_version (vm, ad, &ver)))
920  return error;
921 
922  if (ver.major != VIRTCHNL_VERSION_MAJOR ||
924  return clib_error_return (0, "incompatible protocol version "
925  "(remote %d.%d)", ver.major, ver.minor);
926 
927  /*
928  * OP_GET_VF_RESOURCES
929  */
930  if ((error = avf_op_get_vf_resources (vm, ad, &res)))
931  return error;
932 
933  if (res.num_vsis != 1 || res.vsi_res[0].vsi_type != VIRTCHNL_VSI_SRIOV)
934  return clib_error_return (0, "unexpected GET_VF_RESOURCE reply received");
935 
936  ad->vsi_id = res.vsi_res[0].vsi_id;
939  ad->max_vectors = res.max_vectors;
940  ad->max_mtu = res.max_mtu;
941  ad->rss_key_size = res.rss_key_size;
942  ad->rss_lut_size = res.rss_lut_size;
943  wb_on_itr = (ad->feature_bitmap & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) != 0;
944 
946 
947  /*
948  * Disable VLAN stripping
949  */
950  if ((error = avf_op_disable_vlan_stripping (vm, ad)))
951  return error;
952 
953  /*
954  * Init Queues
955  */
956  if (args->rxq_num == 0)
957  {
958  args->rxq_num = 1;
959  }
960  else if (args->rxq_num > ad->num_queue_pairs)
961  {
962  args->rxq_num = ad->num_queue_pairs;
963  avf_log_warn (ad, "Requested more rx queues than queue pairs available."
964  "Using %u rx queues.", args->rxq_num);
965  }
966 
967  for (i = 0; i < args->rxq_num; i++)
968  if ((error = avf_rxq_init (vm, ad, i, args->rxq_size)))
969  return error;
970 
971  for (i = 0; i < tm->n_vlib_mains; i++)
972  if ((error = avf_txq_init (vm, ad, i, args->txq_size)))
973  return error;
974 
975  if (ad->max_vectors > ad->n_rx_queues)
976  {
977  ad->flags |= AVF_DEVICE_F_RX_INT;
978  ad->n_rx_irqs = args->rxq_num;
979  }
980  else
981  ad->n_rx_irqs = 1;
982 
983 
984  if ((ad->feature_bitmap & VIRTCHNL_VF_OFFLOAD_RSS_PF) &&
985  (error = avf_op_config_rss_lut (vm, ad)))
986  return error;
987 
988  if ((ad->feature_bitmap & VIRTCHNL_VF_OFFLOAD_RSS_PF) &&
989  (error = avf_op_config_rss_key (vm, ad)))
990  return error;
991 
992  if ((error = avf_op_config_vsi_queues (vm, ad)))
993  return error;
994 
995  if ((error = avf_op_config_irq_map (vm, ad)))
996  return error;
997 
999 
1000  for (i = 0; i < ad->n_rx_irqs; i++)
1001  avf_irq_n_set_state (ad, i, wb_on_itr ? AVF_IRQ_STATE_WB_ON_ITR :
1003 
1004  if ((error = avf_op_add_eth_addr (vm, ad, 1, ad->hwaddr)))
1005  return error;
1006 
1007  if ((error = avf_op_enable_queues (vm, ad, pow2_mask (ad->n_rx_queues),
1008  pow2_mask (ad->n_tx_queues))))
1009  return error;
1010 
1011  ad->flags |= AVF_DEVICE_F_INITIALIZED;
1012  return error;
1013 }
1014 
1015 void
1017 {
1018  avf_main_t *am = &avf_main;
1019  vnet_main_t *vnm = vnet_get_main ();
1021  u32 r;
1022 
1023  if (ad->flags & AVF_DEVICE_F_ERROR)
1024  return;
1025 
1026  if ((ad->flags & AVF_DEVICE_F_INITIALIZED) == 0)
1027  return;
1028 
1029  ASSERT (ad->error == 0);
1030 
1031  /* do not process device in reset state */
1032  r = avf_get_u32 (ad->bar0, AVFGEN_RSTAT);
1033  if (r != VIRTCHNL_VFR_VFACTIVE)
1034  return;
1035 
1036  r = avf_get_u32 (ad->bar0, AVF_ARQLEN);
1037  if ((r & 0xf0000000) != (1ULL << 31))
1038  {
1039  ad->error = clib_error_return (0, "arq not enabled, arqlen = 0x%x", r);
1040  avf_log_err (ad, "error: %U", format_clib_error, ad->error);
1041  goto error;
1042  }
1043 
1044  r = avf_get_u32 (ad->bar0, AVF_ATQLEN);
1045  if ((r & 0xf0000000) != (1ULL << 31))
1046  {
1047  ad->error = clib_error_return (0, "atq not enabled, atqlen = 0x%x", r);
1048  avf_log_err (ad, "error: %U", format_clib_error, ad->error);
1049  goto error;
1050  }
1051 
1052  if (is_irq == 0)
1053  avf_op_get_stats (vm, ad, &ad->eth_stats);
1054 
1055  /* *INDENT-OFF* */
1056  vec_foreach (e, ad->events)
1057  {
1058  avf_log_debug (ad, "event: %s (%u) sev %d",
1060  if (e->event == VIRTCHNL_EVENT_LINK_CHANGE)
1061  {
1062  int link_up;
1063  virtchnl_link_speed_t speed = e->event_data.link_event.link_speed;
1064  u32 flags = 0;
1065  u32 mbps = 0;
1066 
1067  if (ad->feature_bitmap & VIRTCHNL_VF_CAP_ADV_LINK_SPEED)
1068  link_up = e->event_data.link_event_adv.link_status;
1069  else
1070  link_up = e->event_data.link_event.link_status;
1071 
1072  if (ad->feature_bitmap & VIRTCHNL_VF_CAP_ADV_LINK_SPEED)
1073  mbps = e->event_data.link_event_adv.link_speed;
1074  if (speed == VIRTCHNL_LINK_SPEED_40GB)
1075  mbps = 40000;
1076  else if (speed == VIRTCHNL_LINK_SPEED_25GB)
1077  mbps = 25000;
1078  else if (speed == VIRTCHNL_LINK_SPEED_10GB)
1079  mbps = 10000;
1080  else if (speed == VIRTCHNL_LINK_SPEED_5GB)
1081  mbps = 5000;
1082  else if (speed == VIRTCHNL_LINK_SPEED_2_5GB)
1083  mbps = 2500;
1084  else if (speed == VIRTCHNL_LINK_SPEED_1GB)
1085  mbps = 1000;
1086  else if (speed == VIRTCHNL_LINK_SPEED_100MB)
1087  mbps = 100;
1088 
1089  avf_log_debug (ad, "event_link_change: status %d speed %u mbps",
1090  link_up, mbps);
1091 
1092  if (link_up && (ad->flags & AVF_DEVICE_F_LINK_UP) == 0)
1093  {
1094  ad->flags |= AVF_DEVICE_F_LINK_UP;
1097  vnet_hw_interface_set_flags (vnm, ad->hw_if_index, flags);
1099  mbps * 1000);
1100  ad->link_speed = mbps;
1101  }
1102  else if (!link_up && (ad->flags & AVF_DEVICE_F_LINK_UP) != 0)
1103  {
1104  ad->flags &= ~AVF_DEVICE_F_LINK_UP;
1105  ad->link_speed = 0;
1106  }
1107 
1108  if (ad->flags & AVF_DEVICE_F_ELOG)
1109  {
1110  ELOG_TYPE_DECLARE (el) =
1111  {
1112  .format = "avf[%d] link change: link_status %d "
1113  "link_speed %d mbps",
1114  .format_args = "i4i1i4",
1115  };
1116  struct
1117  {
1118  u32 dev_instance;
1119  u8 link_status;
1120  u32 link_speed;
1121  } *ed;
1122  ed = ELOG_DATA (&vm->elog_main, el);
1123  ed->dev_instance = ad->dev_instance;
1124  ed->link_status = link_up;
1125  ed->link_speed = mbps;
1126  }
1127  }
1128  else
1129  {
1130  if (ad->flags & AVF_DEVICE_F_ELOG)
1131  {
1132  ELOG_TYPE_DECLARE (el) =
1133  {
1134  .format = "avf[%d] unknown event: event %d severity %d",
1135  .format_args = "i4i4i1i1",
1136  };
1137  struct
1138  {
1139  u32 dev_instance;
1140  u32 event;
1141  u32 severity;
1142  } *ed;
1143  ed = ELOG_DATA (&vm->elog_main, el);
1144  ed->dev_instance = ad->dev_instance;
1145  ed->event = e->event;
1146  ed->severity = e->severity;
1147  }
1148  }
1149  }
1150  /* *INDENT-ON* */
1151  vec_reset_length (ad->events);
1152 
1153  return;
1154 
1155 error:
1156  ad->flags |= AVF_DEVICE_F_ERROR;
1157  ASSERT (ad->error != 0);
1158  vlib_log_err (am->log_class, "%U", format_clib_error, ad->error);
1159 }
1160 
1161 static u32
1163 {
1166  u8 promisc_enabled;
1167 
1168  switch (flags)
1169  {
1171  ad->flags &= ~AVF_DEVICE_F_PROMISC;
1172  break;
1174  ad->flags |= AVF_DEVICE_F_PROMISC;
1175  break;
1176  default:
1177  return ~0;
1178  }
1179 
1180  promisc_enabled = ((ad->flags & AVF_DEVICE_F_PROMISC) != 0);
1181 
1183  promisc_enabled ?
1186  hw->dev_instance);
1187  return 0;
1188 }
1189 
1190 static uword
1192 {
1193  avf_main_t *am = &avf_main;
1194  uword *event_data = 0, event_type;
1195  int enabled = 0, irq;
1196  f64 last_run_duration = 0;
1197  f64 last_periodic_time = 0;
1198  avf_device_t **dev_pointers = 0;
1199  u32 i;
1200 
1201  while (1)
1202  {
1203  if (enabled)
1204  vlib_process_wait_for_event_or_clock (vm, 5.0 - last_run_duration);
1205  else
1207 
1208  event_type = vlib_process_get_events (vm, &event_data);
1209  irq = 0;
1210 
1211  switch (event_type)
1212  {
1213  case ~0:
1214  last_periodic_time = vlib_time_now (vm);
1215  break;
1217  enabled = 1;
1218  break;
1220  for (int i = 0; i < vec_len (event_data); i++)
1221  {
1222  avf_device_t *ad = avf_get_device (event_data[i]);
1223  avf_delete_if (vm, ad, /* with_barrier */ 1);
1224  }
1225  if (pool_elts (am->devices) < 1)
1226  enabled = 0;
1227  break;
1229  irq = 1;
1230  break;
1233  for (int i = 0; i < vec_len (event_data); i++)
1234  {
1235  avf_device_t *ad = avf_get_device (event_data[i]);
1236  clib_error_t *err;
1237  int is_enable = 0;
1238 
1239  if (event_type == AVF_PROCESS_EVENT_SET_PROMISC_ENABLE)
1240  is_enable = 1;
1241 
1242  if ((err = avf_config_promisc_mode (vm, ad, is_enable)))
1243  {
1244  avf_log_err (ad, "error: %U", format_clib_error, err);
1245  clib_error_free (err);
1246  }
1247  }
1248  break;
1249 
1250  default:
1251  ASSERT (0);
1252  }
1253 
1254  vec_reset_length (event_data);
1255 
1256  if (enabled == 0)
1257  continue;
1258 
1259  /* create local list of device pointers as device pool may grow
1260  * during suspend */
1261  vec_reset_length (dev_pointers);
1262  /* *INDENT-OFF* */
1263  pool_foreach_index (i, am->devices,
1264  {
1265  vec_add1 (dev_pointers, avf_get_device (i));
1266  });
1267 
1268  vec_foreach_index (i, dev_pointers)
1269  {
1270  avf_process_one_device (vm, dev_pointers[i], irq);
1271  };
1272  /* *INDENT-ON* */
1273  last_run_duration = vlib_time_now (vm) - last_periodic_time;
1274  }
1275  return 0;
1276 }
1277 
1278 /* *INDENT-OFF* */
1280  .function = avf_process,
1281  .type = VLIB_NODE_TYPE_PROCESS,
1282  .name = "avf-process",
1283 };
1284 /* *INDENT-ON* */
1285 
1286 static void
1288 {
1289  uword pd = vlib_pci_get_private_data (vm, h);
1290  avf_device_t *ad = avf_get_device (pd);
1291  u32 icr0;
1292 
1293  icr0 = avf_reg_read (ad, AVFINT_ICR0);
1294 
1295  if (ad->flags & AVF_DEVICE_F_ELOG)
1296  {
1297  /* *INDENT-OFF* */
1298  ELOG_TYPE_DECLARE (el) =
1299  {
1300  .format = "avf[%d] irq 0: icr0 0x%x",
1301  .format_args = "i4i4",
1302  };
1303  /* *INDENT-ON* */
1304  struct
1305  {
1306  u32 dev_instance;
1307  u32 icr0;
1308  } *ed;
1309 
1310  ed = ELOG_DATA (&vm->elog_main, el);
1311  ed->dev_instance = ad->dev_instance;
1312  ed->icr0 = icr0;
1313  }
1314 
1316 
1317  /* bit 30 - Send/Receive Admin queue interrupt indication */
1318  if (icr0 & (1 << 30))
1321 }
1322 
1323 static void
1325 {
1326  vnet_main_t *vnm = vnet_get_main ();
1327  uword pd = vlib_pci_get_private_data (vm, h);
1328  avf_device_t *ad = avf_get_device (pd);
1329 
1330  if (ad->flags & AVF_DEVICE_F_ELOG)
1331  {
1332  /* *INDENT-OFF* */
1333  ELOG_TYPE_DECLARE (el) =
1334  {
1335  .format = "avf[%d] irq %d: received",
1336  .format_args = "i4i2",
1337  };
1338  /* *INDENT-ON* */
1339  struct
1340  {
1341  u32 dev_instance;
1342  u16 line;
1343  } *ed;
1344 
1345  ed = ELOG_DATA (&vm->elog_main, el);
1346  ed->dev_instance = ad->dev_instance;
1347  ed->line = line;
1348  }
1349 
1350  line--;
1351 
1352  if (ad->flags & AVF_DEVICE_F_RX_INT && ad->rxqs[line].int_mode)
1355 }
1356 
1357 void
1358 avf_delete_if (vlib_main_t * vm, avf_device_t * ad, int with_barrier)
1359 {
1360  vnet_main_t *vnm = vnet_get_main ();
1361  avf_main_t *am = &avf_main;
1362  int i;
1363 
1364  ad->flags &= ~AVF_DEVICE_F_ADMIN_UP;
1365 
1366  if (ad->hw_if_index)
1367  {
1368  if (with_barrier)
1373  if (with_barrier)
1375  }
1376 
1378 
1379  vlib_physmem_free (vm, ad->atq);
1380  vlib_physmem_free (vm, ad->arq);
1381  vlib_physmem_free (vm, ad->atq_bufs);
1382  vlib_physmem_free (vm, ad->arq_bufs);
1383 
1384  /* *INDENT-OFF* */
1385  vec_foreach_index (i, ad->rxqs)
1386  {
1387  avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, i);
1388  vlib_physmem_free (vm, (void *) rxq->descs);
1389  if (rxq->n_enqueued)
1390  vlib_buffer_free_from_ring (vm, rxq->bufs, rxq->next, rxq->size,
1391  rxq->n_enqueued);
1392  vec_free (rxq->bufs);
1393  }
1394  /* *INDENT-ON* */
1395  vec_free (ad->rxqs);
1396 
1397  /* *INDENT-OFF* */
1398  vec_foreach_index (i, ad->txqs)
1399  {
1400  avf_txq_t *txq = vec_elt_at_index (ad->txqs, i);
1401  vlib_physmem_free (vm, (void *) txq->descs);
1402  if (txq->n_enqueued)
1403  {
1404  u16 first = (txq->next - txq->n_enqueued) & (txq->size -1);
1405  vlib_buffer_free_from_ring (vm, txq->bufs, first, txq->size,
1406  txq->n_enqueued);
1407  }
1408  vec_free (txq->bufs);
1409  clib_ring_free (txq->rs_slots);
1410  }
1411  /* *INDENT-ON* */
1412  vec_free (ad->txqs);
1413  vec_free (ad->name);
1414 
1415  clib_error_free (ad->error);
1416  clib_memset (ad, 0, sizeof (*ad));
1417  pool_put_index (am->devices, ad->dev_instance);
1418  clib_mem_free (ad);
1419 }
1420 
1421 void
1423 {
1424  vnet_main_t *vnm = vnet_get_main ();
1425  avf_main_t *am = &avf_main;
1426  avf_device_t *ad, **adp;
1428  clib_error_t *error = 0;
1429  int i;
1430 
1431  /* check input args */
1432  args->rxq_size = (args->rxq_size == 0) ? AVF_RXQ_SZ : args->rxq_size;
1433  args->txq_size = (args->txq_size == 0) ? AVF_TXQ_SZ : args->txq_size;
1434 
1435  if ((args->rxq_size & (args->rxq_size - 1))
1436  || (args->txq_size & (args->txq_size - 1)))
1437  {
1438  args->rv = VNET_API_ERROR_INVALID_VALUE;
1439  args->error =
1440  clib_error_return (error, "queue size must be a power of two");
1441  return;
1442  }
1443 
1444  pool_get (am->devices, adp);
1445  adp[0] = ad = clib_mem_alloc_aligned (sizeof (avf_device_t),
1447  clib_memset (ad, 0, sizeof (avf_device_t));
1448  ad->dev_instance = adp - am->devices;
1449  ad->per_interface_next_index = ~0;
1450  ad->name = vec_dup (args->name);
1451 
1452  if (args->enable_elog)
1453  ad->flags |= AVF_DEVICE_F_ELOG;
1454 
1455  if ((error = vlib_pci_device_open (vm, &args->addr, avf_pci_device_ids,
1456  &h)))
1457  {
1458  pool_put (am->devices, adp);
1459  clib_mem_free (ad);
1460  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1461  args->error =
1462  clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
1463  &args->addr);
1464  return;
1465  }
1466  ad->pci_dev_handle = h;
1467  ad->pci_addr = args->addr;
1468  ad->numa_node = vlib_pci_get_numa_node (vm, h);
1469 
1471 
1472  if ((error = vlib_pci_bus_master_enable (vm, h)))
1473  goto error;
1474 
1475  if ((error = vlib_pci_map_region (vm, h, 0, &ad->bar0)))
1476  goto error;
1477 
1479  AVF_MBOX_LEN,
1481  ad->numa_node);
1482  if (ad->atq == 0)
1483  {
1484  error = vlib_physmem_last_error (vm);
1485  goto error;
1486  }
1487 
1488  if ((error = vlib_pci_map_dma (vm, h, ad->atq)))
1489  goto error;
1490 
1492  AVF_MBOX_LEN,
1494  ad->numa_node);
1495  if (ad->arq == 0)
1496  {
1497  error = vlib_physmem_last_error (vm);
1498  goto error;
1499  }
1500 
1501  if ((error = vlib_pci_map_dma (vm, h, ad->arq)))
1502  goto error;
1503 
1505  AVF_MBOX_LEN,
1507  ad->numa_node);
1508  if (ad->atq_bufs == 0)
1509  {
1510  error = vlib_physmem_last_error (vm);
1511  goto error;
1512  }
1513 
1514  if ((error = vlib_pci_map_dma (vm, h, ad->atq_bufs)))
1515  goto error;
1516 
1518  AVF_MBOX_LEN,
1520  ad->numa_node);
1521  if (ad->arq_bufs == 0)
1522  {
1523  error = vlib_physmem_last_error (vm);
1524  goto error;
1525  }
1526 
1527  if ((error = vlib_pci_map_dma (vm, h, ad->arq_bufs)))
1528  goto error;
1529 
1531  ad->flags |= AVF_DEVICE_F_VA_DMA;
1532 
1533  if ((error = avf_device_init (vm, am, ad, args)))
1534  goto error;
1535 
1536  if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
1537  &avf_irq_0_handler)))
1538  goto error;
1539 
1540  if ((error = vlib_pci_register_msix_handler (vm, h, 1, ad->n_rx_irqs,
1541  &avf_irq_n_handler)))
1542  goto error;
1543 
1544  if ((error = vlib_pci_enable_msix_irq (vm, h, 0, ad->n_rx_irqs + 1)))
1545  goto error;
1546 
1547  if ((error = vlib_pci_intr_enable (vm, h)))
1548  goto error;
1549 
1550  /* create interface */
1551  error = ethernet_register_interface (vnm, avf_device_class.index,
1552  ad->dev_instance, ad->hwaddr,
1554 
1555  if (error)
1556  goto error;
1557 
1558  /* Indicate ability to support L3 DMAC filtering and
1559  * initialize interface to L3 non-promisc mode */
1562  ethernet_set_flags (vnm, ad->hw_if_index,
1564 
1566  args->sw_if_index = ad->sw_if_index = sw->sw_if_index;
1567 
1571  avf_input_node.index);
1572 
1573  for (i = 0; i < ad->n_rx_queues; i++)
1575 
1576  if (pool_elts (am->devices) == 1)
1579 
1580  return;
1581 
1582 error:
1583  avf_delete_if (vm, ad, /* with_barrier */ 0);
1584  args->rv = VNET_API_ERROR_INVALID_INTERFACE;
1585  args->error = clib_error_return (error, "pci-addr %U",
1586  format_vlib_pci_addr, &args->addr);
1587  avf_log_err (ad, "error: %U", format_clib_error, args->error);
1588 }
1589 
1590 static clib_error_t *
1592 {
1593  vnet_hw_interface_t *hi = vnet_get_hw_interface (vnm, hw_if_index);
1595  uword is_up = (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) != 0;
1596 
1597  if (ad->flags & AVF_DEVICE_F_ERROR)
1598  return clib_error_return (0, "device is in error state");
1599 
1600  if (is_up)
1601  {
1604  ad->flags |= AVF_DEVICE_F_ADMIN_UP;
1605  }
1606  else
1607  {
1609  ad->flags &= ~AVF_DEVICE_F_ADMIN_UP;
1610  }
1611  return 0;
1612 }
1613 
1614 static clib_error_t *
1617 {
1618  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
1620  avf_rxq_t *rxq = vec_elt_at_index (ad->rxqs, qid);
1621 
1623  {
1624  if (rxq->int_mode == 0)
1625  return 0;
1626  if (ad->feature_bitmap & VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
1628  else
1630  rxq->int_mode = 0;
1631  }
1632  else
1633  {
1634  if (rxq->int_mode == 1)
1635  return 0;
1636  if (ad->n_rx_irqs != ad->n_rx_queues)
1637  return clib_error_return (0, "not enough interrupt lines");
1638  rxq->int_mode = 1;
1640  }
1641 
1642  return 0;
1643 }
1644 
1645 static void
1647  u32 node_index)
1648 {
1649  vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
1651 
1652  /* Shut off redirection */
1653  if (node_index == ~0)
1654  {
1655  ad->per_interface_next_index = node_index;
1656  return;
1657  }
1658 
1660  vlib_node_add_next (vlib_get_main (), avf_input_node.index, node_index);
1661 }
1662 
1663 static char *avf_tx_func_error_strings[] = {
1664 #define _(n,s) s,
1666 #undef _
1667 };
1668 
1669 static void
1671 {
1672  avf_device_t *ad = avf_get_device (instance);
1674  &ad->eth_stats, sizeof (ad->eth_stats));
1675 }
1676 
1677 /* *INDENT-OFF* */
1679 {
1680  .name = "Adaptive Virtual Function (AVF) interface",
1681  .clear_counters = avf_clear_hw_interface_counters,
1682  .format_device = format_avf_device,
1683  .format_device_name = format_avf_device_name,
1684  .admin_up_down_function = avf_interface_admin_up_down,
1685  .rx_mode_change_function = avf_interface_rx_mode_change,
1686  .rx_redirect_to_node = avf_set_interface_next_node,
1687  .tx_function_n_errors = AVF_TX_N_ERROR,
1688  .tx_function_error_strings = avf_tx_func_error_strings,
1689 };
1690 /* *INDENT-ON* */
1691 
1692 clib_error_t *
1694 {
1695  avf_main_t *am = &avf_main;
1697 
1700 
1701  am->log_class = vlib_log_register_class ("avf", 0);
1702  vlib_log_debug (am->log_class, "initialized");
1703 
1704  return 0;
1705 }
1706 
1707 /* *INDENT-OFF* */
1709 {
1710  .runs_after = VLIB_INITS ("pci_bus_init"),
1711 };
1712 /* *INDENT-OFF* */
1713 
1714 /*
1715  * fd.io coding-style-patch-verification: ON
1716  *
1717  * Local Variables:
1718  * eval: (c-set-style "gnu")
1719  * End:
1720  */
vlib_log_class_t vlib_log_register_class(char *class, char *subclass)
Definition: log.c:209
#define AVF_TXQ_SZ
Definition: device.c:29
clib_error_t * avf_op_get_vf_resources(vlib_main_t *vm, avf_device_t *ad, virtchnl_vf_resource_t *res)
Definition: device.c:538
u8 count
Definition: dhcp.api:208
clib_error_t * avf_op_add_eth_addr(vlib_main_t *vm, avf_device_t *ad, u8 count, u8 *macs)
Definition: device.c:738
vlib_node_registration_t avf_process_node
(constructor) VLIB_REGISTER_NODE (avf_process_node)
Definition: device.c:1279
u8 int_mode
Definition: avf.h:130
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define AVF_ARQLEN
Definition: virtchnl.h:47
virtchnl_queue_pair_info_t qpair[1]
Definition: virtchnl.h:311
u32 hw_if_index
Definition: avf.h:155
u8 * format_clib_error(u8 *s, va_list *va)
Definition: error.c:191
static clib_error_t * vlib_pci_intr_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:239
#define PCI_DEVICE_ID_INTEL_AVF
Definition: device.c:33
#define AVF_ATQH
Definition: virtchnl.h:40
#define clib_min(x, y)
Definition: clib.h:327
static void avf_irq_0_set_state(avf_device_t *ad, avf_irq_state_t state)
Definition: device.c:61
static uword random_default_seed(void)
Default random seed (unix/linux user-mode)
Definition: random.h:91
static clib_error_t * vlib_pci_bus_master_enable(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.h:271
static void * vlib_physmem_alloc_aligned_on_numa(vlib_main_t *vm, uword n_bytes, uword alignment, u32 numa_node)
Definition: physmem_funcs.h:63
static f64 vlib_process_wait_for_event_or_clock(vlib_main_t *vm, f64 dt)
Suspend a cooperative multi-tasking thread Waits for an event, or for the indicated number of seconds...
Definition: node_funcs.h:751
void avf_arq_slot_init(avf_device_t *ad, u16 slot)
Definition: device.c:337
vl_api_wireguard_peer_flags_t flags
Definition: wireguard.api:103
clib_error_t * avf_send_to_pf(vlib_main_t *vm, avf_device_t *ad, virtchnl_ops_t op, void *in, int in_len, void *out, int out_len)
Definition: device.c:394
void avf_create_if(vlib_main_t *vm, avf_create_if_args_t *args)
Definition: device.c:1422
static uword * vlib_process_wait_for_event(vlib_main_t *vm)
Definition: node_funcs.h:656
void ethernet_delete_interface(vnet_main_t *vnm, u32 hw_if_index)
Definition: interface.c:378
#define AVF_ATQBAH
Definition: virtchnl.h:45
clib_error_t * avf_device_init(vlib_main_t *vm, avf_main_t *am, avf_device_t *ad, avf_create_if_args_t *args)
Definition: device.c:893
vnet_main_t * vnet_get_main(void)
Definition: misc.c:46
clib_error_t * error
Definition: avf.h:195
#define avf_log_warn(dev, f,...)
Definition: avf.h:55
#define AVF_AQ_ENQ_SUSPEND_TIME
Definition: avf.h:25
clib_error_t * avf_op_config_vsi_queues(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:651
clib_error_t * avf_op_version(vlib_main_t *vm, avf_device_t *ad, virtchnl_version_info_t *ver)
Definition: device.c:516
u64 atq_bufs_pa
Definition: avf.h:172
static uword vlib_buffer_get_pa(vlib_main_t *vm, vlib_buffer_t *b)
Definition: buffer_funcs.h:457
virtchnl_vsi_type_t vsi_type
Definition: virtchnl.h:165
#define AVF_SEND_TO_PF_SUSPEND_TIME
Definition: avf.h:31
unsigned long u64
Definition: types.h:89
void vlib_pci_device_close(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1297
virtchnl_vector_map_t vecmap[1]
Definition: virtchnl.h:339
#define clib_memcpy_fast(a, b, c)
Definition: string.h:81
u16 n_rx_irqs
Definition: avf.h:183
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
static f64 vlib_time_now(vlib_main_t *vm)
Definition: main.h:333
#define clib_ring_new_aligned(ring, size, align)
Definition: ring.h:53
virtchnl_link_speed_t link_speed
Definition: avf.h:187
#define AVF_ARQBAH
Definition: virtchnl.h:39
static uword avf_dma_addr(vlib_main_t *vm, avf_device_t *ad, void *p)
Definition: device.c:350
static clib_error_t * vlib_physmem_last_error(struct vlib_main_t *vm)
clib_error_t * avf_aq_desc_enq(vlib_main_t *vm, avf_device_t *ad, avf_aq_desc_t *dt, void *data, int len)
Definition: device.c:122
static vnet_hw_interface_t * vnet_get_hw_interface(vnet_main_t *vnm, u32 hw_if_index)
#define AVF_QRX_TAIL(q)
Definition: virtchnl.h:51
#define AVF_ARQT
Definition: virtchnl.h:43
format_function_t format_avf_device
Definition: avf.h:257
#define vec_add2(V, P, N)
Add N elements to end of vector V, return pointer to new elements in P.
Definition: vec.h:630
static void avf_irq_n_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: device.c:1324
vlib_pci_addr_t addr
Definition: avf.h:238
struct virtchnl_pf_event_t::@33::@35 link_event_adv
clib_error_t * avf_init(vlib_main_t *vm)
Definition: device.c:1693
#define AVF_AQ_F_SI
Definition: virtchnl.h:61
u32 dev_instance
Definition: avf.h:153
clib_error_t * vlib_pci_enable_msix_irq(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 start, u16 count)
Definition: pci.c:894
vlib_main_t * vm
Definition: in2out_ed.c:1582
#define AVF_QTX_TAIL(q)
Definition: virtchnl.h:50
clib_error_t * avf_rxq_init(vlib_main_t *vm, avf_device_t *ad, u16 qid, u16 rxq_size)
Definition: device.c:240
virtchnl_link_speed_t
Definition: virtchnl.h:205
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:520
#define ETHERNET_INTERFACE_FLAG_DEFAULT_L3
Definition: ethernet.h:149
#define pool_get(P, E)
Allocate an object E from a pool P (unspecified alignment).
Definition: pool.h:252
volatile u32 * qtx_tail
Definition: avf.h:137
#define AVF_ATQLEN
Definition: virtchnl.h:41
static uword vlib_node_add_next(vlib_main_t *vm, uword node, uword next_node)
Definition: node_funcs.h:1173
void avf_process_one_device(vlib_main_t *vm, avf_device_t *ad, int is_irq)
Definition: device.c:1016
unsigned char u8
Definition: types.h:56
vnet_device_class_t avf_device_class
#define AVF_ARQH
Definition: virtchnl.h:44
u8 data[128]
Definition: ipsec_types.api:89
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
double f64
Definition: types.h:142
#define vlib_worker_thread_barrier_sync(X)
Definition: threads.h:205
void avf_delete_if(vlib_main_t *vm, avf_device_t *ad, int with_barrier)
Definition: device.c:1358
virtchnl_ops_t
Definition: virtchnl.h:103
static vnet_sw_interface_t * vnet_get_hw_sw_interface(vnet_main_t *vnm, u32 hw_if_index)
u8 buffer_pool_index
Definition: avf.h:131
static uword vlib_process_suspend(vlib_main_t *vm, f64 dt)
Suspend a vlib cooperative multi-tasking thread for a period of time.
Definition: node_funcs.h:482
#define AVF_AQ_F_DD
Definition: virtchnl.h:53
vnet_hw_interface_rx_mode
Definition: interface.h:53
#define AVF_RESET_SUSPEND_TIME
Definition: avf.h:28
u16 * rs_slots
Definition: avf.h:144
#define AVFINT_ICR0_ENA1
Definition: virtchnl.h:37
VNET_DEVICE_CLASS(af_xdp_device_class)
u8 * format_ethernet_address(u8 *s, va_list *args)
Definition: format.c:44
avf_irq_state_t
Definition: device.c:53
clib_error_t * vlib_pci_map_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h, void *ptr)
Definition: pci.c:1218
#define VLIB_INIT_FUNCTION(x)
Definition: init.h:173
static uword vlib_process_get_events(vlib_main_t *vm, uword **data_vector)
Return the first event type which has occurred and a vector of per-event data of that type...
Definition: node_funcs.h:579
clib_spinlock_t lock
Definition: avf.h:140
#define AVF_SEND_TO_PF_MAX_WAIT_TIME
Definition: avf.h:32
static uword pow2_mask(uword x)
Definition: clib.h:237
static u32 avf_reg_read(avf_device_t *ad, u32 addr)
Definition: avf.h:314
static_always_inline void vnet_device_input_set_interrupt_pending(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.h:136
static void avf_irq_0_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u16 line)
Definition: device.c:1287
vnet_hw_interface_flags_t flags
Definition: interface.h:537
volatile u32 * qrx_tail
Definition: avf.h:124
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
Definition: error.h:99
static void avf_set_interface_next_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: device.c:1646
static const char * virtchnl_event_names[]
Definition: device.c:47
unsigned int u32
Definition: types.h:88
vlib_pci_dev_handle_t pci_dev_handle
Definition: avf.h:156
clib_error_t * avf_op_config_rss_lut(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:576
#define vlib_log_debug(...)
Definition: log.h:109
virtchnl_ether_addr_t list[1]
Definition: virtchnl.h:354
void * arq_bufs
Definition: avf.h:171
avf_aq_desc_t * arq
Definition: avf.h:169
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
static void vlib_buffer_free_from_ring(vlib_main_t *vm, u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Free buffers from ring.
Definition: buffer_funcs.h:984
static u32 avf_flag_change(vnet_main_t *vnm, vnet_hw_interface_t *hw, u32 flags)
Definition: device.c:1162
static heap_elt_t * first(heap_header_t *h)
Definition: heap.c:59
u8 * name
Definition: avf.h:159
virtchnl_eth_stats_t last_cleared_eth_stats
Definition: avf.h:192
static void vlib_process_signal_event(vlib_main_t *vm, uword node_index, uword type_opaque, uword data)
Definition: node_funcs.h:1015
void vlib_pci_set_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h, uword private_data)
Definition: pci.c:155
static u32 avf_get_u32(void *start, int offset)
Definition: avf.h:268
virtchnl_txq_info_t txq
Definition: virtchnl.h:302
#define AVF_ATQT
Definition: virtchnl.h:48
unsigned short u16
Definition: types.h:57
#define AVF_RESET_MAX_WAIT_TIME
Definition: avf.h:29
vec_header_t h
Definition: buffer.c:322
static void avf_clear_hw_interface_counters(u32 instance)
Definition: device.c:1670
#define pool_put(P, E)
Free an object E in pool P.
Definition: pool.h:302
#define vec_dup(V)
Return copy of vector (no header, no alignment)
Definition: vec.h:429
static void avf_adminq_init(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:357
clib_error_t * vlib_pci_register_msix_handler(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 start, u32 count, pci_msix_handler_function_t *msix_handler)
Definition: pci.c:837
u64 qword[4]
Definition: avf.h:99
#define ELOG_DATA(em, f)
Definition: elog.h:484
#define AVF_AQ_F_RD
Definition: virtchnl.h:58
#define VIRTCHNL_VERSION_MAJOR
Definition: virtchnl.h:21
uword vlib_pci_get_private_data(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:148
clib_error_t * avf_op_disable_vlan_stripping(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:623
static void avf_reg_flush(avf_device_t *ad)
Definition: avf.h:320
u32 vlib_pci_dev_handle_t
Definition: pci.h:97
#define AVFINT_ICR0
Definition: virtchnl.h:36
vl_api_tunnel_mode_t mode
Definition: gre.api:48
clib_error_t * avf_device_reset(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:806
#define PCI_DEVICE_ID_INTEL_X710_VF
Definition: device.c:34
clib_error_t * avf_op_config_irq_map(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:707
avf_device_t ** devices
Definition: avf.h:228
#define PCI_VENDOR_ID_INTEL
Definition: device.c:32
u8 len
Definition: ip_types.api:92
clib_error_t * avf_request_queues(vlib_main_t *vm, avf_device_t *ad, u16 num_queue_pairs)
Definition: device.c:844
u16 n_rx_queues
Definition: avf.h:165
static_always_inline u32 vlib_buffer_get_default_data_size(vlib_main_t *vm)
Definition: buffer_funcs.h:96
u8 slot
Definition: pci_types.api:22
u8 hwaddr[6]
Definition: avf.h:180
u16 atq_next_slot
Definition: avf.h:174
u32 vlib_pci_get_numa_node(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:170
#define AVF_AQ_F_BUF
Definition: virtchnl.h:60
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
u32 numa_node
Definition: avf.h:157
static void vlib_physmem_free(vlib_main_t *vm, void *p)
Definition: physmem_funcs.h:89
vlib_node_registration_t avf_input_node
(constructor) VLIB_REGISTER_NODE (avf_input_node)
Definition: input.c:462
#define avf_log_debug(dev, f,...)
Definition: avf.h:60
sll srl srl sll sra u16x4 i
Definition: vector_sse42.h:317
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:380
Definition: avf.h:121
vlib_log_class_t log_class
Definition: avf.h:231
elog_main_t elog_main
Definition: main.h:224
avf_tx_desc_t * descs
Definition: avf.h:141
#define ETHERNET_INTERFACE_FLAG_ACCEPT_ALL
Definition: ethernet.h:152
static void avf_irq_n_set_state(avf_device_t *ad, u8 line, avf_irq_state_t state)
Definition: device.c:90
#define ELOG_TYPE_DECLARE(f)
Definition: elog.h:442
virtchnl_ops_t v_opcode
Definition: virtchnl.h:249
static_always_inline avf_device_t * avf_get_device(u32 dev_instance)
Definition: avf.h:262
#define AVFINT_DYN_CTL0
Definition: virtchnl.h:38
u16 vsi_id
Definition: avf.h:178
u32 per_interface_next_index
Definition: avf.h:151
u32 feature_bitmap
Definition: avf.h:179
virtchnl_status_code_t v_retval
Definition: virtchnl.h:254
u32 * bufs
Definition: avf.h:142
#define AVF_ITR_INT
Definition: device.c:30
clib_error_t * avf_op_enable_queues(vlib_main_t *vm, avf_device_t *ad, u32 rx, u32 tx)
Definition: device.c:766
#define pool_put_index(p, i)
Free pool element with given index.
Definition: pool.h:331
#define ASSERT(truth)
#define AVF_RXQ_SZ
Definition: device.c:28
avf_aq_desc_t * atq
Definition: avf.h:168
void vnet_hw_interface_assign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id, uword thread_index)
Definition: devices.c:139
u32 flags
Definition: avf.h:150
union virtchnl_pf_event_t::@33 event_data
#define AVFINT_DYN_CTLN(x)
Definition: virtchnl.h:35
Definition: avf.h:134
u32 * bufs
Definition: avf.h:128
struct virtchnl_pf_event_t::@33::@34 link_event
void * bar0
Definition: avf.h:158
static void clib_mem_free(void *p)
Definition: mem.h:215
#define AVF_MBOX_BUF_SZ
Definition: device.c:27
u16 n_enqueued
Definition: avf.h:143
clib_error_t * avf_txq_init(vlib_main_t *vm, avf_device_t *ad, u16 qid, u16 txq_size)
Definition: device.c:291
u16 n_enqueued
Definition: avf.h:129
u8 * format_hex_bytes_no_wrap(u8 *s, va_list *va)
Definition: std-formats.c:112
virtchnl_pf_event_t * events
Definition: avf.h:176
virtchnl_event_codes_t event
Definition: virtchnl.h:215
#define AVF_AQ_ENQ_MAX_WAIT_TIME
Definition: avf.h:26
static void avf_reg_write(avf_device_t *ad, u32 addr, u32 val)
Definition: avf.h:308
static uword pointer_to_uword(const void *p)
Definition: types.h:131
static char * avf_tx_func_error_strings[]
Definition: device.c:1663
#define clib_max(x, y)
Definition: clib.h:320
virtchnl_eth_stats_t eth_stats
Definition: avf.h:191
void * atq_bufs
Definition: avf.h:170
static vlib_main_t * vlib_get_main(void)
Definition: global_funcs.h:23
vl_api_ip4_address_t hi
Definition: arp.api:37
clib_error_t * avf_config_promisc_mode(vlib_main_t *vm, avf_device_t *ad, int is_enable)
Definition: device.c:632
#define AVFGEN_RSTAT
Definition: virtchnl.h:49
u16 num_queue_pairs
Definition: avf.h:181
u16 next
Definition: avf.h:138
#define PCI_DEVICE_ID_INTEL_X722_VF
Definition: device.c:35
clib_error_t * avf_op_config_rss_key(vlib_main_t *vm, avf_device_t *ad)
Definition: device.c:599
virtchnl_rxq_info_t rxq
Definition: virtchnl.h:303
static u64 vlib_physmem_get_pa(vlib_main_t *vm, void *mem)
#define AVF_ARQBAL
Definition: virtchnl.h:42
u32 rss_lut_size
Definition: avf.h:186
u16 n_tx_queues
Definition: avf.h:164
static u32 vlib_get_current_process_node_index(vlib_main_t *vm)
Definition: node_funcs.h:459
#define AVF_AQ_F_CMP
Definition: virtchnl.h:54
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
clib_error_t * ethernet_register_interface(vnet_main_t *vnm, u32 dev_class_index, u32 dev_instance, const u8 *address, u32 *hw_if_index_return, ethernet_flag_change_function_t flag_change)
Definition: interface.c:331
u32 instance
Definition: gre.api:51
clib_error_t * avf_cmd_rx_ctl_reg_write(vlib_main_t *vm, avf_device_t *ad, u32 reg, u32 val)
Definition: device.c:209
format_function_t format_avf_device_name
Definition: avf.h:258
clib_error_t * vnet_hw_interface_set_flags(vnet_main_t *vnm, u32 hw_if_index, vnet_hw_interface_flags_t flags)
Definition: interface.c:498
avf_main_t avf_main
Definition: device.c:37
#define foreach_virtchnl_op
Definition: virtchnl.h:66
VLIB buffer representation.
Definition: buffer.h:102
#define foreach_avf_tx_func_error
Definition: avf.h:346
u64 uword
Definition: types.h:112
u16 size
Definition: avf.h:126
u16 arq_next_slot
Definition: avf.h:175
#define clib_error_free(e)
Definition: error.h:86
clib_error_t * vlib_pci_map_region(vlib_main_t *vm, vlib_pci_dev_handle_t h, u32 resource, void **result)
Definition: pci.c:1168
clib_error_t * avf_op_get_stats(vlib_main_t *vm, avf_device_t *ad, virtchnl_eth_stats_t *es)
Definition: device.c:792
avf_rxq_t * rxqs
Definition: avf.h:162
virtchnl_vsi_resource_t vsi_res[1]
Definition: virtchnl.h:179
int vnet_hw_interface_unassign_rx_thread(vnet_main_t *vnm, u32 hw_if_index, u16 queue_id)
Definition: devices.c:188
int vlib_pci_supports_virtual_addr_dma(vlib_main_t *vm, vlib_pci_dev_handle_t h)
Definition: pci.c:1229
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:165
clib_error_t * error
Definition: avf.h:247
static uword avf_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: device.c:1191
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
void vlib_worker_thread_barrier_release(vlib_main_t *vm)
Definition: threads.c:1554
avf_per_thread_data_t * per_thread_data
Definition: avf.h:229
static uword vlib_buffer_get_va(vlib_buffer_t *b)
Definition: buffer.h:217
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
u16 size
Definition: avf.h:139
vlib_pci_addr_t pci_addr
Definition: avf.h:188
vl_api_dhcp_client_state_t state
Definition: dhcp.api:201
u32 sw_if_index
Definition: avf.h:154
#define ETHERNET_MAX_PACKET_BYTES
Definition: ethernet.h:133
#define vec_foreach(var, vec)
Vector iterator.
#define vlib_log_err(...)
Definition: log.h:105
u64 arq_bufs_pa
Definition: avf.h:173
#define CLIB_MEMORY_BARRIER()
Definition: clib.h:132
static clib_error_t * avf_interface_admin_up_down(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: device.c:1591
#define avf_log_err(dev, f,...)
Definition: avf.h:50
#define AVF_ATQBAL
Definition: virtchnl.h:46
#define pool_foreach_index(i, v, body)
Iterate pool by index.
Definition: pool.h:558
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static clib_error_t * avf_interface_rx_mode_change(vnet_main_t *vnm, u32 hw_if_index, u32 qid, vnet_hw_interface_rx_mode mode)
Definition: device.c:1615
#define VIRTCHNL_VERSION_MINOR
Definition: virtchnl.h:22
static u8 vlib_buffer_pool_get_default_for_numa(vlib_main_t *vm, u32 numa_node)
Definition: buffer_funcs.h:199
#define clib_ring_free(f)
Definition: ring.h:59
#define AVF_MBOX_LEN
Definition: device.c:26
clib_error_t * vlib_pci_device_open(vlib_main_t *vm, vlib_pci_addr_t *addr, pci_device_id_t ids[], vlib_pci_dev_handle_t *handle)
Definition: pci.c:1237
u16 next
Definition: avf.h:125
#define VLIB_INITS(...)
Definition: init.h:357
static vlib_buffer_t * vlib_get_buffer(vlib_main_t *vm, u32 buffer_index)
Translate buffer index into buffer pointer.
Definition: buffer_funcs.h:85
static void vnet_hw_interface_set_link_speed(vnet_main_t *vnm, u32 hw_if_index, u32 link_speed)
static void vnet_hw_interface_set_input_node(vnet_main_t *vnm, u32 hw_if_index, u32 node_index)
Definition: devices.h:79
avf_txq_t * txqs
Definition: avf.h:163
avf_rx_desc_t * descs
Definition: avf.h:127
u16 vendor_id
Definition: pci.h:127
format_function_t format_vlib_pci_addr
Definition: pci.h:324
#define AVF_AQ_F_ERR
Definition: virtchnl.h:55
static __clib_warn_unused_result u32 vlib_buffer_alloc_from_pool(vlib_main_t *vm, u32 *buffers, u32 n_buffers, u8 buffer_pool_index)
Allocate buffers from specific pool into supplied array.
Definition: buffer_funcs.h:566
u32 ethernet_set_flags(vnet_main_t *vnm, u32 hw_if_index, u32 flags)
Definition: interface.c:426
u16 max_vectors
Definition: avf.h:182
u32 rss_key_size
Definition: avf.h:185
u16 max_mtu
Definition: avf.h:184
static uword pool_elts(void *v)
Number of active elements in a pool.
Definition: pool.h:128