FD.io VPP  v19.04-6-g6f05f72
Vector Packet Processing
ipsec.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Intel and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vnet/ip/ip.h>
17 #include <vnet/api_errno.h>
18 #include <vnet/ipsec/ipsec.h>
19 #include <vlib/node_funcs.h>
20 
21 #include <dpdk/device/dpdk.h>
22 #include <dpdk/buffer.h>
23 #include <dpdk/ipsec/ipsec.h>
24 
26 
27 #define EMPTY_STRUCT {0}
28 #define NUM_CRYPTO_MBUFS 16384
29 
30 static void
31 algos_init (u32 n_mains)
32 {
34  crypto_alg_t *a;
35 
37 
38  {
39 #define _(v,f,str) \
40  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].name = str; \
41  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].disabled = n_mains;
43 #undef _
44  }
45 
46  /* Minimum boundary for ciphers is 4B, required by ESP */
47  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_NONE];
48  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
49  a->alg = RTE_CRYPTO_CIPHER_NULL;
50  a->boundary = 4; /* 1 */
51  a->key_len = 0;
52  a->iv_len = 0;
53 
54  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
55  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
56  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
57  a->boundary = 16;
58  a->key_len = 16;
59  a->iv_len = 16;
60 
61  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
62  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
63  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
64  a->boundary = 16;
65  a->key_len = 24;
66  a->iv_len = 16;
67 
68  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
69  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
70  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
71  a->boundary = 16;
72  a->key_len = 32;
73  a->iv_len = 16;
74 
75  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_128];
76  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
77  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
78  a->boundary = 4; /* 1 */
79  a->key_len = 16;
80  a->iv_len = 8;
81 
82  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_192];
83  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
84  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
85  a->boundary = 4; /* 1 */
86  a->key_len = 24;
87  a->iv_len = 8;
88 
89  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_256];
90  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
91  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
92  a->boundary = 4; /* 1 */
93  a->key_len = 32;
94  a->iv_len = 8;
95 
96 #define AES_GCM_TYPE RTE_CRYPTO_SYM_XFORM_AEAD
97 #define AES_GCM_ALG RTE_CRYPTO_AEAD_AES_GCM
98 
99  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
100  a->type = AES_GCM_TYPE;
101  a->alg = AES_GCM_ALG;
102  a->boundary = 4; /* 1 */
103  a->key_len = 16;
104  a->iv_len = 8;
105  a->trunc_size = 16;
106 
107  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_192];
108  a->type = AES_GCM_TYPE;
109  a->alg = AES_GCM_ALG;
110  a->boundary = 4; /* 1 */
111  a->key_len = 24;
112  a->iv_len = 8;
113  a->trunc_size = 16;
114 
115  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_256];
116  a->type = AES_GCM_TYPE;
117  a->alg = AES_GCM_ALG;
118  a->boundary = 4; /* 1 */
119  a->key_len = 32;
120  a->iv_len = 8;
121  a->trunc_size = 16;
122 
124 
125  {
126 #define _(v,f,str) \
127  dcm->auth_algs[IPSEC_INTEG_ALG_##f].name = str; \
128  dcm->auth_algs[IPSEC_INTEG_ALG_##f].disabled = n_mains;
130 #undef _
131  }
132 
133  a = &dcm->auth_algs[IPSEC_INTEG_ALG_NONE];
134  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
135  a->alg = RTE_CRYPTO_AUTH_NULL;
136  a->key_len = 0;
137  a->trunc_size = 0;
138 
139  a = &dcm->auth_algs[IPSEC_INTEG_ALG_MD5_96];
140  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
141  a->alg = RTE_CRYPTO_AUTH_MD5_HMAC;
142  a->key_len = 16;
143  a->trunc_size = 12;
144 
145  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA1_96];
146  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
147  a->alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
148  a->key_len = 20;
149  a->trunc_size = 12;
150 
151  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_96];
152  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
153  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
154  a->key_len = 32;
155  a->trunc_size = 12;
156 
157  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_128];
158  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
159  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
160  a->key_len = 32;
161  a->trunc_size = 16;
162 
163  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_384_192];
164  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
165  a->alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
166  a->key_len = 48;
167  a->trunc_size = 24;
168 
169  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_512_256];
170  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
171  a->alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
172  a->key_len = 64;
173  a->trunc_size = 32;
174 }
175 
176 static u8
178 {
180 
181  return (alg - dcm->cipher_algs);
182 }
183 
184 static u8
186 {
188 
189  return (alg - dcm->auth_algs);
190 }
191 
192 static crypto_alg_t *
193 cipher_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 key_len)
194 {
196  crypto_alg_t *alg;
197 
198  if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
199  return NULL;
200 
201  /* *INDENT-OFF* */
202  vec_foreach (alg, dcm->cipher_algs)
203  {
204  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
205  (alg->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
206  (cap->sym.cipher.algo == alg->alg) &&
207  (alg->key_len == key_len))
208  return alg;
209  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
210  (alg->type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
211  (cap->sym.aead.algo == alg->alg) &&
212  (alg->key_len == key_len))
213  return alg;
214  }
215  /* *INDENT-ON* */
216 
217  return NULL;
218 }
219 
220 static crypto_alg_t *
221 auth_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
222 {
224  crypto_alg_t *alg;
225 
226  if ((cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) ||
227  (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH))
228  return NULL;
229 
230  /* *INDENT-OFF* */
231  vec_foreach (alg, dcm->auth_algs)
232  {
233  if ((cap->sym.auth.algo == alg->alg) &&
234  (alg->trunc_size == trunc_size))
235  return alg;
236  }
237  /* *INDENT-ON* */
238 
239  return NULL;
240 }
241 
242 static void
243 crypto_set_aead_xform (struct rte_crypto_sym_xform *xform,
244  ipsec_sa_t * sa, u8 is_outbound)
245 {
247  crypto_alg_t *c;
248 
249  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
250 
251  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_AEAD);
252 
253  xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
254  xform->aead.algo = c->alg;
255  xform->aead.key.data = sa->crypto_key.data;
256  xform->aead.key.length = c->key_len;
257  xform->aead.iv.offset =
258  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
259  xform->aead.iv.length = 12;
260  xform->aead.digest_length = c->trunc_size;
261  xform->aead.aad_length = ipsec_sa_is_set_USE_ESN (sa) ? 12 : 8;
262  xform->next = NULL;
263 
264  if (is_outbound)
265  xform->aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
266  else
267  xform->aead.op = RTE_CRYPTO_AEAD_OP_DECRYPT;
268 }
269 
270 static void
271 crypto_set_cipher_xform (struct rte_crypto_sym_xform *xform,
272  ipsec_sa_t * sa, u8 is_outbound)
273 {
275  crypto_alg_t *c;
276 
277  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
278 
279  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_CIPHER);
280 
281  xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
282  xform->cipher.algo = c->alg;
283  xform->cipher.key.data = sa->crypto_key.data;
284  xform->cipher.key.length = c->key_len;
285  xform->cipher.iv.offset =
286  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
287  xform->cipher.iv.length = c->iv_len;
288  xform->next = NULL;
289 
290  if (is_outbound)
291  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
292  else
293  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
294 }
295 
296 static void
297 crypto_set_auth_xform (struct rte_crypto_sym_xform *xform,
298  ipsec_sa_t * sa, u8 is_outbound)
299 {
301  crypto_alg_t *a;
302 
303  a = vec_elt_at_index (dcm->auth_algs, sa->integ_alg);
304 
305  ASSERT (a->type == RTE_CRYPTO_SYM_XFORM_AUTH);
306 
307  xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
308  xform->auth.algo = a->alg;
309  xform->auth.key.data = sa->integ_key.data;
310  xform->auth.key.length = a->key_len;
311  xform->auth.digest_length = a->trunc_size;
312  xform->next = NULL;
313 
314  if (is_outbound)
315  xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
316  else
317  xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
318 }
319 
320 clib_error_t *
321 create_sym_session (struct rte_cryptodev_sym_session **session,
322  u32 sa_idx,
323  crypto_resource_t * res,
325 {
327  ipsec_main_t *im = &ipsec_main;
329  ipsec_sa_t *sa;
330  struct rte_crypto_sym_xform cipher_xform = { 0 };
331  struct rte_crypto_sym_xform auth_xform = { 0 };
332  struct rte_crypto_sym_xform *xfs;
333  struct rte_cryptodev_sym_session **s;
334  clib_error_t *error = 0;
335 
336 
337  sa = pool_elt_at_index (im->sad, sa_idx);
338 
339  if ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
340  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
341  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256))
342  {
343  crypto_set_aead_xform (&cipher_xform, sa, is_outbound);
344  xfs = &cipher_xform;
345  }
346  else
347  {
348  crypto_set_cipher_xform (&cipher_xform, sa, is_outbound);
349  crypto_set_auth_xform (&auth_xform, sa, is_outbound);
350 
351  if (is_outbound)
352  {
353  cipher_xform.next = &auth_xform;
354  xfs = &cipher_xform;
355  }
356  else
357  {
358  auth_xform.next = &cipher_xform;
359  xfs = &auth_xform;
360  }
361  }
362 
363  data = vec_elt_at_index (dcm->data, res->numa);
365 
366  /*
367  * DPDK_VER >= 1708:
368  * Multiple worker/threads share the session for an SA
369  * Single session per SA, initialized for each device driver
370  */
371  s = (void *) hash_get (data->session_by_sa_index, sa_idx);
372 
373  if (!s)
374  {
375  session[0] = rte_cryptodev_sym_session_create (data->session_h);
376  if (!session[0])
377  {
378  data->session_h_failed += 1;
379  error = clib_error_return (0, "failed to create session header");
380  goto done;
381  }
382  hash_set (data->session_by_sa_index, sa_idx, session[0]);
383  }
384  else
385  session[0] = s[0];
386 
387  struct rte_mempool **mp;
388  mp = vec_elt_at_index (data->session_drv, res->drv_id);
389  ASSERT (mp[0] != NULL);
390 
391  i32 ret =
392  rte_cryptodev_sym_session_init (res->dev_id, session[0], xfs, mp[0]);
393  if (ret)
394  {
395  data->session_drv_failed[res->drv_id] += 1;
396  error = clib_error_return (0, "failed to init session for drv %u",
397  res->drv_id);
398  goto done;
399  }
400 
401  add_session_by_drv_and_sa_idx (session[0], data, res->drv_id, sa_idx);
402 
403 done:
405  return error;
406 }
407 
408 static void __attribute__ ((unused)) clear_and_free_obj (void *obj)
409 {
410  struct rte_mempool *mp = rte_mempool_from_obj (obj);
411 
412  clib_memset (obj, 0, mp->elt_size);
413 
414  rte_mempool_put (mp, obj);
415 }
416 
417 /* This is from rte_cryptodev_pmd.h */
418 static inline void *
419 get_session_private_data (const struct rte_cryptodev_sym_session *sess,
420  uint8_t driver_id)
421 {
422 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
423  return sess->sess_private_data[driver_id];
424 #else
425  if (unlikely (sess->nb_drivers <= driver_id))
426  return 0;
427 
428  return sess->sess_data[driver_id].data;
429 #endif
430 }
431 
432 /* This is from rte_cryptodev_pmd.h */
433 static inline void
434 set_session_private_data (struct rte_cryptodev_sym_session *sess,
435  uint8_t driver_id, void *private_data)
436 {
437 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
438  sess->sess_private_data[driver_id] = private_data;
439 #else
440  if (unlikely (sess->nb_drivers <= driver_id))
441  return;
442  sess->sess_data[driver_id].data = private_data;
443 #endif
444 }
445 
446 static clib_error_t *
448 {
451  void *drv_session;
452  u32 drv_id;
453  i32 ret;
454 
455  /* *INDENT-OFF* */
456  vec_foreach (s, v)
457  {
458  /* ordered vector by timestamp */
459  if (!(s->ts + dcm->session_timeout < ts))
460  break;
461 
462  vec_foreach_index (drv_id, dcm->drv)
463  {
464  drv_session = get_session_private_data (s->session, drv_id);
465  if (!drv_session)
466  continue;
467 
468  /*
469  * Custom clear to avoid finding a dev_id for drv_id:
470  * ret = rte_cryptodev_sym_session_clear (dev_id, drv_session);
471  * ASSERT (!ret);
472  */
473  clear_and_free_obj (drv_session);
474 
475  set_session_private_data (s->session, drv_id, NULL);
476  }
477 
478  if (rte_mempool_from_obj(s->session))
479  {
480  ret = rte_cryptodev_sym_session_free (s->session);
481  ASSERT (!ret);
482  }
483  }
484  /* *INDENT-ON* */
485 
486  if (s < vec_end (v))
487  vec_delete (v, s - v, 0);
488  else
489  vec_reset_length (v);
490 
491  return 0;
492 }
493 
494 static clib_error_t *
496 {
497  ipsec_main_t *im = &ipsec_main;
500  struct rte_cryptodev_sym_session *s;
501  uword *val;
502  u32 drv_id;
503 
504  if (is_add)
505  {
506 #if 1
507  ipsec_sa_t *sa = pool_elt_at_index (im->sad, sa_index);
508  u32 seed;
509  switch (sa->crypto_alg)
510  {
511  case IPSEC_CRYPTO_ALG_AES_GCM_128:
512  case IPSEC_CRYPTO_ALG_AES_GCM_192:
513  case IPSEC_CRYPTO_ALG_AES_GCM_256:
514  clib_memcpy (&sa->salt,
515  &sa->crypto_key.data[sa->crypto_key.len - 4], 4);
516  break;
517  default:
518  seed = (u32) clib_cpu_time_now ();
519  sa->salt = random_u32 (&seed);
520  }
521 #endif
522  return 0;
523  }
524 
525  /* *INDENT-OFF* */
526  vec_foreach (data, dcm->data)
527  {
529  val = hash_get (data->session_by_sa_index, sa_index);
530  if (val)
531  {
532  s = (struct rte_cryptodev_sym_session *) val[0];
533  vec_foreach_index (drv_id, dcm->drv)
534  {
535  val = (uword*) get_session_by_drv_and_sa_idx (data, drv_id, sa_index);
536  if (val)
537  add_session_by_drv_and_sa_idx(NULL, data, drv_id, sa_index);
538  }
539 
540  hash_unset (data->session_by_sa_index, sa_index);
541 
542  u64 ts = unix_time_now_nsec ();
544 
546  sd.ts = ts;
547  sd.session = s;
548 
549  vec_add1 (data->session_disposal, sd);
550  }
552  }
553  /* *INDENT-ON* */
554 
555  return 0;
556 }
557 
558 static clib_error_t *
560 {
562 
563  if (sa->integ_alg == IPSEC_INTEG_ALG_NONE)
564  switch (sa->crypto_alg)
565  {
566  case IPSEC_CRYPTO_ALG_NONE:
567  case IPSEC_CRYPTO_ALG_AES_GCM_128:
568  case IPSEC_CRYPTO_ALG_AES_GCM_192:
569  case IPSEC_CRYPTO_ALG_AES_GCM_256:
570  break;
571  default:
572  return clib_error_return (0, "unsupported integ-alg %U crypto-alg %U",
575  }
576 
577  /* XXX do we need the NONE check? */
578  if (sa->crypto_alg != IPSEC_CRYPTO_ALG_NONE &&
579  dcm->cipher_algs[sa->crypto_alg].disabled)
580  return clib_error_return (0, "disabled crypto-alg %U",
582 
583  /* XXX do we need the NONE check? */
584  if (sa->integ_alg != IPSEC_INTEG_ALG_NONE &&
585  dcm->auth_algs[sa->integ_alg].disabled)
586  return clib_error_return (0, "disabled integ-alg %U",
588  return NULL;
589 }
590 
591 static void
593  const struct rte_cryptodev_capabilities *cap,
594  u32 n_mains)
595 {
597  crypto_alg_t *alg;
598  u8 len, inc;
599 
600  for (; cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; cap++)
601  {
602  /* A single capability maps to multiple cipher/auth algorithms */
603  switch (cap->sym.xform_type)
604  {
605  case RTE_CRYPTO_SYM_XFORM_AEAD:
606  case RTE_CRYPTO_SYM_XFORM_CIPHER:
607  inc = cap->sym.cipher.key_size.increment;
608  inc = inc ? inc : 1;
609  for (len = cap->sym.cipher.key_size.min;
610  len <= cap->sym.cipher.key_size.max; len += inc)
611  {
612  alg = cipher_cap_to_alg (cap, len);
613  if (!alg)
614  continue;
615  dev->cipher_support[cipher_alg_index (alg)] = 1;
616  alg->resources += vec_len (dev->free_resources);
617  /* At least enough resources to support one algo */
618  dcm->enabled |= (alg->resources >= n_mains);
619  }
620  break;
621  case RTE_CRYPTO_SYM_XFORM_AUTH:
622  inc = cap->sym.auth.digest_size.increment;
623  inc = inc ? inc : 1;
624  for (len = cap->sym.auth.digest_size.min;
625  len <= cap->sym.auth.digest_size.max; len += inc)
626  {
627  alg = auth_cap_to_alg (cap, len);
628  if (!alg)
629  continue;
630  dev->auth_support[auth_alg_index (alg)] = 1;
631  alg->resources += vec_len (dev->free_resources);
632  /* At least enough resources to support one algo */
633  dcm->enabled |= (alg->resources >= n_mains);
634  }
635  break;
636  default:
637  ;
638  }
639  }
640 }
641 
642 #define DPDK_CRYPTO_N_QUEUE_DESC 2048
643 #define DPDK_CRYPTO_NB_SESS_OBJS 20000
644 
645 static clib_error_t *
646 crypto_dev_conf (u8 dev, u16 n_qp, u8 numa)
647 {
648  struct rte_cryptodev_config dev_conf = { 0 };
649  struct rte_cryptodev_qp_conf qp_conf = { 0 };
650  i32 ret;
651  u16 qp;
652  char *error_str;
653 
654  dev_conf.socket_id = numa;
655  dev_conf.nb_queue_pairs = n_qp;
656 
657  error_str = "failed to configure crypto device %u";
658  ret = rte_cryptodev_configure (dev, &dev_conf);
659  if (ret < 0)
660  return clib_error_return (0, error_str, dev);
661 
662  error_str = "failed to setup crypto device %u queue pair %u";
663  qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
664  for (qp = 0; qp < n_qp; qp++)
665  {
666 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
667  ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa, NULL);
668 #else
669  ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa);
670 #endif
671  if (ret < 0)
672  return clib_error_return (0, error_str, dev, qp);
673  }
674 
675  error_str = "failed to start crypto device %u";
676  if (rte_cryptodev_start (dev))
677  return clib_error_return (0, error_str, dev);
678 
679  return 0;
680 }
681 
682 static void
684 {
686  struct rte_cryptodev *cryptodev;
687  struct rte_cryptodev_info info = { 0 };
688  crypto_dev_t *dev;
689  crypto_resource_t *res;
690  clib_error_t *error;
691  u32 i;
692  u16 max_res_idx, res_idx, j;
693  u8 drv_id;
694 
695  vec_validate_init_empty (dcm->dev, rte_cryptodev_count () - 1,
697 
698  for (i = 0; i < rte_cryptodev_count (); i++)
699  {
700  dev = vec_elt_at_index (dcm->dev, i);
701 
702  cryptodev = &rte_cryptodevs[i];
703  rte_cryptodev_info_get (i, &info);
704 
705  dev->id = i;
706  dev->name = cryptodev->data->name;
707  dev->numa = rte_cryptodev_socket_id (i);
708  dev->features = info.feature_flags;
709  dev->max_qp = info.max_nb_queue_pairs;
710  drv_id = info.driver_id;
711  if (drv_id >= vec_len (dcm->drv))
712  vec_validate_init_empty (dcm->drv, drv_id,
713  (crypto_drv_t) EMPTY_STRUCT);
714  vec_elt_at_index (dcm->drv, drv_id)->name = info.driver_name;
715  dev->drv_id = drv_id;
716  vec_add1 (vec_elt_at_index (dcm->drv, drv_id)->devs, i);
717 
718  if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
719  continue;
720 
721  if ((error = crypto_dev_conf (i, dev->max_qp, dev->numa)))
722  {
723  clib_error_report (error);
724  continue;
725  }
726 
727  max_res_idx = dev->max_qp - 1;
728 
729  vec_validate (dev->free_resources, max_res_idx);
730 
731  res_idx = vec_len (dcm->resource);
732  vec_validate_init_empty_aligned (dcm->resource, res_idx + max_res_idx,
733  (crypto_resource_t) EMPTY_STRUCT,
735 
736  for (j = 0; j <= max_res_idx; j++)
737  {
738  vec_elt (dev->free_resources, max_res_idx - j) = res_idx + j;
739  res = &dcm->resource[res_idx + j];
740  res->dev_id = i;
741  res->drv_id = drv_id;
742  res->qp_id = j;
743  res->numa = dev->numa;
744  res->thread_idx = (u16) ~ 0;
745  }
746 
747  crypto_parse_capabilities (dev, info.capabilities, n_mains);
748  }
749 }
750 
751 void
753 {
755  crypto_resource_t *res;
757  crypto_dev_t *dev;
758  u32 thread_idx, skip_master;
759  u16 res_idx, *idx;
760  u8 used;
761  u16 i;
762 
763  skip_master = vlib_num_workers () > 0;
764 
765  /* *INDENT-OFF* */
766  vec_foreach (dev, dcm->dev)
767  {
768  vec_foreach_index (thread_idx, dcm->workers_main)
769  {
770  if (vec_len (dev->free_resources) == 0)
771  break;
772 
773  if (thread_idx < skip_master)
774  continue;
775 
776  /* Check thread is not already using the device */
777  vec_foreach (idx, dev->used_resources)
778  if (dcm->resource[idx[0]].thread_idx == thread_idx)
779  continue;
780 
781  cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
782 
783  used = 0;
784  res_idx = vec_pop (dev->free_resources);
785 
786  /* Set device only for supported algos */
787  for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
788  if (dev->cipher_support[i] &&
789  cwm->cipher_resource_idx[i] == (u16) ~0)
790  {
791  dcm->cipher_algs[i].disabled--;
792  cwm->cipher_resource_idx[i] = res_idx;
793  used = 1;
794  }
795 
796  for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
797  if (dev->auth_support[i] &&
798  cwm->auth_resource_idx[i] == (u16) ~0)
799  {
800  dcm->auth_algs[i].disabled--;
801  cwm->auth_resource_idx[i] = res_idx;
802  used = 1;
803  }
804 
805  if (!used)
806  {
807  vec_add1 (dev->free_resources, res_idx);
808  continue;
809  }
810 
811  vec_add1 (dev->used_resources, res_idx);
812 
813  res = vec_elt_at_index (dcm->resource, res_idx);
814 
815  ASSERT (res->thread_idx == (u16) ~0);
816  res->thread_idx = thread_idx;
817 
818  /* Add device to vector of polling resources */
819  vec_add1 (cwm->resource_idx, res_idx);
820  }
821  }
822  /* *INDENT-ON* */
823 }
824 
825 static void
826 crypto_op_init (struct rte_mempool *mempool,
827  void *_arg __attribute__ ((unused)),
828  void *_obj, unsigned i __attribute__ ((unused)))
829 {
830  struct rte_crypto_op *op = _obj;
831 
832  op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
833  op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
834  op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
835  op->phys_addr = rte_mempool_virt2iova (_obj);
836  op->mempool = mempool;
837 }
838 
839 static clib_error_t *
841 {
845  u8 *pool_name;
846  u32 pool_priv_size = sizeof (struct rte_crypto_op_pool_private);
847  struct rte_crypto_op_pool_private *priv;
848  struct rte_mempool *mp;
849 
850  data = vec_elt_at_index (dcm->data, numa);
851 
852  /* Already allocated */
853  if (data->crypto_op)
854  return NULL;
855 
856  pool_name = format (0, "crypto_pool_numa%u%c", numa, 0);
857 
858  if (conf->num_crypto_mbufs == 0)
860 
861  mp = rte_mempool_create ((char *) pool_name, conf->num_crypto_mbufs,
862  crypto_op_len (), 512, pool_priv_size, NULL, NULL,
863  crypto_op_init, NULL, numa, 0);
864 
865  vec_free (pool_name);
866 
867  if (!mp)
868  return clib_error_return (0, "failed to create crypto op mempool");
869 
870  /* Initialize mempool private data */
871  priv = rte_mempool_get_priv (mp);
872  priv->priv_size = pool_priv_size;
873  priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
874 
875  data->crypto_op = mp;
876 
877  return NULL;
878 }
879 
880 static clib_error_t *
882 {
885  u8 *pool_name;
886  struct rte_mempool *mp;
887  u32 elt_size;
888 
889  data = vec_elt_at_index (dcm->data, numa);
890 
891  if (data->session_h)
892  return NULL;
893 
894  pool_name = format (0, "session_h_pool_numa%u%c", numa, 0);
895 
896 
897  elt_size = rte_cryptodev_sym_get_header_session_size ();
898 
899 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
900  mp = rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
901  elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
902 #else
903  /* XXX Experimental tag in DPDK 19.02 */
904  mp = rte_cryptodev_sym_session_pool_create ((char *) pool_name,
906  elt_size, 512, 0, numa);
907 #endif
908  vec_free (pool_name);
909 
910  if (!mp)
911  return clib_error_return (0, "failed to create crypto session mempool");
912 
913  data->session_h = mp;
914 
915  return NULL;
916 }
917 
918 static clib_error_t *
920 {
923  u8 *pool_name;
924  struct rte_mempool *mp;
925  u32 elt_size;
926  u8 numa = dev->numa;
927 
928  data = vec_elt_at_index (dcm->data, numa);
929 
930  vec_validate (data->session_drv, dev->drv_id);
931  vec_validate (data->session_drv_failed, dev->drv_id);
934 
935  if (data->session_drv[dev->drv_id])
936  return NULL;
937 
938  pool_name = format (0, "session_drv%u_pool_numa%u%c", dev->drv_id, numa, 0);
939 
940  elt_size = rte_cryptodev_sym_get_private_session_size (dev->id);
941  mp =
942  rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
943  elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
944 
945  vec_free (pool_name);
946 
947  if (!mp)
948  return clib_error_return (0, "failed to create session drv mempool");
949 
950  data->session_drv[dev->drv_id] = mp;
951  clib_spinlock_init (&data->lockp);
952 
953  return NULL;
954 }
955 
956 static clib_error_t *
958 {
960  clib_error_t *error = NULL;
961  crypto_dev_t *dev;
962 
963  /* *INDENT-OFF* */
964  vec_foreach (dev, dcm->dev)
965  {
967 
968  error = crypto_create_crypto_op_pool (vm, dev->numa);
969  if (error)
970  return error;
971 
972  error = crypto_create_session_h_pool (vm, dev->numa);
973  if (error)
974  return error;
975 
976  error = crypto_create_session_drv_pool (vm, dev);
977  if (error)
978  return error;
979  }
980  /* *INDENT-ON* */
981 
982  return NULL;
983 }
984 
985 static void
987 {
990  u8 i;
991 
992  dcm->enabled = 0;
993 
994  /* *INDENT-OFF* */
995  vec_foreach (data, dcm->data)
996  {
997  rte_mempool_free (data->crypto_op);
998  rte_mempool_free (data->session_h);
999 
1000  vec_foreach_index (i, data->session_drv)
1001  rte_mempool_free (data->session_drv[i]);
1002 
1003  vec_free (data->session_drv);
1004  clib_spinlock_free (&data->lockp);
1005  }
1006  /* *INDENT-ON* */
1007 
1008  vec_free (dcm->data);
1009  vec_free (dcm->workers_main);
1010  vec_free (dcm->dev);
1011  vec_free (dcm->resource);
1012  vec_free (dcm->cipher_algs);
1013  vec_free (dcm->auth_algs);
1014 }
1015 
1016 static uword
1018  vlib_frame_t * f)
1019 {
1020  ipsec_main_t *im = &ipsec_main;
1023  crypto_worker_main_t *cwm;
1024  clib_error_t *error = NULL;
1025  u32 i, skip_master, n_mains;
1026 
1027  n_mains = tm->n_vlib_mains;
1028  skip_master = vlib_num_workers () > 0;
1029 
1030  algos_init (n_mains - skip_master);
1031 
1032  crypto_scan_devs (n_mains - skip_master);
1033 
1034  if (!(dcm->enabled))
1035  {
1036  clib_warning ("not enough DPDK crypto resources, default to OpenSSL");
1037  crypto_disable ();
1038  return 0;
1039  }
1040 
1041  dcm->session_timeout = 10e9;
1042 
1043  vec_validate_init_empty_aligned (dcm->workers_main, n_mains - 1,
1046 
1047  /* *INDENT-OFF* */
1048  vec_foreach (cwm, dcm->workers_main)
1049  {
1052  clib_memset (cwm->cipher_resource_idx, ~0,
1053  IPSEC_CRYPTO_N_ALG * sizeof(*cwm->cipher_resource_idx));
1054  clib_memset (cwm->auth_resource_idx, ~0,
1055  IPSEC_INTEG_N_ALG * sizeof(*cwm->auth_resource_idx));
1056  }
1057  /* *INDENT-ON* */
1058 
1060 
1061  error = crypto_create_pools (vm);
1062  if (error)
1063  {
1064  clib_error_report (error);
1065  crypto_disable ();
1066  return 0;
1067  }
1068 
1069 
1070  u32 idx = ipsec_register_esp_backend (vm, im, "dpdk backend",
1071  "dpdk-esp4-encrypt",
1072  "dpdk-esp4-encrypt-tun",
1073  "dpdk-esp4-decrypt",
1074  "dpdk-esp6-encrypt",
1075  "dpdk-esp6-encrypt-tun",
1076  "dpdk-esp6-decrypt",
1079  int rv = ipsec_select_esp_backend (im, idx);
1080  ASSERT (rv == 0);
1081 
1082  vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "dpdk-crypto-input");
1083  ASSERT (node);
1084  for (i = skip_master; i < n_mains; i++)
1085  vlib_node_set_state (vlib_mains[i], node->index, VLIB_NODE_STATE_POLLING);
1086  return 0;
1087 }
1088 
1089 /* *INDENT-OFF* */
1090 VLIB_REGISTER_NODE (dpdk_ipsec_process_node,static) = {
1091  .function = dpdk_ipsec_process,
1092  .type = VLIB_NODE_TYPE_PROCESS,
1093  .name = "dpdk-ipsec-process",
1094  .process_log2_n_stack_bytes = 17,
1095 };
1096 /* *INDENT-ON* */
1097 
1098 /*
1099  * fd.io coding-style-patch-verification: ON
1100  *
1101  * Local Variables:
1102  * eval: (c-set-style "gnu")
1103  * End:
1104  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
u32 alg
Definition: ipsec.h:81
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:109
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:255
#define hash_unset(h, key)
Definition: hash.h:261
a
Definition: bitmap.h:538
u8 numa
Definition: ipsec.h:97
static u8 cipher_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:177
unsigned long u64
Definition: types.h:89
#define NULL
Definition: clib.h:58
u32 index
Definition: node.h:279
u64 session_h_failed
Definition: ipsec.h:148
#define foreach_ipsec_crypto_alg
Definition: ipsec_sa.h:24
#define DPDK_CRYPTO_NB_SESS_OBJS
Definition: ipsec.c:643
ipsec_key_t crypto_key
Definition: ipsec_sa.h:151
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
ipsec_integ_alg_t integ_alg
Definition: ipsec_sa.h:153
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static clib_error_t * crypto_create_session_drv_pool(vlib_main_t *vm, crypto_dev_t *dev)
Definition: ipsec.c:919
static u64 clib_cpu_time_now(void)
Definition: time.h:75
#define EMPTY_STRUCT
Definition: ipsec.c:27
static_always_inline u32 crypto_op_get_priv_offset(void)
Definition: ipsec.h:189
int i
#define foreach_ipsec_integ_alg
Definition: ipsec_sa.h:51
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
struct rte_cryptodev_sym_session * session
Definition: ipsec.h:128
u8 disabled
Definition: ipsec.h:86
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
u8 data[128]
Definition: ipsec.api:248
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
static void crypto_set_cipher_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:271
vlib_main_t ** vlib_mains
Definition: buffer.c:321
static clib_error_t * crypto_create_session_h_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:881
unsigned char u8
Definition: types.h:56
#define vec_pop(V)
Returns last element of a vector and decrements its length.
Definition: vec.h:615
static crypto_alg_t * cipher_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 key_len)
Definition: ipsec.c:193
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:64
static clib_error_t * add_del_sa_session(u32 sa_index, u8 is_add)
Definition: ipsec.c:495
#define clib_memcpy(d, s, n)
Definition: string.h:180
u16 * resource_idx
Definition: ipsec.h:69
#define AES_GCM_TYPE
u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:71
static void algos_init(u32 n_mains)
Definition: ipsec.c:31
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:25
dpdk_config_main_t dpdk_config_main
Definition: init.c:45
ipsec_main_t ipsec_main
Definition: ipsec.c:28
u8 resources
Definition: ipsec.h:87
int ipsec_select_esp_backend(ipsec_main_t *im, u32 backend_idx)
Definition: ipsec.c:218
#define NUM_CRYPTO_MBUFS
Definition: ipsec.c:28
crypto_drv_t * drv
Definition: ipsec.h:162
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 drv_id
Definition: ipsec.h:96
#define clib_error_return(e, args...)
Definition: error.h:99
static void crypto_parse_capabilities(crypto_dev_t *dev, const struct rte_cryptodev_capabilities *cap, u32 n_mains)
Definition: ipsec.c:592
u16 * free_resources
Definition: ipsec.h:92
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:77
unsigned int u32
Definition: types.h:88
#define vec_end(v)
End (last data address) of vector.
crypto_alg_t * auth_algs
Definition: ipsec.h:160
#define VLIB_FRAME_SIZE
Definition: node.h:376
static clib_error_t * crypto_dev_conf(u8 dev, u16 n_qp, u8 numa)
Definition: ipsec.c:646
u8 trunc_size
Definition: ipsec.h:84
static_always_inline void add_session_by_drv_and_sa_idx(struct rte_cryptodev_sym_session *session, crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:208
static void crypto_disable(void)
Definition: ipsec.c:986
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
clib_spinlock_t lockp
Definition: ipsec.h:151
static void * get_session_private_data(const struct rte_cryptodev_sym_session *sess, uint8_t driver_id)
Definition: ipsec.c:419
u16 * used_resources
Definition: ipsec.h:93
#define hash_get(h, key)
Definition: hash.h:249
u32 ipsec_register_esp_backend(vlib_main_t *vm, ipsec_main_t *im, const char *name, const char *esp4_encrypt_node_name, const char *esp4_encrypt_node_tun_name, const char *esp4_decrypt_node_name, const char *esp6_encrypt_node_name, const char *esp6_encrypt_node_tun_name, const char *esp6_decrypt_node_name, check_support_cb_t esp_check_support_cb, add_del_sa_sess_cb_t esp_add_del_sa_sess_cb)
Definition: ipsec.c:142
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
crypto_alg_t * cipher_algs
Definition: ipsec.h:159
u32 salt
Definition: ipsec_sa.h:163
unsigned short u16
Definition: types.h:57
u64 features
Definition: ipsec.h:101
u64 session_timeout
Definition: ipsec.h:163
static_always_inline u32 crypto_op_len(void)
Definition: ipsec.h:179
#define AES_GCM_ALG
static clib_error_t * crypto_create_crypto_op_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:840
static u8 auth_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:185
struct rte_mempool ** session_drv
Definition: ipsec.h:144
u8 len
Definition: ip_types.api:49
vlib node functions
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
crypto_session_by_drv_t * session_by_drv_id_and_sa_index
Definition: ipsec.h:150
u32 num_crypto_mbufs
Definition: dpdk.h:367
svmdb_client_t * c
vlib_main_t * vm
Definition: buffer.c:312
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
#define clib_warning(format, args...)
Definition: error.h:59
u8 is_outbound
Definition: ipsec.api:92
static u64 unix_time_now_nsec(void)
Definition: time.h:255
crypto_session_disposal_t * session_disposal
Definition: ipsec.h:145
static void crypto_set_aead_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:243
vlib_node_t * vlib_get_node_by_name(vlib_main_t *vm, u8 *name)
Definition: node.c:45
u16 id
Definition: ipsec.h:98
static void set_session_private_data(struct rte_cryptodev_sym_session *sess, uint8_t driver_id, void *private_data)
Definition: ipsec.c:434
u8 iv_len
Definition: ipsec.h:83
signed int i32
Definition: types.h:77
u8 data[IPSEC_KEY_MAX_LEN]
Definition: ipsec_sa.h:80
#define ASSERT(truth)
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:784
u8 is_add
Definition: ipsec_gre.api:36
struct rte_mempool * crypto_op
Definition: ipsec.h:142
u32 max_qp
Definition: ipsec.h:100
ipsec_sa_t * sad
Definition: ipsec.h:95
static clib_error_t * dpdk_crypto_session_disposal(crypto_session_disposal_t *v, u64 ts)
Definition: ipsec.c:447
crypto_worker_main_t * workers_main
Definition: ipsec.h:156
#define clib_error_report(e)
Definition: error.h:113
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
Definition: node_funcs.h:148
crypto_resource_t * resource
Definition: ipsec.h:158
#define vec_elt(v, i)
Get vector value at index i.
static clib_error_t * crypto_create_pools(vlib_main_t *vm)
Definition: ipsec.c:957
struct rte_mempool * session_h
Definition: ipsec.h:143
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void crypto_auto_placement(void)
Definition: ipsec.c:752
u64 * session_drv_failed
Definition: ipsec.h:149
u64 uword
Definition: types.h:112
uword * session_by_sa_index
Definition: ipsec.h:146
static void crypto_scan_devs(u32 n_mains)
Definition: ipsec.c:683
const char * name
Definition: ipsec.h:99
static void crypto_op_init(struct rte_mempool *mempool, void *_arg, void *_obj, unsigned i)
Definition: ipsec.c:826
struct rte_crypto_op ** ops
Definition: ipsec.h:70
clib_error_t * create_sym_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.c:321
crypto_dev_t * dev
Definition: ipsec.h:157
enum rte_crypto_sym_xform_type type
Definition: ipsec.h:80
#define DPDK_CRYPTO_N_QUEUE_DESC
Definition: ipsec.c:642
crypto_data_t * data
Definition: ipsec.h:161
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
ipsec_crypto_alg_t crypto_alg
Definition: ipsec_sa.h:150
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static u32 vlib_num_workers()
Definition: threads.h:366
static uword dpdk_ipsec_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: ipsec.c:1017
u16 auth_resource_idx[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:72
u8 auth_support[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:95
#define vec_validate_init_empty_aligned(V, I, INIT, A)
Make sure vector is long enough for given index and initialize empty space (no header, alignment alignment)
Definition: vec.h:498
#define vec_foreach(var, vec)
Vector iterator.
static_always_inline struct rte_cryptodev_sym_session * get_session_by_drv_and_sa_idx(crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:220
u8 key_len
Definition: ipsec.h:82
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:486
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
ipsec_key_t integ_key
Definition: ipsec_sa.h:154
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
static void crypto_set_auth_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:297
static clib_error_t * dpdk_ipsec_check_support(ipsec_sa_t *sa)
Definition: ipsec.c:559
u8 cipher_support[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:94
static void clear_and_free_obj(void *obj)
Definition: ipsec.c:408
static crypto_alg_t * auth_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
Definition: ipsec.c:221