FD.io VPP  v19.04.2-12-g66b1689
Vector Packet Processing
ipsec.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Intel and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vnet/ip/ip.h>
17 #include <vnet/api_errno.h>
18 #include <vnet/ipsec/ipsec.h>
19 #include <vlib/node_funcs.h>
20 
21 #include <dpdk/device/dpdk.h>
22 #include <dpdk/buffer.h>
23 #include <dpdk/ipsec/ipsec.h>
24 
26 
27 #define EMPTY_STRUCT {0}
28 #define NUM_CRYPTO_MBUFS 16384
29 
30 static void
31 algos_init (u32 n_mains)
32 {
34  crypto_alg_t *a;
35 
37 
38  {
39 #define _(v,f,str) \
40  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].name = str; \
41  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].disabled = n_mains;
43 #undef _
44  }
45 
46  /* Minimum boundary for ciphers is 4B, required by ESP */
47  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_NONE];
48  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
49  a->alg = RTE_CRYPTO_CIPHER_NULL;
50  a->boundary = 4; /* 1 */
51  a->key_len = 0;
52  a->iv_len = 0;
53 
54  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
55  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
56  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
57  a->boundary = 16;
58  a->key_len = 16;
59  a->iv_len = 16;
60 
61  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
62  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
63  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
64  a->boundary = 16;
65  a->key_len = 24;
66  a->iv_len = 16;
67 
68  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
69  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
70  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
71  a->boundary = 16;
72  a->key_len = 32;
73  a->iv_len = 16;
74 
75  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_128];
76  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
77  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
78  a->boundary = 4; /* 1 */
79  a->key_len = 16;
80  a->iv_len = 8;
81 
82  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_192];
83  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
84  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
85  a->boundary = 4; /* 1 */
86  a->key_len = 24;
87  a->iv_len = 8;
88 
89  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_256];
90  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
91  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
92  a->boundary = 4; /* 1 */
93  a->key_len = 32;
94  a->iv_len = 8;
95 
96 #define AES_GCM_TYPE RTE_CRYPTO_SYM_XFORM_AEAD
97 #define AES_GCM_ALG RTE_CRYPTO_AEAD_AES_GCM
98 
99  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
100  a->type = AES_GCM_TYPE;
101  a->alg = AES_GCM_ALG;
102  a->boundary = 4; /* 1 */
103  a->key_len = 16;
104  a->iv_len = 8;
105  a->trunc_size = 16;
106 
107  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_192];
108  a->type = AES_GCM_TYPE;
109  a->alg = AES_GCM_ALG;
110  a->boundary = 4; /* 1 */
111  a->key_len = 24;
112  a->iv_len = 8;
113  a->trunc_size = 16;
114 
115  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_256];
116  a->type = AES_GCM_TYPE;
117  a->alg = AES_GCM_ALG;
118  a->boundary = 4; /* 1 */
119  a->key_len = 32;
120  a->iv_len = 8;
121  a->trunc_size = 16;
122 
124 
125  {
126 #define _(v,f,str) \
127  dcm->auth_algs[IPSEC_INTEG_ALG_##f].name = str; \
128  dcm->auth_algs[IPSEC_INTEG_ALG_##f].disabled = n_mains;
130 #undef _
131  }
132 
133  a = &dcm->auth_algs[IPSEC_INTEG_ALG_NONE];
134  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
135  a->alg = RTE_CRYPTO_AUTH_NULL;
136  a->key_len = 0;
137  a->trunc_size = 0;
138 
139  a = &dcm->auth_algs[IPSEC_INTEG_ALG_MD5_96];
140  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
141  a->alg = RTE_CRYPTO_AUTH_MD5_HMAC;
142  a->key_len = 16;
143  a->trunc_size = 12;
144 
145  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA1_96];
146  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
147  a->alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
148  a->key_len = 20;
149  a->trunc_size = 12;
150 
151  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_96];
152  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
153  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
154  a->key_len = 32;
155  a->trunc_size = 12;
156 
157  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_128];
158  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
159  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
160  a->key_len = 32;
161  a->trunc_size = 16;
162 
163  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_384_192];
164  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
165  a->alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
166  a->key_len = 48;
167  a->trunc_size = 24;
168 
169  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_512_256];
170  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
171  a->alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
172  a->key_len = 64;
173  a->trunc_size = 32;
174 }
175 
176 static u8
178 {
180 
181  return (alg - dcm->cipher_algs);
182 }
183 
184 static u8
186 {
188 
189  return (alg - dcm->auth_algs);
190 }
191 
192 static crypto_alg_t *
193 cipher_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 key_len)
194 {
196  crypto_alg_t *alg;
197 
198  if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
199  return NULL;
200 
201  /* *INDENT-OFF* */
202  vec_foreach (alg, dcm->cipher_algs)
203  {
204  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
205  (alg->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
206  (cap->sym.cipher.algo == alg->alg) &&
207  (alg->key_len == key_len))
208  return alg;
209  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
210  (alg->type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
211  (cap->sym.aead.algo == alg->alg) &&
212  (alg->key_len == key_len))
213  return alg;
214  }
215  /* *INDENT-ON* */
216 
217  return NULL;
218 }
219 
220 static crypto_alg_t *
221 auth_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
222 {
224  crypto_alg_t *alg;
225 
226  if ((cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) ||
227  (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH))
228  return NULL;
229 
230  /* *INDENT-OFF* */
231  vec_foreach (alg, dcm->auth_algs)
232  {
233  if ((cap->sym.auth.algo == alg->alg) &&
234  (alg->trunc_size == trunc_size))
235  return alg;
236  }
237  /* *INDENT-ON* */
238 
239  return NULL;
240 }
241 
242 static void
243 crypto_set_aead_xform (struct rte_crypto_sym_xform *xform,
244  ipsec_sa_t * sa, u8 is_outbound)
245 {
247  crypto_alg_t *c;
248 
249  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
250 
251  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_AEAD);
252 
253  xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
254  xform->aead.algo = c->alg;
255  xform->aead.key.data = sa->crypto_key.data;
256  xform->aead.key.length = c->key_len;
257  xform->aead.iv.offset =
258  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
259  xform->aead.iv.length = 12;
260  xform->aead.digest_length = c->trunc_size;
261  xform->aead.aad_length = ipsec_sa_is_set_USE_ESN (sa) ? 12 : 8;
262  xform->next = NULL;
263 
264  if (is_outbound)
265  xform->aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
266  else
267  xform->aead.op = RTE_CRYPTO_AEAD_OP_DECRYPT;
268 }
269 
270 static void
271 crypto_set_cipher_xform (struct rte_crypto_sym_xform *xform,
272  ipsec_sa_t * sa, u8 is_outbound)
273 {
275  crypto_alg_t *c;
276 
277  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
278 
279  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_CIPHER);
280 
281  xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
282  xform->cipher.algo = c->alg;
283  xform->cipher.key.data = sa->crypto_key.data;
284  xform->cipher.key.length = c->key_len;
285  xform->cipher.iv.offset =
286  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
287  xform->cipher.iv.length = c->iv_len;
288  xform->next = NULL;
289 
290  if (is_outbound)
291  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
292  else
293  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
294 }
295 
296 static void
297 crypto_set_auth_xform (struct rte_crypto_sym_xform *xform,
298  ipsec_sa_t * sa, u8 is_outbound)
299 {
301  crypto_alg_t *a;
302 
303  a = vec_elt_at_index (dcm->auth_algs, sa->integ_alg);
304 
305  ASSERT (a->type == RTE_CRYPTO_SYM_XFORM_AUTH);
306 
307  xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
308  xform->auth.algo = a->alg;
309  xform->auth.key.data = sa->integ_key.data;
310  xform->auth.key.length = a->key_len;
311  xform->auth.digest_length = a->trunc_size;
312  xform->next = NULL;
313 
314  if (is_outbound)
315  xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
316  else
317  xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
318 }
319 
320 clib_error_t *
321 create_sym_session (struct rte_cryptodev_sym_session **session,
322  u32 sa_idx,
323  crypto_resource_t * res,
325 {
327  ipsec_main_t *im = &ipsec_main;
329  ipsec_sa_t *sa;
330  struct rte_crypto_sym_xform cipher_xform = { 0 };
331  struct rte_crypto_sym_xform auth_xform = { 0 };
332  struct rte_crypto_sym_xform *xfs;
333  struct rte_cryptodev_sym_session **s;
334  clib_error_t *error = 0;
335 
336 
337  sa = pool_elt_at_index (im->sad, sa_idx);
338 
339  if ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
340  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
341  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256))
342  {
343  crypto_set_aead_xform (&cipher_xform, sa, is_outbound);
344  xfs = &cipher_xform;
345  }
346  else
347  {
348  crypto_set_cipher_xform (&cipher_xform, sa, is_outbound);
349  crypto_set_auth_xform (&auth_xform, sa, is_outbound);
350 
351  if (is_outbound)
352  {
353  cipher_xform.next = &auth_xform;
354  xfs = &cipher_xform;
355  }
356  else
357  {
358  auth_xform.next = &cipher_xform;
359  xfs = &auth_xform;
360  }
361  }
362 
363  data = vec_elt_at_index (dcm->data, res->numa);
365 
366  /*
367  * DPDK_VER >= 1708:
368  * Multiple worker/threads share the session for an SA
369  * Single session per SA, initialized for each device driver
370  */
371  s = (void *) hash_get (data->session_by_sa_index, sa_idx);
372 
373  if (!s)
374  {
375  session[0] = rte_cryptodev_sym_session_create (data->session_h);
376  if (!session[0])
377  {
378  data->session_h_failed += 1;
379  error = clib_error_return (0, "failed to create session header");
380  goto done;
381  }
382  hash_set (data->session_by_sa_index, sa_idx, session[0]);
383  }
384  else
385  session[0] = s[0];
386 
387  struct rte_mempool **mp;
388  mp = vec_elt_at_index (data->session_drv, res->drv_id);
389  ASSERT (mp[0] != NULL);
390 
391  i32 ret =
392  rte_cryptodev_sym_session_init (res->dev_id, session[0], xfs, mp[0]);
393  if (ret)
394  {
395  data->session_drv_failed[res->drv_id] += 1;
396  error = clib_error_return (0, "failed to init session for drv %u",
397  res->drv_id);
398  goto done;
399  }
400 
401  add_session_by_drv_and_sa_idx (session[0], data, res->drv_id, sa_idx);
402 
403 done:
405  return error;
406 }
407 
408 static void __attribute__ ((unused)) clear_and_free_obj (void *obj)
409 {
410  struct rte_mempool *mp = rte_mempool_from_obj (obj);
411 
412  clib_memset (obj, 0, mp->elt_size);
413 
414  rte_mempool_put (mp, obj);
415 }
416 
417 /* This is from rte_cryptodev_pmd.h */
418 static inline void *
419 get_session_private_data (const struct rte_cryptodev_sym_session *sess,
420  uint8_t driver_id)
421 {
422 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
423  return sess->sess_private_data[driver_id];
424 #else
425  if (unlikely (sess->nb_drivers <= driver_id))
426  return 0;
427 
428  return sess->sess_data[driver_id].data;
429 #endif
430 }
431 
432 /* This is from rte_cryptodev_pmd.h */
433 static inline void
434 set_session_private_data (struct rte_cryptodev_sym_session *sess,
435  uint8_t driver_id, void *private_data)
436 {
437 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
438  sess->sess_private_data[driver_id] = private_data;
439 #else
440  if (unlikely (sess->nb_drivers <= driver_id))
441  return;
442  sess->sess_data[driver_id].data = private_data;
443 #endif
444 }
445 
446 static clib_error_t *
448 {
451  void *drv_session;
452  u32 drv_id;
453  i32 ret;
454 
455  /* *INDENT-OFF* */
456  vec_foreach (s, v)
457  {
458  /* ordered vector by timestamp */
459  if (!(s->ts + dcm->session_timeout < ts))
460  break;
461 
462  vec_foreach_index (drv_id, dcm->drv)
463  {
464  drv_session = get_session_private_data (s->session, drv_id);
465  if (!drv_session)
466  continue;
467 
468  /*
469  * Custom clear to avoid finding a dev_id for drv_id:
470  * ret = rte_cryptodev_sym_session_clear (dev_id, drv_session);
471  * ASSERT (!ret);
472  */
473  clear_and_free_obj (drv_session);
474 
475  set_session_private_data (s->session, drv_id, NULL);
476  }
477 
478  if (rte_mempool_from_obj(s->session))
479  {
480  ret = rte_cryptodev_sym_session_free (s->session);
481  ASSERT (!ret);
482  }
483  }
484  /* *INDENT-ON* */
485 
486  if (s < vec_end (v))
487  vec_delete (v, s - v, 0);
488  else
489  vec_reset_length (v);
490 
491  return 0;
492 }
493 
494 static clib_error_t *
496 {
497  ipsec_main_t *im = &ipsec_main;
500  struct rte_cryptodev_sym_session *s;
501  uword *val;
502  u32 drv_id;
503 
504  if (is_add)
505  {
506 #if 1
507  ipsec_sa_t *sa = pool_elt_at_index (im->sad, sa_index);
508  u32 seed;
509  switch (sa->crypto_alg)
510  {
511  case IPSEC_CRYPTO_ALG_AES_GCM_128:
512  case IPSEC_CRYPTO_ALG_AES_GCM_192:
513  case IPSEC_CRYPTO_ALG_AES_GCM_256:
514  clib_memcpy (&sa->salt,
515  &sa->crypto_key.data[sa->crypto_key.len - 4], 4);
516  break;
517  default:
518  seed = (u32) clib_cpu_time_now ();
519  sa->salt = random_u32 (&seed);
520  }
521 #endif
522  return 0;
523  }
524 
525  /* *INDENT-OFF* */
526  vec_foreach (data, dcm->data)
527  {
529  val = hash_get (data->session_by_sa_index, sa_index);
530  if (val)
531  {
532  s = (struct rte_cryptodev_sym_session *) val[0];
533  vec_foreach_index (drv_id, dcm->drv)
534  {
535  val = (uword*) get_session_by_drv_and_sa_idx (data, drv_id, sa_index);
536  if (val)
537  add_session_by_drv_and_sa_idx(NULL, data, drv_id, sa_index);
538  }
539 
540  hash_unset (data->session_by_sa_index, sa_index);
541 
542  u64 ts = unix_time_now_nsec ();
544 
546  sd.ts = ts;
547  sd.session = s;
548 
549  vec_add1 (data->session_disposal, sd);
550  }
552  }
553  /* *INDENT-ON* */
554 
555  return 0;
556 }
557 
558 static clib_error_t *
560 {
562 
563  if (sa->integ_alg == IPSEC_INTEG_ALG_NONE)
564  switch (sa->crypto_alg)
565  {
566  case IPSEC_CRYPTO_ALG_NONE:
567  case IPSEC_CRYPTO_ALG_AES_GCM_128:
568  case IPSEC_CRYPTO_ALG_AES_GCM_192:
569  case IPSEC_CRYPTO_ALG_AES_GCM_256:
570  break;
571  default:
572  return clib_error_return (0, "unsupported integ-alg %U crypto-alg %U",
575  }
576 
577  /* XXX do we need the NONE check? */
578  if (sa->crypto_alg != IPSEC_CRYPTO_ALG_NONE &&
579  dcm->cipher_algs[sa->crypto_alg].disabled)
580  return clib_error_return (0, "disabled crypto-alg %U",
582 
583  /* XXX do we need the NONE check? */
584  if (sa->integ_alg != IPSEC_INTEG_ALG_NONE &&
585  dcm->auth_algs[sa->integ_alg].disabled)
586  return clib_error_return (0, "disabled integ-alg %U",
588  return NULL;
589 }
590 
591 static void
593  const struct rte_cryptodev_capabilities *cap,
594  u32 n_mains)
595 {
597  crypto_alg_t *alg;
598  u8 len, inc;
599 
600  for (; cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; cap++)
601  {
602  /* A single capability maps to multiple cipher/auth algorithms */
603  switch (cap->sym.xform_type)
604  {
605  case RTE_CRYPTO_SYM_XFORM_AEAD:
606  case RTE_CRYPTO_SYM_XFORM_CIPHER:
607  inc = cap->sym.cipher.key_size.increment;
608  inc = inc ? inc : 1;
609  for (len = cap->sym.cipher.key_size.min;
610  len <= cap->sym.cipher.key_size.max; len += inc)
611  {
612  alg = cipher_cap_to_alg (cap, len);
613  if (!alg)
614  continue;
615  dev->cipher_support[cipher_alg_index (alg)] = 1;
616  alg->resources += vec_len (dev->free_resources);
617  /* At least enough resources to support one algo */
618  dcm->enabled |= (alg->resources >= n_mains);
619  }
620  break;
621  case RTE_CRYPTO_SYM_XFORM_AUTH:
622  inc = cap->sym.auth.digest_size.increment;
623  inc = inc ? inc : 1;
624  for (len = cap->sym.auth.digest_size.min;
625  len <= cap->sym.auth.digest_size.max; len += inc)
626  {
627  alg = auth_cap_to_alg (cap, len);
628  if (!alg)
629  continue;
630  dev->auth_support[auth_alg_index (alg)] = 1;
631  alg->resources += vec_len (dev->free_resources);
632  /* At least enough resources to support one algo */
633  dcm->enabled |= (alg->resources >= n_mains);
634  }
635  break;
636  default:
637  ;
638  }
639  }
640 }
641 
642 static clib_error_t *
643 crypto_dev_conf (u8 dev, u16 n_qp, u8 numa)
644 {
645  struct rte_cryptodev_config dev_conf = { 0 };
646  struct rte_cryptodev_qp_conf qp_conf = { 0 };
647  i32 ret;
648  u16 qp;
649  char *error_str;
650 
651  dev_conf.socket_id = numa;
652  dev_conf.nb_queue_pairs = n_qp;
653 
654  error_str = "failed to configure crypto device %u";
655  ret = rte_cryptodev_configure (dev, &dev_conf);
656  if (ret < 0)
657  return clib_error_return (0, error_str, dev);
658 
659  error_str = "failed to setup crypto device %u queue pair %u";
660  qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
661  for (qp = 0; qp < n_qp; qp++)
662  {
663 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
664  ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa, NULL);
665 #else
666  ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa);
667 #endif
668  if (ret < 0)
669  return clib_error_return (0, error_str, dev, qp);
670  }
671 
672  error_str = "failed to start crypto device %u";
673  if (rte_cryptodev_start (dev))
674  return clib_error_return (0, error_str, dev);
675 
676  return 0;
677 }
678 
679 static void
681 {
683  struct rte_cryptodev *cryptodev;
684  struct rte_cryptodev_info info = { 0 };
685  crypto_dev_t *dev;
686  crypto_resource_t *res;
687  clib_error_t *error;
688  u32 i;
689  u16 max_res_idx, res_idx, j;
690  u8 drv_id;
691 
692  vec_validate_init_empty (dcm->dev, rte_cryptodev_count () - 1,
694 
695  for (i = 0; i < rte_cryptodev_count (); i++)
696  {
697  dev = vec_elt_at_index (dcm->dev, i);
698 
699  cryptodev = &rte_cryptodevs[i];
700  rte_cryptodev_info_get (i, &info);
701 
702  dev->id = i;
703  dev->name = cryptodev->data->name;
704  dev->numa = rte_cryptodev_socket_id (i);
705  dev->features = info.feature_flags;
706  dev->max_qp = info.max_nb_queue_pairs;
707  drv_id = info.driver_id;
708  if (drv_id >= vec_len (dcm->drv))
709  vec_validate_init_empty (dcm->drv, drv_id,
710  (crypto_drv_t) EMPTY_STRUCT);
711  vec_elt_at_index (dcm->drv, drv_id)->name = info.driver_name;
712  dev->drv_id = drv_id;
713  vec_add1 (vec_elt_at_index (dcm->drv, drv_id)->devs, i);
714 
715  if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
716  continue;
717 
718  if ((error = crypto_dev_conf (i, dev->max_qp, dev->numa)))
719  {
720  clib_error_report (error);
721  continue;
722  }
723 
724  max_res_idx = dev->max_qp - 1;
725 
726  vec_validate (dev->free_resources, max_res_idx);
727 
728  res_idx = vec_len (dcm->resource);
729  vec_validate_init_empty_aligned (dcm->resource, res_idx + max_res_idx,
730  (crypto_resource_t) EMPTY_STRUCT,
732 
733  for (j = 0; j <= max_res_idx; j++)
734  {
735  vec_elt (dev->free_resources, max_res_idx - j) = res_idx + j;
736  res = &dcm->resource[res_idx + j];
737  res->dev_id = i;
738  res->drv_id = drv_id;
739  res->qp_id = j;
740  res->numa = dev->numa;
741  res->thread_idx = (u16) ~ 0;
742  }
743 
744  crypto_parse_capabilities (dev, info.capabilities, n_mains);
745  }
746 }
747 
748 void
750 {
752  crypto_resource_t *res;
754  crypto_dev_t *dev;
755  u32 thread_idx, skip_master;
756  u16 res_idx, *idx;
757  u8 used;
758  u16 i;
759 
760  skip_master = vlib_num_workers () > 0;
761 
762  /* *INDENT-OFF* */
763  vec_foreach (dev, dcm->dev)
764  {
765  vec_foreach_index (thread_idx, dcm->workers_main)
766  {
767  if (vec_len (dev->free_resources) == 0)
768  break;
769 
770  if (thread_idx < skip_master)
771  continue;
772 
773  /* Check thread is not already using the device */
774  vec_foreach (idx, dev->used_resources)
775  if (dcm->resource[idx[0]].thread_idx == thread_idx)
776  continue;
777 
778  cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
779 
780  used = 0;
781  res_idx = vec_pop (dev->free_resources);
782 
783  /* Set device only for supported algos */
784  for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
785  if (dev->cipher_support[i] &&
786  cwm->cipher_resource_idx[i] == (u16) ~0)
787  {
788  dcm->cipher_algs[i].disabled--;
789  cwm->cipher_resource_idx[i] = res_idx;
790  used = 1;
791  }
792 
793  for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
794  if (dev->auth_support[i] &&
795  cwm->auth_resource_idx[i] == (u16) ~0)
796  {
797  dcm->auth_algs[i].disabled--;
798  cwm->auth_resource_idx[i] = res_idx;
799  used = 1;
800  }
801 
802  if (!used)
803  {
804  vec_add1 (dev->free_resources, res_idx);
805  continue;
806  }
807 
808  vec_add1 (dev->used_resources, res_idx);
809 
810  res = vec_elt_at_index (dcm->resource, res_idx);
811 
812  ASSERT (res->thread_idx == (u16) ~0);
813  res->thread_idx = thread_idx;
814 
815  /* Add device to vector of polling resources */
816  vec_add1 (cwm->resource_idx, res_idx);
817  }
818  }
819  /* *INDENT-ON* */
820 }
821 
822 static void
823 crypto_op_init (struct rte_mempool *mempool,
824  void *_arg __attribute__ ((unused)),
825  void *_obj, unsigned i __attribute__ ((unused)))
826 {
827  struct rte_crypto_op *op = _obj;
828 
829  op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
830  op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
831  op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
832  op->phys_addr = rte_mempool_virt2iova (_obj);
833  op->mempool = mempool;
834 }
835 
836 static clib_error_t *
838 {
842  u8 *pool_name;
843  u32 pool_priv_size = sizeof (struct rte_crypto_op_pool_private);
844  struct rte_crypto_op_pool_private *priv;
845  struct rte_mempool *mp;
846 
847  data = vec_elt_at_index (dcm->data, numa);
848 
849  /* Already allocated */
850  if (data->crypto_op)
851  return NULL;
852 
853  pool_name = format (0, "crypto_pool_numa%u%c", numa, 0);
854 
855  if (conf->num_crypto_mbufs == 0)
857 
858  mp = rte_mempool_create ((char *) pool_name, conf->num_crypto_mbufs,
859  crypto_op_len (), 512, pool_priv_size, NULL, NULL,
860  crypto_op_init, NULL, numa, 0);
861 
862  vec_free (pool_name);
863 
864  if (!mp)
865  return clib_error_return (0, "failed to create crypto op mempool");
866 
867  /* Initialize mempool private data */
868  priv = rte_mempool_get_priv (mp);
869  priv->priv_size = pool_priv_size;
870  priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
871 
872  data->crypto_op = mp;
873 
874  return NULL;
875 }
876 
877 static clib_error_t *
879 {
882  u8 *pool_name;
883  struct rte_mempool *mp;
884  u32 elt_size;
885 
886  data = vec_elt_at_index (dcm->data, numa);
887 
888  if (data->session_h)
889  return NULL;
890 
891  pool_name = format (0, "session_h_pool_numa%u%c", numa, 0);
892 
893 
894  elt_size = rte_cryptodev_sym_get_header_session_size ();
895 
896 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
897  mp = rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
898  elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
899 #else
900  /* XXX Experimental tag in DPDK 19.02 */
901  mp = rte_cryptodev_sym_session_pool_create ((char *) pool_name,
903  elt_size, 512, 0, numa);
904 #endif
905  vec_free (pool_name);
906 
907  if (!mp)
908  return clib_error_return (0, "failed to create crypto session mempool");
909 
910  data->session_h = mp;
911 
912  return NULL;
913 }
914 
915 static clib_error_t *
917 {
920  u8 *pool_name;
921  struct rte_mempool *mp;
922  u32 elt_size;
923  u8 numa = dev->numa;
924 
925  data = vec_elt_at_index (dcm->data, numa);
926 
927  vec_validate (data->session_drv, dev->drv_id);
928  vec_validate (data->session_drv_failed, dev->drv_id);
931 
932  if (data->session_drv[dev->drv_id])
933  return NULL;
934 
935  pool_name = format (0, "session_drv%u_pool_numa%u%c", dev->drv_id, numa, 0);
936 
937  elt_size = rte_cryptodev_sym_get_private_session_size (dev->id);
938  mp =
939  rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
940  elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
941 
942  vec_free (pool_name);
943 
944  if (!mp)
945  return clib_error_return (0, "failed to create session drv mempool");
946 
947  data->session_drv[dev->drv_id] = mp;
948  clib_spinlock_init (&data->lockp);
949 
950  return NULL;
951 }
952 
953 static clib_error_t *
955 {
957  clib_error_t *error = NULL;
958  crypto_dev_t *dev;
959 
960  /* *INDENT-OFF* */
961  vec_foreach (dev, dcm->dev)
962  {
964 
965  error = crypto_create_crypto_op_pool (vm, dev->numa);
966  if (error)
967  return error;
968 
969  error = crypto_create_session_h_pool (vm, dev->numa);
970  if (error)
971  return error;
972 
973  error = crypto_create_session_drv_pool (vm, dev);
974  if (error)
975  return error;
976  }
977  /* *INDENT-ON* */
978 
979  return NULL;
980 }
981 
982 static void
984 {
987  u8 i;
988 
989  dcm->enabled = 0;
990 
991  /* *INDENT-OFF* */
992  vec_foreach (data, dcm->data)
993  {
994  rte_mempool_free (data->crypto_op);
995  rte_mempool_free (data->session_h);
996 
997  vec_foreach_index (i, data->session_drv)
998  rte_mempool_free (data->session_drv[i]);
999 
1000  vec_free (data->session_drv);
1001  clib_spinlock_free (&data->lockp);
1002  }
1003  /* *INDENT-ON* */
1004 
1005  vec_free (dcm->data);
1006  vec_free (dcm->workers_main);
1007  vec_free (dcm->dev);
1008  vec_free (dcm->resource);
1009  vec_free (dcm->cipher_algs);
1010  vec_free (dcm->auth_algs);
1011 }
1012 
1013 static uword
1015  vlib_frame_t * f)
1016 {
1017  ipsec_main_t *im = &ipsec_main;
1020  crypto_worker_main_t *cwm;
1021  clib_error_t *error = NULL;
1022  u32 i, skip_master, n_mains;
1023 
1024  n_mains = tm->n_vlib_mains;
1025  skip_master = vlib_num_workers () > 0;
1026 
1027  algos_init (n_mains - skip_master);
1028 
1029  crypto_scan_devs (n_mains - skip_master);
1030 
1031  if (!(dcm->enabled))
1032  {
1033  clib_warning ("not enough DPDK crypto resources, default to OpenSSL");
1034  crypto_disable ();
1035  return 0;
1036  }
1037 
1038  dcm->session_timeout = 10e9;
1039 
1040  vec_validate_init_empty_aligned (dcm->workers_main, n_mains - 1,
1043 
1044  /* *INDENT-OFF* */
1045  vec_foreach (cwm, dcm->workers_main)
1046  {
1049  clib_memset (cwm->cipher_resource_idx, ~0,
1050  IPSEC_CRYPTO_N_ALG * sizeof(*cwm->cipher_resource_idx));
1051  clib_memset (cwm->auth_resource_idx, ~0,
1052  IPSEC_INTEG_N_ALG * sizeof(*cwm->auth_resource_idx));
1053  }
1054  /* *INDENT-ON* */
1055 
1057 
1058  error = crypto_create_pools (vm);
1059  if (error)
1060  {
1061  clib_error_report (error);
1062  crypto_disable ();
1063  return 0;
1064  }
1065 
1066 
1067  u32 idx = ipsec_register_esp_backend (vm, im, "dpdk backend",
1068  "dpdk-esp4-encrypt",
1069  "dpdk-esp4-encrypt-tun",
1070  "dpdk-esp4-decrypt",
1071  "dpdk-esp6-encrypt",
1072  "dpdk-esp6-encrypt-tun",
1073  "dpdk-esp6-decrypt",
1076  int rv = ipsec_select_esp_backend (im, idx);
1077  ASSERT (rv == 0);
1078 
1079  vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "dpdk-crypto-input");
1080  ASSERT (node);
1081  for (i = skip_master; i < n_mains; i++)
1082  vlib_node_set_state (vlib_mains[i], node->index, VLIB_NODE_STATE_POLLING);
1083  return 0;
1084 }
1085 
1086 /* *INDENT-OFF* */
1087 VLIB_REGISTER_NODE (dpdk_ipsec_process_node,static) = {
1088  .function = dpdk_ipsec_process,
1089  .type = VLIB_NODE_TYPE_PROCESS,
1090  .name = "dpdk-ipsec-process",
1091  .process_log2_n_stack_bytes = 17,
1092 };
1093 /* *INDENT-ON* */
1094 
1095 /*
1096  * fd.io coding-style-patch-verification: ON
1097  *
1098  * Local Variables:
1099  * eval: (c-set-style "gnu")
1100  * End:
1101  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
u32 alg
Definition: ipsec.h:86
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:109
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:255
#define hash_unset(h, key)
Definition: hash.h:261
a
Definition: bitmap.h:538
static u8 cipher_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:177
unsigned long u64
Definition: types.h:89
#define NULL
Definition: clib.h:58
u32 index
Definition: node.h:279
u64 session_h_failed
Definition: ipsec.h:152
#define foreach_ipsec_crypto_alg
Definition: ipsec_sa.h:24
ipsec_key_t crypto_key
Definition: ipsec_sa.h:151
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
ipsec_integ_alg_t integ_alg
Definition: ipsec_sa.h:153
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static clib_error_t * crypto_create_session_drv_pool(vlib_main_t *vm, crypto_dev_t *dev)
Definition: ipsec.c:916
static u64 clib_cpu_time_now(void)
Definition: time.h:75
#define EMPTY_STRUCT
Definition: ipsec.c:27
static_always_inline u32 crypto_op_get_priv_offset(void)
Definition: ipsec.h:195
int i
#define foreach_ipsec_integ_alg
Definition: ipsec_sa.h:51
#define DPDK_CRYPTO_NB_SESS_OBJS
Definition: ipsec.h:34
clib_memset(h->entries, 0, sizeof(h->entries[0])*entries)
struct rte_cryptodev_sym_session * session
Definition: ipsec.h:134
u8 disabled
Definition: ipsec.h:91
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
u8 data[128]
Definition: ipsec.api:248
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
static void crypto_set_cipher_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:271
vlib_main_t ** vlib_mains
Definition: buffer.c:321
static clib_error_t * crypto_create_session_h_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:878
unsigned char u8
Definition: types.h:56
#define vec_pop(V)
Returns last element of a vector and decrements its length.
Definition: vec.h:615
static crypto_alg_t * cipher_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 key_len)
Definition: ipsec.c:193
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:64
static clib_error_t * add_del_sa_session(u32 sa_index, u8 is_add)
Definition: ipsec.c:495
#define clib_memcpy(d, s, n)
Definition: string.h:180
u16 * resource_idx
Definition: ipsec.h:74
#define AES_GCM_TYPE
#define DPDK_CRYPTO_N_QUEUE_DESC
Definition: ipsec.h:33
u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:76
static void algos_init(u32 n_mains)
Definition: ipsec.c:31
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:25
dpdk_config_main_t dpdk_config_main
Definition: init.c:45
ipsec_main_t ipsec_main
Definition: ipsec.c:28
u8 resources
Definition: ipsec.h:92
int ipsec_select_esp_backend(ipsec_main_t *im, u32 backend_idx)
Definition: ipsec.c:218
#define NUM_CRYPTO_MBUFS
Definition: ipsec.c:28
crypto_drv_t * drv
Definition: ipsec.h:168
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 drv_id
Definition: ipsec.h:101
#define clib_error_return(e, args...)
Definition: error.h:99
static void crypto_parse_capabilities(crypto_dev_t *dev, const struct rte_cryptodev_capabilities *cap, u32 n_mains)
Definition: ipsec.c:592
u16 * free_resources
Definition: ipsec.h:97
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:77
unsigned int u32
Definition: types.h:88
#define vec_end(v)
End (last data address) of vector.
crypto_alg_t * auth_algs
Definition: ipsec.h:166
#define VLIB_FRAME_SIZE
Definition: node.h:376
static clib_error_t * crypto_dev_conf(u8 dev, u16 n_qp, u8 numa)
Definition: ipsec.c:643
u8 trunc_size
Definition: ipsec.h:89
static_always_inline void add_session_by_drv_and_sa_idx(struct rte_cryptodev_sym_session *session, crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:214
static void crypto_disable(void)
Definition: ipsec.c:983
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
clib_spinlock_t lockp
Definition: ipsec.h:155
static void * get_session_private_data(const struct rte_cryptodev_sym_session *sess, uint8_t driver_id)
Definition: ipsec.c:419
u16 * used_resources
Definition: ipsec.h:98
#define hash_get(h, key)
Definition: hash.h:249
u32 ipsec_register_esp_backend(vlib_main_t *vm, ipsec_main_t *im, const char *name, const char *esp4_encrypt_node_name, const char *esp4_encrypt_node_tun_name, const char *esp4_decrypt_node_name, const char *esp6_encrypt_node_name, const char *esp6_encrypt_node_tun_name, const char *esp6_decrypt_node_name, check_support_cb_t esp_check_support_cb, add_del_sa_sess_cb_t esp_add_del_sa_sess_cb)
Definition: ipsec.c:142
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
crypto_alg_t * cipher_algs
Definition: ipsec.h:165
u32 salt
Definition: ipsec_sa.h:163
unsigned short u16
Definition: types.h:57
u64 features
Definition: ipsec.h:106
u64 session_timeout
Definition: ipsec.h:169
static_always_inline u32 crypto_op_len(void)
Definition: ipsec.h:185
#define AES_GCM_ALG
static clib_error_t * crypto_create_crypto_op_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:837
static u8 auth_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:185
struct rte_mempool ** session_drv
Definition: ipsec.h:148
u8 len
Definition: ip_types.api:49
vlib node functions
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
crypto_session_by_drv_t * session_by_drv_id_and_sa_index
Definition: ipsec.h:154
u32 num_crypto_mbufs
Definition: dpdk.h:367
svmdb_client_t * c
vlib_main_t * vm
Definition: buffer.c:312
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
#define clib_warning(format, args...)
Definition: error.h:59
u8 is_outbound
Definition: ipsec.api:92
static u64 unix_time_now_nsec(void)
Definition: time.h:255
crypto_session_disposal_t * session_disposal
Definition: ipsec.h:149
static void crypto_set_aead_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:243
vlib_node_t * vlib_get_node_by_name(vlib_main_t *vm, u8 *name)
Definition: node.c:45
u16 id
Definition: ipsec.h:103
static void set_session_private_data(struct rte_cryptodev_sym_session *sess, uint8_t driver_id, void *private_data)
Definition: ipsec.c:434
u8 iv_len
Definition: ipsec.h:88
signed int i32
Definition: types.h:77
u8 data[IPSEC_KEY_MAX_LEN]
Definition: ipsec_sa.h:80
#define ASSERT(truth)
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:784
u8 is_add
Definition: ipsec_gre.api:36
struct rte_mempool * crypto_op
Definition: ipsec.h:146
u32 max_qp
Definition: ipsec.h:105
ipsec_sa_t * sad
Definition: ipsec.h:95
static clib_error_t * dpdk_crypto_session_disposal(crypto_session_disposal_t *v, u64 ts)
Definition: ipsec.c:447
crypto_worker_main_t * workers_main
Definition: ipsec.h:162
#define clib_error_report(e)
Definition: error.h:113
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
Definition: node_funcs.h:148
crypto_resource_t * resource
Definition: ipsec.h:164
#define vec_elt(v, i)
Get vector value at index i.
static clib_error_t * crypto_create_pools(vlib_main_t *vm)
Definition: ipsec.c:954
struct rte_mempool * session_h
Definition: ipsec.h:147
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void crypto_auto_placement(void)
Definition: ipsec.c:749
u64 * session_drv_failed
Definition: ipsec.h:153
u64 uword
Definition: types.h:112
uword * session_by_sa_index
Definition: ipsec.h:150
static void crypto_scan_devs(u32 n_mains)
Definition: ipsec.c:680
const char * name
Definition: ipsec.h:104
static void crypto_op_init(struct rte_mempool *mempool, void *_arg, void *_obj, unsigned i)
Definition: ipsec.c:823
struct rte_crypto_op ** ops
Definition: ipsec.h:75
clib_error_t * create_sym_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.c:321
crypto_dev_t * dev
Definition: ipsec.h:163
enum rte_crypto_sym_xform_type type
Definition: ipsec.h:85
crypto_data_t * data
Definition: ipsec.h:167
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
ipsec_crypto_alg_t crypto_alg
Definition: ipsec_sa.h:150
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static u32 vlib_num_workers()
Definition: threads.h:366
static uword dpdk_ipsec_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: ipsec.c:1014
u16 auth_resource_idx[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:77
u8 auth_support[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:100
#define vec_validate_init_empty_aligned(V, I, INIT, A)
Make sure vector is long enough for given index and initialize empty space (no header, alignment alignment)
Definition: vec.h:498
#define vec_foreach(var, vec)
Vector iterator.
static_always_inline struct rte_cryptodev_sym_session * get_session_by_drv_and_sa_idx(crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:226
u8 key_len
Definition: ipsec.h:87
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:486
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
ipsec_key_t integ_key
Definition: ipsec_sa.h:154
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
static void crypto_set_auth_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:297
static clib_error_t * dpdk_ipsec_check_support(ipsec_sa_t *sa)
Definition: ipsec.c:559
u8 cipher_support[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:99
static void clear_and_free_obj(void *obj)
Definition: ipsec.c:408
static crypto_alg_t * auth_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
Definition: ipsec.c:221