FD.io VPP  v19.08.3-2-gbabecb413
Vector Packet Processing
ipsec.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Intel and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vnet/ip/ip.h>
17 #include <vnet/api_errno.h>
18 #include <vnet/ipsec/ipsec.h>
19 #include <vlib/node_funcs.h>
20 
21 #include <dpdk/device/dpdk.h>
22 #include <dpdk/buffer.h>
23 #include <dpdk/ipsec/ipsec.h>
24 
26 
27 #define EMPTY_STRUCT {0}
28 #define NUM_CRYPTO_MBUFS 16384
29 
30 static void
31 algos_init (u32 n_mains)
32 {
34  crypto_alg_t *a;
35 
37 
38  {
39 #define _(v,f,str) \
40  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].name = str; \
41  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].disabled = n_mains;
43 #undef _
44  }
45 
46  /* Minimum boundary for ciphers is 4B, required by ESP */
47  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_NONE];
48  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
49  a->alg = RTE_CRYPTO_CIPHER_NULL;
50  a->boundary = 4; /* 1 */
51  a->key_len = 0;
52  a->iv_len = 0;
53 
54  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
55  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
56  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
57  a->boundary = 16;
58  a->key_len = 16;
59  a->iv_len = 16;
60 
61  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
62  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
63  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
64  a->boundary = 16;
65  a->key_len = 24;
66  a->iv_len = 16;
67 
68  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
69  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
70  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
71  a->boundary = 16;
72  a->key_len = 32;
73  a->iv_len = 16;
74 
75  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_128];
76  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
77  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
78  a->boundary = 4; /* 1 */
79  a->key_len = 16;
80  a->iv_len = 8;
81 
82  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_192];
83  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
84  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
85  a->boundary = 4; /* 1 */
86  a->key_len = 24;
87  a->iv_len = 8;
88 
89  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_256];
90  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
91  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
92  a->boundary = 4; /* 1 */
93  a->key_len = 32;
94  a->iv_len = 8;
95 
96 #define AES_GCM_TYPE RTE_CRYPTO_SYM_XFORM_AEAD
97 #define AES_GCM_ALG RTE_CRYPTO_AEAD_AES_GCM
98 
99  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
100  a->type = AES_GCM_TYPE;
101  a->alg = AES_GCM_ALG;
102  a->boundary = 4; /* 1 */
103  a->key_len = 16;
104  a->iv_len = 8;
105  a->trunc_size = 16;
106 
107  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_192];
108  a->type = AES_GCM_TYPE;
109  a->alg = AES_GCM_ALG;
110  a->boundary = 4; /* 1 */
111  a->key_len = 24;
112  a->iv_len = 8;
113  a->trunc_size = 16;
114 
115  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_256];
116  a->type = AES_GCM_TYPE;
117  a->alg = AES_GCM_ALG;
118  a->boundary = 4; /* 1 */
119  a->key_len = 32;
120  a->iv_len = 8;
121  a->trunc_size = 16;
122 
124 
125  {
126 #define _(v,f,str) \
127  dcm->auth_algs[IPSEC_INTEG_ALG_##f].name = str; \
128  dcm->auth_algs[IPSEC_INTEG_ALG_##f].disabled = n_mains;
130 #undef _
131  }
132 
133  a = &dcm->auth_algs[IPSEC_INTEG_ALG_NONE];
134  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
135  a->alg = RTE_CRYPTO_AUTH_NULL;
136  a->key_len = 0;
137  a->trunc_size = 0;
138 
139  a = &dcm->auth_algs[IPSEC_INTEG_ALG_MD5_96];
140  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
141  a->alg = RTE_CRYPTO_AUTH_MD5_HMAC;
142  a->key_len = 16;
143  a->trunc_size = 12;
144 
145  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA1_96];
146  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
147  a->alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
148  a->key_len = 20;
149  a->trunc_size = 12;
150 
151  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_96];
152  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
153  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
154  a->key_len = 32;
155  a->trunc_size = 12;
156 
157  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_128];
158  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
159  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
160  a->key_len = 32;
161  a->trunc_size = 16;
162 
163  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_384_192];
164  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
165  a->alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
166  a->key_len = 48;
167  a->trunc_size = 24;
168 
169  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_512_256];
170  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
171  a->alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
172  a->key_len = 64;
173  a->trunc_size = 32;
174 }
175 
176 static u8
178 {
180 
181  return (alg - dcm->cipher_algs);
182 }
183 
184 static u8
186 {
188 
189  return (alg - dcm->auth_algs);
190 }
191 
192 static crypto_alg_t *
193 cipher_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 key_len)
194 {
196  crypto_alg_t *alg;
197 
198  if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
199  return NULL;
200 
201  /* *INDENT-OFF* */
202  vec_foreach (alg, dcm->cipher_algs)
203  {
204  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
205  (alg->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
206  (cap->sym.cipher.algo == alg->alg) &&
207  (alg->key_len == key_len))
208  return alg;
209  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
210  (alg->type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
211  (cap->sym.aead.algo == alg->alg) &&
212  (alg->key_len == key_len))
213  return alg;
214  }
215  /* *INDENT-ON* */
216 
217  return NULL;
218 }
219 
220 static crypto_alg_t *
221 auth_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
222 {
224  crypto_alg_t *alg;
225 
226  if ((cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) ||
227  (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH))
228  return NULL;
229 
230  /* *INDENT-OFF* */
231  vec_foreach (alg, dcm->auth_algs)
232  {
233  if ((cap->sym.auth.algo == alg->alg) &&
234  (alg->trunc_size == trunc_size))
235  return alg;
236  }
237  /* *INDENT-ON* */
238 
239  return NULL;
240 }
241 
242 static void
243 crypto_set_aead_xform (struct rte_crypto_sym_xform *xform,
244  ipsec_sa_t * sa, u8 is_outbound)
245 {
247  crypto_alg_t *c;
248 
249  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
250 
251  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_AEAD);
252 
253  xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
254  xform->aead.algo = c->alg;
255  xform->aead.key.data = sa->crypto_key.data;
256  xform->aead.key.length = c->key_len;
257  xform->aead.iv.offset =
258  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
259  xform->aead.iv.length = 12;
260  xform->aead.digest_length = c->trunc_size;
261  xform->aead.aad_length = ipsec_sa_is_set_USE_ESN (sa) ? 12 : 8;
262  xform->next = NULL;
263 
264  if (is_outbound)
265  xform->aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
266  else
267  xform->aead.op = RTE_CRYPTO_AEAD_OP_DECRYPT;
268 }
269 
270 static void
271 crypto_set_cipher_xform (struct rte_crypto_sym_xform *xform,
272  ipsec_sa_t * sa, u8 is_outbound)
273 {
275  crypto_alg_t *c;
276 
277  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
278 
279  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_CIPHER);
280 
281  xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
282  xform->cipher.algo = c->alg;
283  xform->cipher.key.data = sa->crypto_key.data;
284  xform->cipher.key.length = c->key_len;
285  xform->cipher.iv.offset =
286  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
287  xform->cipher.iv.length = c->iv_len;
288  xform->next = NULL;
289 
290  if (is_outbound)
291  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
292  else
293  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
294 }
295 
296 static void
297 crypto_set_auth_xform (struct rte_crypto_sym_xform *xform,
298  ipsec_sa_t * sa, u8 is_outbound)
299 {
301  crypto_alg_t *a;
302 
303  a = vec_elt_at_index (dcm->auth_algs, sa->integ_alg);
304 
305  ASSERT (a->type == RTE_CRYPTO_SYM_XFORM_AUTH);
306 
307  xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
308  xform->auth.algo = a->alg;
309  xform->auth.key.data = sa->integ_key.data;
310  xform->auth.key.length = a->key_len;
311  xform->auth.digest_length = a->trunc_size;
312  xform->next = NULL;
313 
314  if (is_outbound)
315  xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
316  else
317  xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
318 }
319 
320 clib_error_t *
321 create_sym_session (struct rte_cryptodev_sym_session **session,
322  u32 sa_idx,
323  crypto_resource_t * res,
325 {
327  ipsec_main_t *im = &ipsec_main;
329  ipsec_sa_t *sa;
330  struct rte_crypto_sym_xform cipher_xform = { 0 };
331  struct rte_crypto_sym_xform auth_xform = { 0 };
332  struct rte_crypto_sym_xform *xfs;
333  struct rte_cryptodev_sym_session **s;
334  clib_error_t *error = 0;
335 
336 
337  sa = pool_elt_at_index (im->sad, sa_idx);
338 
339  if ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
340  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
341  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256))
342  {
343  crypto_set_aead_xform (&cipher_xform, sa, is_outbound);
344  xfs = &cipher_xform;
345  }
346  else
347  {
348  crypto_set_cipher_xform (&cipher_xform, sa, is_outbound);
349  crypto_set_auth_xform (&auth_xform, sa, is_outbound);
350 
351  if (is_outbound)
352  {
353  cipher_xform.next = &auth_xform;
354  xfs = &cipher_xform;
355  }
356  else
357  {
358  auth_xform.next = &cipher_xform;
359  xfs = &auth_xform;
360  }
361  }
362 
363  data = vec_elt_at_index (dcm->data, res->numa);
365 
366  /*
367  * DPDK_VER >= 1708:
368  * Multiple worker/threads share the session for an SA
369  * Single session per SA, initialized for each device driver
370  */
371  s = (void *) hash_get (data->session_by_sa_index, sa_idx);
372 
373  if (!s)
374  {
375  session[0] = rte_cryptodev_sym_session_create (data->session_h);
376  if (!session[0])
377  {
378  data->session_h_failed += 1;
379  error = clib_error_return (0, "failed to create session header");
380  goto done;
381  }
382  hash_set (data->session_by_sa_index, sa_idx, session[0]);
383  }
384  else
385  session[0] = s[0];
386 
387  struct rte_mempool **mp;
388  mp = vec_elt_at_index (data->session_drv, res->drv_id);
389  ASSERT (mp[0] != NULL);
390 
391  i32 ret =
392  rte_cryptodev_sym_session_init (res->dev_id, session[0], xfs, mp[0]);
393  if (ret)
394  {
395  data->session_drv_failed[res->drv_id] += 1;
396  error = clib_error_return (0, "failed to init session for drv %u",
397  res->drv_id);
398  goto done;
399  }
400 
401  add_session_by_drv_and_sa_idx (session[0], data, res->drv_id, sa_idx);
402 
403 done:
405  return error;
406 }
407 
408 static void __attribute__ ((unused)) clear_and_free_obj (void *obj)
409 {
410  struct rte_mempool *mp = rte_mempool_from_obj (obj);
411 
412  clib_memset (obj, 0, mp->elt_size);
413 
414  rte_mempool_put (mp, obj);
415 }
416 
417 /* This is from rte_cryptodev_pmd.h */
418 static inline void *
419 get_session_private_data (const struct rte_cryptodev_sym_session *sess,
420  uint8_t driver_id)
421 {
422 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
423  return sess->sess_private_data[driver_id];
424 #else
425  if (unlikely (sess->nb_drivers <= driver_id))
426  return 0;
427 
428  return sess->sess_data[driver_id].data;
429 #endif
430 }
431 
432 /* This is from rte_cryptodev_pmd.h */
433 static inline void
434 set_session_private_data (struct rte_cryptodev_sym_session *sess,
435  uint8_t driver_id, void *private_data)
436 {
437 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
438  sess->sess_private_data[driver_id] = private_data;
439 #else
440  if (unlikely (sess->nb_drivers <= driver_id))
441  return;
442  sess->sess_data[driver_id].data = private_data;
443 #endif
444 }
445 
446 static clib_error_t *
448 {
451  void *drv_session;
452  u32 drv_id;
453  i32 ret;
454 
455  /* *INDENT-OFF* */
456  vec_foreach (s, v)
457  {
458  /* ordered vector by timestamp */
459  if (!(s->ts + dcm->session_timeout < ts))
460  break;
461 
462  vec_foreach_index (drv_id, dcm->drv)
463  {
464  drv_session = get_session_private_data (s->session, drv_id);
465  if (!drv_session)
466  continue;
467 
468  /*
469  * Custom clear to avoid finding a dev_id for drv_id:
470  * ret = rte_cryptodev_sym_session_clear (dev_id, drv_session);
471  * ASSERT (!ret);
472  */
473  clear_and_free_obj (drv_session);
474 
475  set_session_private_data (s->session, drv_id, NULL);
476  }
477 
478  if (rte_mempool_from_obj(s->session))
479  {
480  ret = rte_cryptodev_sym_session_free (s->session);
481  ASSERT (!ret);
482  }
483  }
484  /* *INDENT-ON* */
485 
486  if (s < vec_end (v))
487  vec_delete (v, s - v, 0);
488  else
489  vec_reset_length (v);
490 
491  return 0;
492 }
493 
494 static clib_error_t *
495 add_del_sa_session (u32 sa_index, u8 is_add)
496 {
499  struct rte_cryptodev_sym_session *s;
500  uword *val;
501  u32 drv_id;
502 
503  if (is_add)
504  return 0;
505 
506  /* *INDENT-OFF* */
507  vec_foreach (data, dcm->data)
508  {
510  val = hash_get (data->session_by_sa_index, sa_index);
511  if (val)
512  {
513  s = (struct rte_cryptodev_sym_session *) val[0];
514  vec_foreach_index (drv_id, dcm->drv)
515  {
516  val = (uword*) get_session_by_drv_and_sa_idx (data, drv_id, sa_index);
517  if (val)
518  add_session_by_drv_and_sa_idx(NULL, data, drv_id, sa_index);
519  }
520 
521  hash_unset (data->session_by_sa_index, sa_index);
522 
523  u64 ts = unix_time_now_nsec ();
525 
527  sd.ts = ts;
528  sd.session = s;
529 
530  vec_add1 (data->session_disposal, sd);
531  }
533  }
534  /* *INDENT-ON* */
535 
536  return 0;
537 }
538 
539 static clib_error_t *
541 {
543 
544  if (sa->integ_alg == IPSEC_INTEG_ALG_NONE)
545  switch (sa->crypto_alg)
546  {
547  case IPSEC_CRYPTO_ALG_NONE:
548  case IPSEC_CRYPTO_ALG_AES_GCM_128:
549  case IPSEC_CRYPTO_ALG_AES_GCM_192:
550  case IPSEC_CRYPTO_ALG_AES_GCM_256:
551  break;
552  default:
553  return clib_error_return (0, "unsupported integ-alg %U crypto-alg %U",
556  }
557 
558  /* XXX do we need the NONE check? */
559  if (sa->crypto_alg != IPSEC_CRYPTO_ALG_NONE &&
560  dcm->cipher_algs[sa->crypto_alg].disabled)
561  return clib_error_return (0, "disabled crypto-alg %U",
563 
564  /* XXX do we need the NONE check? */
565  if (sa->integ_alg != IPSEC_INTEG_ALG_NONE &&
566  dcm->auth_algs[sa->integ_alg].disabled)
567  return clib_error_return (0, "disabled integ-alg %U",
569  return NULL;
570 }
571 
572 static void
574  const struct rte_cryptodev_capabilities *cap,
575  u32 n_mains)
576 {
578  crypto_alg_t *alg;
579  u8 len, inc;
580 
581  for (; cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; cap++)
582  {
583  /* A single capability maps to multiple cipher/auth algorithms */
584  switch (cap->sym.xform_type)
585  {
586  case RTE_CRYPTO_SYM_XFORM_AEAD:
587  case RTE_CRYPTO_SYM_XFORM_CIPHER:
588  inc = cap->sym.cipher.key_size.increment;
589  inc = inc ? inc : 1;
590  for (len = cap->sym.cipher.key_size.min;
591  len <= cap->sym.cipher.key_size.max; len += inc)
592  {
593  alg = cipher_cap_to_alg (cap, len);
594  if (!alg)
595  continue;
596  dev->cipher_support[cipher_alg_index (alg)] = 1;
597  alg->resources += vec_len (dev->free_resources);
598  /* At least enough resources to support one algo */
599  dcm->enabled |= (alg->resources >= n_mains);
600  }
601  break;
602  case RTE_CRYPTO_SYM_XFORM_AUTH:
603  inc = cap->sym.auth.digest_size.increment;
604  inc = inc ? inc : 1;
605  for (len = cap->sym.auth.digest_size.min;
606  len <= cap->sym.auth.digest_size.max; len += inc)
607  {
608  alg = auth_cap_to_alg (cap, len);
609  if (!alg)
610  continue;
611  dev->auth_support[auth_alg_index (alg)] = 1;
612  alg->resources += vec_len (dev->free_resources);
613  /* At least enough resources to support one algo */
614  dcm->enabled |= (alg->resources >= n_mains);
615  }
616  break;
617  default:
618  ;
619  }
620  }
621 }
622 
623 static clib_error_t *
624 crypto_dev_conf (u8 dev, u16 n_qp, u8 numa)
625 {
626  struct rte_cryptodev_config dev_conf = { 0 };
627  struct rte_cryptodev_qp_conf qp_conf = { 0 };
628  i32 ret;
629  u16 qp;
630  char *error_str;
631 
632  dev_conf.socket_id = numa;
633  dev_conf.nb_queue_pairs = n_qp;
634 
635  error_str = "failed to configure crypto device %u";
636  ret = rte_cryptodev_configure (dev, &dev_conf);
637  if (ret < 0)
638  return clib_error_return (0, error_str, dev);
639 
640  error_str = "failed to setup crypto device %u queue pair %u";
641  qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
642  for (qp = 0; qp < n_qp; qp++)
643  {
644 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
645  ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa, NULL);
646 #else
647  ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa);
648 #endif
649  if (ret < 0)
650  return clib_error_return (0, error_str, dev, qp);
651  }
652 
653  error_str = "failed to start crypto device %u";
654  if (rte_cryptodev_start (dev))
655  return clib_error_return (0, error_str, dev);
656 
657  return 0;
658 }
659 
660 static void
662 {
664  struct rte_cryptodev *cryptodev;
665  struct rte_cryptodev_info info = { 0 };
666  crypto_dev_t *dev;
667  crypto_resource_t *res;
668  clib_error_t *error;
669  u32 i;
670  u16 max_res_idx, res_idx, j;
671  u8 drv_id;
672 
673  vec_validate_init_empty (dcm->dev, rte_cryptodev_count () - 1,
675 
676  for (i = 0; i < rte_cryptodev_count (); i++)
677  {
678  dev = vec_elt_at_index (dcm->dev, i);
679 
680  cryptodev = &rte_cryptodevs[i];
681  rte_cryptodev_info_get (i, &info);
682 
683  dev->id = i;
684  dev->name = cryptodev->data->name;
685  dev->numa = rte_cryptodev_socket_id (i);
686  dev->features = info.feature_flags;
687  dev->max_qp = info.max_nb_queue_pairs;
688  drv_id = info.driver_id;
689  if (drv_id >= vec_len (dcm->drv))
690  vec_validate_init_empty (dcm->drv, drv_id,
691  (crypto_drv_t) EMPTY_STRUCT);
692  vec_elt_at_index (dcm->drv, drv_id)->name = info.driver_name;
693  dev->drv_id = drv_id;
694  vec_add1 (vec_elt_at_index (dcm->drv, drv_id)->devs, i);
695 
696  if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
697  continue;
698 
699  if ((error = crypto_dev_conf (i, dev->max_qp, dev->numa)))
700  {
701  clib_error_report (error);
702  continue;
703  }
704 
705  max_res_idx = dev->max_qp - 1;
706 
707  vec_validate (dev->free_resources, max_res_idx);
708 
709  res_idx = vec_len (dcm->resource);
710  vec_validate_init_empty_aligned (dcm->resource, res_idx + max_res_idx,
711  (crypto_resource_t) EMPTY_STRUCT,
713 
714  for (j = 0; j <= max_res_idx; j++)
715  {
716  vec_elt (dev->free_resources, max_res_idx - j) = res_idx + j;
717  res = &dcm->resource[res_idx + j];
718  res->dev_id = i;
719  res->drv_id = drv_id;
720  res->qp_id = j;
721  res->numa = dev->numa;
722  res->thread_idx = (u16) ~ 0;
723  }
724 
725  crypto_parse_capabilities (dev, info.capabilities, n_mains);
726  }
727 }
728 
729 void
731 {
733  crypto_resource_t *res;
735  crypto_dev_t *dev;
736  u32 thread_idx, skip_master;
737  u16 res_idx, *idx;
738  u8 used;
739  u16 i;
740 
741  skip_master = vlib_num_workers () > 0;
742 
743  /* *INDENT-OFF* */
744  vec_foreach (dev, dcm->dev)
745  {
746  vec_foreach_index (thread_idx, dcm->workers_main)
747  {
748  if (vec_len (dev->free_resources) == 0)
749  break;
750 
751  if (thread_idx < skip_master)
752  continue;
753 
754  /* Check thread is not already using the device */
755  vec_foreach (idx, dev->used_resources)
756  if (dcm->resource[idx[0]].thread_idx == thread_idx)
757  continue;
758 
759  cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
760 
761  used = 0;
762  res_idx = vec_pop (dev->free_resources);
763 
764  /* Set device only for supported algos */
765  for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
766  if (dev->cipher_support[i] &&
767  cwm->cipher_resource_idx[i] == (u16) ~0)
768  {
769  dcm->cipher_algs[i].disabled--;
770  cwm->cipher_resource_idx[i] = res_idx;
771  used = 1;
772  }
773 
774  for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
775  if (dev->auth_support[i] &&
776  cwm->auth_resource_idx[i] == (u16) ~0)
777  {
778  dcm->auth_algs[i].disabled--;
779  cwm->auth_resource_idx[i] = res_idx;
780  used = 1;
781  }
782 
783  if (!used)
784  {
785  vec_add1 (dev->free_resources, res_idx);
786  continue;
787  }
788 
789  vec_add1 (dev->used_resources, res_idx);
790 
791  res = vec_elt_at_index (dcm->resource, res_idx);
792 
793  ASSERT (res->thread_idx == (u16) ~0);
794  res->thread_idx = thread_idx;
795 
796  /* Add device to vector of polling resources */
797  vec_add1 (cwm->resource_idx, res_idx);
798  }
799  }
800  /* *INDENT-ON* */
801 }
802 
803 static void
804 crypto_op_init (struct rte_mempool *mempool,
805  void *_arg __attribute__ ((unused)),
806  void *_obj, unsigned i __attribute__ ((unused)))
807 {
808  struct rte_crypto_op *op = _obj;
809 
810  op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
811  op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
812  op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
813  op->phys_addr = rte_mempool_virt2iova (_obj);
814  op->mempool = mempool;
815 }
816 
817 static clib_error_t *
819 {
823  u8 *pool_name;
824  u32 pool_priv_size = sizeof (struct rte_crypto_op_pool_private);
825  struct rte_crypto_op_pool_private *priv;
826  struct rte_mempool *mp;
827 
828  data = vec_elt_at_index (dcm->data, numa);
829 
830  /* Already allocated */
831  if (data->crypto_op)
832  return NULL;
833 
834  pool_name = format (0, "crypto_pool_numa%u%c", numa, 0);
835 
836  if (conf->num_crypto_mbufs == 0)
838 
839  mp = rte_mempool_create ((char *) pool_name, conf->num_crypto_mbufs,
840  crypto_op_len (), 512, pool_priv_size, NULL, NULL,
841  crypto_op_init, NULL, numa, 0);
842 
843  vec_free (pool_name);
844 
845  if (!mp)
846  return clib_error_return (0, "failed to create crypto op mempool");
847 
848  /* Initialize mempool private data */
849  priv = rte_mempool_get_priv (mp);
850  priv->priv_size = pool_priv_size;
851  priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
852 
853  data->crypto_op = mp;
854 
855  return NULL;
856 }
857 
858 static clib_error_t *
860 {
863  u8 *pool_name;
864  struct rte_mempool *mp;
865  u32 elt_size;
866 
867  data = vec_elt_at_index (dcm->data, numa);
868 
869  if (data->session_h)
870  return NULL;
871 
872  pool_name = format (0, "session_h_pool_numa%u%c", numa, 0);
873 
874 
875  elt_size = rte_cryptodev_sym_get_header_session_size ();
876 
877 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0)
878  mp = rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
879  elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
880 #else
881  /* XXX Experimental tag in DPDK 19.02 */
882  mp = rte_cryptodev_sym_session_pool_create ((char *) pool_name,
884  elt_size, 512, 0, numa);
885 #endif
886  vec_free (pool_name);
887 
888  if (!mp)
889  return clib_error_return (0, "failed to create crypto session mempool");
890 
891  data->session_h = mp;
892 
893  return NULL;
894 }
895 
896 static clib_error_t *
898 {
901  u8 *pool_name;
902  struct rte_mempool *mp;
903  u32 elt_size;
904  u8 numa = dev->numa;
905 
906  data = vec_elt_at_index (dcm->data, numa);
907 
908  vec_validate (data->session_drv, dev->drv_id);
909  vec_validate (data->session_drv_failed, dev->drv_id);
912 
913  if (data->session_drv[dev->drv_id])
914  return NULL;
915 
916  pool_name = format (0, "session_drv%u_pool_numa%u%c", dev->drv_id, numa, 0);
917 
918  elt_size = rte_cryptodev_sym_get_private_session_size (dev->id);
919  mp =
920  rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
921  elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
922 
923  vec_free (pool_name);
924 
925  if (!mp)
926  return clib_error_return (0, "failed to create session drv mempool");
927 
928  data->session_drv[dev->drv_id] = mp;
929  clib_spinlock_init (&data->lockp);
930 
931  return NULL;
932 }
933 
934 static clib_error_t *
936 {
938  clib_error_t *error = NULL;
939  crypto_dev_t *dev;
940 
941  /* *INDENT-OFF* */
942  vec_foreach (dev, dcm->dev)
943  {
945 
946  error = crypto_create_crypto_op_pool (vm, dev->numa);
947  if (error)
948  return error;
949 
950  error = crypto_create_session_h_pool (vm, dev->numa);
951  if (error)
952  return error;
953 
954  error = crypto_create_session_drv_pool (vm, dev);
955  if (error)
956  return error;
957  }
958  /* *INDENT-ON* */
959 
960  return NULL;
961 }
962 
963 static void
965 {
968  u8 i;
969 
970  dcm->enabled = 0;
971 
972  /* *INDENT-OFF* */
973  vec_foreach (data, dcm->data)
974  {
975  rte_mempool_free (data->crypto_op);
976  rte_mempool_free (data->session_h);
977 
978  vec_foreach_index (i, data->session_drv)
979  rte_mempool_free (data->session_drv[i]);
980 
981  vec_free (data->session_drv);
982  clib_spinlock_free (&data->lockp);
983  }
984  /* *INDENT-ON* */
985 
986  vec_free (dcm->data);
987  vec_free (dcm->workers_main);
988  vec_free (dcm->dev);
989  vec_free (dcm->resource);
990  vec_free (dcm->cipher_algs);
991  vec_free (dcm->auth_algs);
992 }
993 
994 static uword
996  vlib_frame_t * f)
997 {
998  ipsec_main_t *im = &ipsec_main;
1001  crypto_worker_main_t *cwm;
1002  clib_error_t *error = NULL;
1003  u32 i, skip_master, n_mains;
1004 
1005  n_mains = tm->n_vlib_mains;
1006  skip_master = vlib_num_workers () > 0;
1007 
1008  algos_init (n_mains - skip_master);
1009 
1010  crypto_scan_devs (n_mains - skip_master);
1011 
1012  if (!(dcm->enabled))
1013  {
1014  clib_warning ("not enough DPDK crypto resources, default to OpenSSL");
1015  crypto_disable ();
1016  return 0;
1017  }
1018 
1019  dcm->session_timeout = 10e9;
1020 
1021  vec_validate_init_empty_aligned (dcm->workers_main, n_mains - 1,
1024 
1025  /* *INDENT-OFF* */
1026  vec_foreach (cwm, dcm->workers_main)
1027  {
1030  clib_memset (cwm->cipher_resource_idx, ~0,
1031  IPSEC_CRYPTO_N_ALG * sizeof(*cwm->cipher_resource_idx));
1032  clib_memset (cwm->auth_resource_idx, ~0,
1033  IPSEC_INTEG_N_ALG * sizeof(*cwm->auth_resource_idx));
1034  }
1035  /* *INDENT-ON* */
1036 
1038 
1039  error = crypto_create_pools (vm);
1040  if (error)
1041  {
1042  clib_error_report (error);
1043  crypto_disable ();
1044  return 0;
1045  }
1046 
1047 
1048  u32 idx = ipsec_register_esp_backend (vm, im, "dpdk backend",
1049  "dpdk-esp4-encrypt",
1050  "dpdk-esp4-encrypt-tun",
1051  "dpdk-esp4-decrypt",
1052  "dpdk-esp6-encrypt",
1053  "dpdk-esp6-encrypt-tun",
1054  "dpdk-esp6-decrypt",
1057  int rv = ipsec_select_esp_backend (im, idx);
1058  ASSERT (rv == 0);
1059 
1060  vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "dpdk-crypto-input");
1061  ASSERT (node);
1062  for (i = skip_master; i < n_mains; i++)
1063  vlib_node_set_state (vlib_mains[i], node->index, VLIB_NODE_STATE_POLLING);
1064  return 0;
1065 }
1066 
1067 /* *INDENT-OFF* */
1068 VLIB_REGISTER_NODE (dpdk_ipsec_process_node,static) = {
1069  .function = dpdk_ipsec_process,
1070  .type = VLIB_NODE_TYPE_PROCESS,
1071  .name = "dpdk-ipsec-process",
1072  .process_log2_n_stack_bytes = 17,
1073 };
1074 /* *INDENT-ON* */
1075 
1076 /*
1077  * fd.io coding-style-patch-verification: ON
1078  *
1079  * Local Variables:
1080  * eval: (c-set-style "gnu")
1081  * End:
1082  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
u32 alg
Definition: ipsec.h:86
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:110
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:255
#define hash_unset(h, key)
Definition: hash.h:261
a
Definition: bitmap.h:538
static u8 cipher_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:177
unsigned long u64
Definition: types.h:89
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 index
Definition: node.h:280
u64 session_h_failed
Definition: ipsec.h:152
#define foreach_ipsec_crypto_alg
Definition: ipsec_sa.h:22
ipsec_key_t crypto_key
Definition: ipsec_sa.h:147
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:110
ipsec_integ_alg_t integ_alg
Definition: ipsec_sa.h:150
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:522
static clib_error_t * crypto_create_session_drv_pool(vlib_main_t *vm, crypto_dev_t *dev)
Definition: ipsec.c:897
#define EMPTY_STRUCT
Definition: ipsec.c:27
static_always_inline u32 crypto_op_get_priv_offset(void)
Definition: ipsec.h:195
int i
#define foreach_ipsec_integ_alg
Definition: ipsec_sa.h:49
#define DPDK_CRYPTO_NB_SESS_OBJS
Definition: ipsec.h:34
struct rte_cryptodev_sym_session * session
Definition: ipsec.h:134
u8 disabled
Definition: ipsec.h:91
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:424
u8 data[128]
Definition: ipsec.api:251
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
static void crypto_set_cipher_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:271
vlib_main_t ** vlib_mains
Definition: buffer.c:332
static clib_error_t * crypto_create_session_h_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:859
unsigned char u8
Definition: types.h:56
#define vec_pop(V)
Returns last element of a vector and decrements its length.
Definition: vec.h:615
static crypto_alg_t * cipher_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 key_len)
Definition: ipsec.c:193
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:70
static clib_error_t * add_del_sa_session(u32 sa_index, u8 is_add)
Definition: ipsec.c:495
u16 * resource_idx
Definition: ipsec.h:74
#define AES_GCM_TYPE
#define DPDK_CRYPTO_N_QUEUE_DESC
Definition: ipsec.h:33
u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:76
static void algos_init(u32 n_mains)
Definition: ipsec.c:31
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:25
dpdk_config_main_t dpdk_config_main
Definition: init.c:46
ipsec_main_t ipsec_main
Definition: ipsec.c:28
u8 resources
Definition: ipsec.h:92
int ipsec_select_esp_backend(ipsec_main_t *im, u32 backend_idx)
Definition: ipsec.c:241
#define NUM_CRYPTO_MBUFS
Definition: ipsec.c:28
crypto_drv_t * drv
Definition: ipsec.h:168
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 drv_id
Definition: ipsec.h:101
#define clib_error_return(e, args...)
Definition: error.h:99
static void crypto_parse_capabilities(crypto_dev_t *dev, const struct rte_cryptodev_capabilities *cap, u32 n_mains)
Definition: ipsec.c:573
u16 * free_resources
Definition: ipsec.h:97
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:78
unsigned int u32
Definition: types.h:88
#define vec_end(v)
End (last data address) of vector.
crypto_alg_t * auth_algs
Definition: ipsec.h:166
#define VLIB_FRAME_SIZE
Definition: node.h:378
static clib_error_t * crypto_dev_conf(u8 dev, u16 n_qp, u8 numa)
Definition: ipsec.c:624
u8 trunc_size
Definition: ipsec.h:89
static_always_inline void add_session_by_drv_and_sa_idx(struct rte_cryptodev_sym_session *session, crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:214
static void crypto_disable(void)
Definition: ipsec.c:964
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:63
clib_spinlock_t lockp
Definition: ipsec.h:155
static void * get_session_private_data(const struct rte_cryptodev_sym_session *sess, uint8_t driver_id)
Definition: ipsec.c:419
u16 * used_resources
Definition: ipsec.h:98
#define hash_get(h, key)
Definition: hash.h:249
u32 ipsec_register_esp_backend(vlib_main_t *vm, ipsec_main_t *im, const char *name, const char *esp4_encrypt_node_name, const char *esp4_encrypt_node_tun_name, const char *esp4_decrypt_node_name, const char *esp6_encrypt_node_name, const char *esp6_encrypt_node_tun_name, const char *esp6_decrypt_node_name, check_support_cb_t esp_check_support_cb, add_del_sa_sess_cb_t esp_add_del_sa_sess_cb)
Definition: ipsec.c:165
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:514
crypto_alg_t * cipher_algs
Definition: ipsec.h:165
unsigned short u16
Definition: types.h:57
u64 features
Definition: ipsec.h:106
u64 session_timeout
Definition: ipsec.h:169
static_always_inline u32 crypto_op_len(void)
Definition: ipsec.h:185
#define AES_GCM_ALG
static clib_error_t * crypto_create_crypto_op_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:818
static u8 auth_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:185
struct rte_mempool ** session_drv
Definition: ipsec.h:148
u8 len
Definition: ip_types.api:90
vlib node functions
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
crypto_session_by_drv_t * session_by_drv_id_and_sa_index
Definition: ipsec.h:154
u32 num_crypto_mbufs
Definition: dpdk.h:359
svmdb_client_t * c
vlib_main_t * vm
Definition: buffer.c:323
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
#define clib_warning(format, args...)
Definition: error.h:59
u8 is_outbound
Definition: ipsec.api:93
static u64 unix_time_now_nsec(void)
Definition: time.h:270
crypto_session_disposal_t * session_disposal
Definition: ipsec.h:149
static void crypto_set_aead_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:243
vlib_node_t * vlib_get_node_by_name(vlib_main_t *vm, u8 *name)
Definition: node.c:45
u16 id
Definition: ipsec.h:103
static void set_session_private_data(struct rte_cryptodev_sym_session *sess, uint8_t driver_id, void *private_data)
Definition: ipsec.c:434
u8 iv_len
Definition: ipsec.h:88
signed int i32
Definition: types.h:77
u8 data[IPSEC_KEY_MAX_LEN]
Definition: ipsec_sa.h:76
#define ASSERT(truth)
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:784
struct rte_mempool * crypto_op
Definition: ipsec.h:146
u32 max_qp
Definition: ipsec.h:105
ipsec_sa_t * sad
Definition: ipsec.h:97
static clib_error_t * dpdk_crypto_session_disposal(crypto_session_disposal_t *v, u64 ts)
Definition: ipsec.c:447
crypto_worker_main_t * workers_main
Definition: ipsec.h:162
#define clib_error_report(e)
Definition: error.h:113
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
Definition: node_funcs.h:148
crypto_resource_t * resource
Definition: ipsec.h:164
#define vec_elt(v, i)
Get vector value at index i.
static clib_error_t * crypto_create_pools(vlib_main_t *vm)
Definition: ipsec.c:935
struct rte_mempool * session_h
Definition: ipsec.h:147
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void crypto_auto_placement(void)
Definition: ipsec.c:730
u64 * session_drv_failed
Definition: ipsec.h:153
u64 uword
Definition: types.h:112
uword * session_by_sa_index
Definition: ipsec.h:150
static void crypto_scan_devs(u32 n_mains)
Definition: ipsec.c:661
const char * name
Definition: ipsec.h:104
static void crypto_op_init(struct rte_mempool *mempool, void *_arg, void *_obj, unsigned i)
Definition: ipsec.c:804
struct rte_crypto_op ** ops
Definition: ipsec.h:75
clib_error_t * create_sym_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.c:321
crypto_dev_t * dev
Definition: ipsec.h:163
enum rte_crypto_sym_xform_type type
Definition: ipsec.h:85
crypto_data_t * data
Definition: ipsec.h:167
ipsec_crypto_alg_t crypto_alg
Definition: ipsec_sa.h:146
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static u32 vlib_num_workers()
Definition: threads.h:372
static uword dpdk_ipsec_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: ipsec.c:995
u16 auth_resource_idx[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:77
u8 auth_support[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:100
#define vec_validate_init_empty_aligned(V, I, INIT, A)
Make sure vector is long enough for given index and initialize empty space (no header, alignment alignment)
Definition: vec.h:498
#define vec_foreach(var, vec)
Vector iterator.
static_always_inline struct rte_cryptodev_sym_session * get_session_by_drv_and_sa_idx(crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:226
u8 key_len
Definition: ipsec.h:87
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:486
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
ipsec_key_t integ_key
Definition: ipsec_sa.h:151
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:95
static void crypto_set_auth_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:297
static clib_error_t * dpdk_ipsec_check_support(ipsec_sa_t *sa)
Definition: ipsec.c:540
u8 cipher_support[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:99
static void clear_and_free_obj(void *obj)
Definition: ipsec.c:408
static crypto_alg_t * auth_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
Definition: ipsec.c:221