FD.io VPP  v19.01.3-6-g70449b9b9
Vector Packet Processing
ipsec.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Intel and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #include <vnet/vnet.h>
16 #include <vnet/ip/ip.h>
17 #include <vnet/api_errno.h>
18 #include <vnet/ipsec/ipsec.h>
19 #include <vlib/node_funcs.h>
20 
21 #include <dpdk/device/dpdk.h>
22 #include <dpdk/ipsec/ipsec.h>
23 
25 
26 #define EMPTY_STRUCT {0}
27 
28 static void
29 algos_init (u32 n_mains)
30 {
32  crypto_alg_t *a;
33 
35 
36  {
37 #define _(v,f,str) \
38  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].name = str; \
39  dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].disabled = n_mains;
41 #undef _
42  }
43 
44  /* Minimum boundary for ciphers is 4B, required by ESP */
45  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_NONE];
46  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
47  a->alg = RTE_CRYPTO_CIPHER_NULL;
48  a->boundary = 4; /* 1 */
49  a->key_len = 0;
50  a->iv_len = 0;
51 
52  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
53  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
54  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
55  a->boundary = 16;
56  a->key_len = 16;
57  a->iv_len = 16;
58 
59  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
60  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
61  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
62  a->boundary = 16;
63  a->key_len = 24;
64  a->iv_len = 16;
65 
66  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
67  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
68  a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
69  a->boundary = 16;
70  a->key_len = 32;
71  a->iv_len = 16;
72 
73  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_128];
74  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
75  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
76  a->boundary = 4; /* 1 */
77  a->key_len = 16;
78  a->iv_len = 8;
79 
80  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_192];
81  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
82  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
83  a->boundary = 4; /* 1 */
84  a->key_len = 24;
85  a->iv_len = 8;
86 
87  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_256];
88  a->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
89  a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
90  a->boundary = 4; /* 1 */
91  a->key_len = 32;
92  a->iv_len = 8;
93 
94 #define AES_GCM_TYPE RTE_CRYPTO_SYM_XFORM_AEAD
95 #define AES_GCM_ALG RTE_CRYPTO_AEAD_AES_GCM
96 
97  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
98  a->type = AES_GCM_TYPE;
99  a->alg = AES_GCM_ALG;
100  a->boundary = 4; /* 1 */
101  a->key_len = 16;
102  a->iv_len = 8;
103  a->trunc_size = 16;
104 
105  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_192];
106  a->type = AES_GCM_TYPE;
107  a->alg = AES_GCM_ALG;
108  a->boundary = 4; /* 1 */
109  a->key_len = 24;
110  a->iv_len = 8;
111  a->trunc_size = 16;
112 
113  a = &dcm->cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_256];
114  a->type = AES_GCM_TYPE;
115  a->alg = AES_GCM_ALG;
116  a->boundary = 4; /* 1 */
117  a->key_len = 32;
118  a->iv_len = 8;
119  a->trunc_size = 16;
120 
122 
123  {
124 #define _(v,f,str) \
125  dcm->auth_algs[IPSEC_INTEG_ALG_##f].name = str; \
126  dcm->auth_algs[IPSEC_INTEG_ALG_##f].disabled = n_mains;
128 #undef _
129  }
130 
131  a = &dcm->auth_algs[IPSEC_INTEG_ALG_NONE];
132  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
133  a->alg = RTE_CRYPTO_AUTH_NULL;
134  a->key_len = 0;
135  a->trunc_size = 0;
136 
137  a = &dcm->auth_algs[IPSEC_INTEG_ALG_MD5_96];
138  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
139  a->alg = RTE_CRYPTO_AUTH_MD5_HMAC;
140  a->key_len = 16;
141  a->trunc_size = 12;
142 
143  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA1_96];
144  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
145  a->alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
146  a->key_len = 20;
147  a->trunc_size = 12;
148 
149  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_96];
150  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
151  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
152  a->key_len = 32;
153  a->trunc_size = 12;
154 
155  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_256_128];
156  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
157  a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
158  a->key_len = 32;
159  a->trunc_size = 16;
160 
161  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_384_192];
162  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
163  a->alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
164  a->key_len = 48;
165  a->trunc_size = 24;
166 
167  a = &dcm->auth_algs[IPSEC_INTEG_ALG_SHA_512_256];
168  a->type = RTE_CRYPTO_SYM_XFORM_AUTH;
169  a->alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
170  a->key_len = 64;
171  a->trunc_size = 32;
172 }
173 
174 static u8
176 {
178 
179  return (alg - dcm->cipher_algs);
180 }
181 
182 static u8
184 {
186 
187  return (alg - dcm->auth_algs);
188 }
189 
190 static crypto_alg_t *
191 cipher_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 key_len)
192 {
194  crypto_alg_t *alg;
195 
196  if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
197  return NULL;
198 
199  /* *INDENT-OFF* */
200  vec_foreach (alg, dcm->cipher_algs)
201  {
202  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
203  (alg->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
204  (cap->sym.cipher.algo == alg->alg) &&
205  (alg->key_len == key_len))
206  return alg;
207  if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
208  (alg->type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
209  (cap->sym.aead.algo == alg->alg) &&
210  (alg->key_len == key_len))
211  return alg;
212  }
213  /* *INDENT-ON* */
214 
215  return NULL;
216 }
217 
218 static crypto_alg_t *
219 auth_cap_to_alg (const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
220 {
222  crypto_alg_t *alg;
223 
224  if ((cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) ||
225  (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH))
226  return NULL;
227 
228  /* *INDENT-OFF* */
229  vec_foreach (alg, dcm->auth_algs)
230  {
231  if ((cap->sym.auth.algo == alg->alg) &&
232  (alg->trunc_size == trunc_size))
233  return alg;
234  }
235  /* *INDENT-ON* */
236 
237  return NULL;
238 }
239 
240 static void
241 crypto_set_aead_xform (struct rte_crypto_sym_xform *xform,
242  ipsec_sa_t * sa, u8 is_outbound)
243 {
245  crypto_alg_t *c;
246 
247  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
248 
249  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_AEAD);
250 
251  xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
252  xform->aead.algo = c->alg;
253  xform->aead.key.data = sa->crypto_key;
254  xform->aead.key.length = c->key_len;
255  xform->aead.iv.offset =
256  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
257  xform->aead.iv.length = 12;
258  xform->aead.digest_length = c->trunc_size;
259  xform->aead.aad_length = sa->use_esn ? 12 : 8;
260  xform->next = NULL;
261 
262  if (is_outbound)
263  xform->aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
264  else
265  xform->aead.op = RTE_CRYPTO_AEAD_OP_DECRYPT;
266 }
267 
268 static void
269 crypto_set_cipher_xform (struct rte_crypto_sym_xform *xform,
270  ipsec_sa_t * sa, u8 is_outbound)
271 {
273  crypto_alg_t *c;
274 
275  c = vec_elt_at_index (dcm->cipher_algs, sa->crypto_alg);
276 
277  ASSERT (c->type == RTE_CRYPTO_SYM_XFORM_CIPHER);
278 
279  xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
280  xform->cipher.algo = c->alg;
281  xform->cipher.key.data = sa->crypto_key;
282  xform->cipher.key.length = c->key_len;
283  xform->cipher.iv.offset =
284  crypto_op_get_priv_offset () + offsetof (dpdk_op_priv_t, cb);
285  xform->cipher.iv.length = c->iv_len;
286  xform->next = NULL;
287 
288  if (is_outbound)
289  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
290  else
291  xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
292 }
293 
294 static void
295 crypto_set_auth_xform (struct rte_crypto_sym_xform *xform,
296  ipsec_sa_t * sa, u8 is_outbound)
297 {
299  crypto_alg_t *a;
300 
301  a = vec_elt_at_index (dcm->auth_algs, sa->integ_alg);
302 
303  ASSERT (a->type == RTE_CRYPTO_SYM_XFORM_AUTH);
304 
305  xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
306  xform->auth.algo = a->alg;
307  xform->auth.key.data = sa->integ_key;
308  xform->auth.key.length = a->key_len;
309  xform->auth.digest_length = a->trunc_size;
310  xform->next = NULL;
311 
312  if (is_outbound)
313  xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
314  else
315  xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
316 }
317 
318 clib_error_t *
319 create_sym_session (struct rte_cryptodev_sym_session **session,
320  u32 sa_idx,
321  crypto_resource_t * res,
322  crypto_worker_main_t * cwm, u8 is_outbound)
323 {
325  ipsec_main_t *im = &ipsec_main;
326  crypto_data_t *data;
327  ipsec_sa_t *sa;
328  struct rte_crypto_sym_xform cipher_xform = { 0 };
329  struct rte_crypto_sym_xform auth_xform = { 0 };
330  struct rte_crypto_sym_xform *xfs;
331  struct rte_cryptodev_sym_session **s;
332  clib_error_t *erorr = 0;
333 
334 
335  sa = pool_elt_at_index (im->sad, sa_idx);
336 
337  if ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
338  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
339  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256))
340  {
341  crypto_set_aead_xform (&cipher_xform, sa, is_outbound);
342  xfs = &cipher_xform;
343  }
344  else
345  {
346  crypto_set_cipher_xform (&cipher_xform, sa, is_outbound);
347  crypto_set_auth_xform (&auth_xform, sa, is_outbound);
348 
349  if (is_outbound)
350  {
351  cipher_xform.next = &auth_xform;
352  xfs = &cipher_xform;
353  }
354  else
355  {
356  auth_xform.next = &cipher_xform;
357  xfs = &auth_xform;
358  }
359  }
360 
361  data = vec_elt_at_index (dcm->data, res->numa);
363 
364  /*
365  * DPDK_VER >= 1708:
366  * Multiple worker/threads share the session for an SA
367  * Single session per SA, initialized for each device driver
368  */
369  s = (void *) hash_get (data->session_by_sa_index, sa_idx);
370 
371  if (!s)
372  {
373  session[0] = rte_cryptodev_sym_session_create (data->session_h);
374  if (!session[0])
375  {
376  data->session_h_failed += 1;
377  erorr = clib_error_return (0, "failed to create session header");
378  goto done;
379  }
380  hash_set (data->session_by_sa_index, sa_idx, session[0]);
381  }
382  else
383  session[0] = s[0];
384 
385  struct rte_mempool **mp;
386  mp = vec_elt_at_index (data->session_drv, res->drv_id);
387  ASSERT (mp[0] != NULL);
388 
389  i32 ret =
390  rte_cryptodev_sym_session_init (res->dev_id, session[0], xfs, mp[0]);
391  if (ret)
392  {
393  data->session_drv_failed[res->drv_id] += 1;
394  erorr = clib_error_return (0, "failed to init session for drv %u",
395  res->drv_id);
396  goto done;
397  }
398 
399  add_session_by_drv_and_sa_idx (session[0], data, res->drv_id, sa_idx);
400 
401 done:
403  return erorr;
404 }
405 
406 static void __attribute__ ((unused)) clear_and_free_obj (void *obj)
407 {
408  struct rte_mempool *mp = rte_mempool_from_obj (obj);
409 
410  clib_memset (obj, 0, mp->elt_size);
411 
412  rte_mempool_put (mp, obj);
413 }
414 
415 /* This is from rte_cryptodev_pmd.h */
416 static inline void *
417 get_session_private_data (const struct rte_cryptodev_sym_session *sess,
418  uint8_t driver_id)
419 {
420  return sess->sess_private_data[driver_id];
421 }
422 
423 /* This is from rte_cryptodev_pmd.h */
424 static inline void
425 set_session_private_data (struct rte_cryptodev_sym_session *sess,
426  uint8_t driver_id, void *private_data)
427 {
428  sess->sess_private_data[driver_id] = private_data;
429 }
430 
431 static clib_error_t *
433 {
436  void *drv_session;
437  u32 drv_id;
438  i32 ret;
439 
440  /* *INDENT-OFF* */
441  vec_foreach (s, v)
442  {
443  /* ordered vector by timestamp */
444  if (!(s->ts + dcm->session_timeout < ts))
445  break;
446 
447  vec_foreach_index (drv_id, dcm->drv)
448  {
449  drv_session = get_session_private_data (s->session, drv_id);
450  if (!drv_session)
451  continue;
452 
453  /*
454  * Custom clear to avoid finding a dev_id for drv_id:
455  * ret = rte_cryptodev_sym_session_clear (dev_id, drv_session);
456  * ASSERT (!ret);
457  */
458  clear_and_free_obj (drv_session);
459 
460  set_session_private_data (s->session, drv_id, NULL);
461  }
462 
463  if (rte_mempool_from_obj(s->session))
464  {
465  ret = rte_cryptodev_sym_session_free (s->session);
466  ASSERT (!ret);
467  }
468  }
469  /* *INDENT-ON* */
470 
471  if (s < vec_end (v))
472  vec_delete (v, s - v, 0);
473  else
474  vec_reset_length (v);
475 
476  return 0;
477 }
478 
479 static clib_error_t *
480 add_del_sa_session (u32 sa_index, u8 is_add)
481 {
482  ipsec_main_t *im = &ipsec_main;
484  crypto_data_t *data;
485  struct rte_cryptodev_sym_session *s;
486  uword *val;
487  u32 drv_id;
488 
489  if (is_add)
490  {
491 #if 1
492  ipsec_sa_t *sa = pool_elt_at_index (im->sad, sa_index);
493  u32 seed;
494  switch (sa->crypto_alg)
495  {
496  case IPSEC_CRYPTO_ALG_AES_GCM_128:
497  case IPSEC_CRYPTO_ALG_AES_GCM_192:
498  case IPSEC_CRYPTO_ALG_AES_GCM_256:
499  clib_memcpy (&sa->salt, &sa->crypto_key[sa->crypto_key_len - 4], 4);
500  break;
501  default:
502  seed = (u32) clib_cpu_time_now ();
503  sa->salt = random_u32 (&seed);
504  }
505 #endif
506  return 0;
507  }
508 
509  /* *INDENT-OFF* */
510  vec_foreach (data, dcm->data)
511  {
513  val = hash_get (data->session_by_sa_index, sa_index);
514  if (val)
515  {
516  s = (struct rte_cryptodev_sym_session *) val[0];
517  vec_foreach_index (drv_id, dcm->drv)
518  {
519  val = (uword*) get_session_by_drv_and_sa_idx (data, drv_id, sa_index);
520  if (val)
521  add_session_by_drv_and_sa_idx(NULL, data, drv_id, sa_index);
522  }
523 
524  hash_unset (data->session_by_sa_index, sa_index);
525 
526  u64 ts = unix_time_now_nsec ();
528 
530  sd.ts = ts;
531  sd.session = s;
532 
533  vec_add1 (data->session_disposal, sd);
534  }
536  }
537  /* *INDENT-ON* */
538 
539  return 0;
540 }
541 
542 static clib_error_t *
544 {
546 
547  if (sa->integ_alg == IPSEC_INTEG_ALG_NONE)
548  switch (sa->crypto_alg)
549  {
550  case IPSEC_CRYPTO_ALG_NONE:
551  case IPSEC_CRYPTO_ALG_AES_GCM_128:
552  case IPSEC_CRYPTO_ALG_AES_GCM_192:
553  case IPSEC_CRYPTO_ALG_AES_GCM_256:
554  break;
555  default:
556  return clib_error_return (0, "unsupported integ-alg %U crypto-alg %U",
559  }
560 
561  /* XXX do we need the NONE check? */
562  if (sa->crypto_alg != IPSEC_CRYPTO_ALG_NONE &&
563  dcm->cipher_algs[sa->crypto_alg].disabled)
564  return clib_error_return (0, "disabled crypto-alg %U",
566 
567  /* XXX do we need the NONE check? */
568  if (sa->integ_alg != IPSEC_INTEG_ALG_NONE &&
569  dcm->auth_algs[sa->integ_alg].disabled)
570  return clib_error_return (0, "disabled integ-alg %U",
572  return NULL;
573 }
574 
575 static void
577  const struct rte_cryptodev_capabilities *cap,
578  u32 n_mains)
579 {
581  crypto_alg_t *alg;
582  u8 len, inc;
583 
584  for (; cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; cap++)
585  {
586  /* A single capability maps to multiple cipher/auth algorithms */
587  switch (cap->sym.xform_type)
588  {
589  case RTE_CRYPTO_SYM_XFORM_AEAD:
590  case RTE_CRYPTO_SYM_XFORM_CIPHER:
591  inc = cap->sym.cipher.key_size.increment;
592  inc = inc ? inc : 1;
593  for (len = cap->sym.cipher.key_size.min;
594  len <= cap->sym.cipher.key_size.max; len += inc)
595  {
596  alg = cipher_cap_to_alg (cap, len);
597  if (!alg)
598  continue;
599  dev->cipher_support[cipher_alg_index (alg)] = 1;
600  alg->resources += vec_len (dev->free_resources);
601  /* At least enough resources to support one algo */
602  dcm->enabled |= (alg->resources >= n_mains);
603  }
604  break;
605  case RTE_CRYPTO_SYM_XFORM_AUTH:
606  inc = cap->sym.auth.digest_size.increment;
607  inc = inc ? inc : 1;
608  for (len = cap->sym.auth.digest_size.min;
609  len <= cap->sym.auth.digest_size.max; len += inc)
610  {
611  alg = auth_cap_to_alg (cap, len);
612  if (!alg)
613  continue;
614  dev->auth_support[auth_alg_index (alg)] = 1;
615  alg->resources += vec_len (dev->free_resources);
616  /* At least enough resources to support one algo */
617  dcm->enabled |= (alg->resources >= n_mains);
618  }
619  break;
620  default:
621  ;
622  }
623  }
624 }
625 
626 #define DPDK_CRYPTO_N_QUEUE_DESC 2048
627 #define DPDK_CRYPTO_NB_SESS_OBJS 20000
628 
629 static clib_error_t *
630 crypto_dev_conf (u8 dev, u16 n_qp, u8 numa)
631 {
632  struct rte_cryptodev_config dev_conf;
633  struct rte_cryptodev_qp_conf qp_conf;
634  i32 ret;
635  u16 qp;
636  char *error_str;
637 
638  dev_conf.socket_id = numa;
639  dev_conf.nb_queue_pairs = n_qp;
640 
641  error_str = "failed to configure crypto device %u";
642  ret = rte_cryptodev_configure (dev, &dev_conf);
643  if (ret < 0)
644  return clib_error_return (0, error_str, dev);
645 
646  error_str = "failed to setup crypto device %u queue pair %u";
647  qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
648  for (qp = 0; qp < n_qp; qp++)
649  {
650  ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa, NULL);
651  if (ret < 0)
652  return clib_error_return (0, error_str, dev, qp);
653  }
654 
655  error_str = "failed to start crypto device %u";
656  if (rte_cryptodev_start (dev))
657  return clib_error_return (0, error_str, dev);
658 
659  return 0;
660 }
661 
662 static void
664 {
666  struct rte_cryptodev *cryptodev;
667  struct rte_cryptodev_info info;
668  crypto_dev_t *dev;
669  crypto_resource_t *res;
670  clib_error_t *error;
671  u32 i;
672  u16 max_res_idx, res_idx, j;
673  u8 drv_id;
674 
675  vec_validate_init_empty (dcm->dev, rte_cryptodev_count () - 1,
677 
678  for (i = 0; i < rte_cryptodev_count (); i++)
679  {
680  dev = vec_elt_at_index (dcm->dev, i);
681 
682  cryptodev = &rte_cryptodevs[i];
683  rte_cryptodev_info_get (i, &info);
684 
685  dev->id = i;
686  dev->name = cryptodev->data->name;
687  dev->numa = rte_cryptodev_socket_id (i);
688  dev->features = info.feature_flags;
689  dev->max_qp = info.max_nb_queue_pairs;
690  drv_id = info.driver_id;
691  if (drv_id >= vec_len (dcm->drv))
692  vec_validate_init_empty (dcm->drv, drv_id,
693  (crypto_drv_t) EMPTY_STRUCT);
694  vec_elt_at_index (dcm->drv, drv_id)->name = info.driver_name;
695  dev->drv_id = drv_id;
696  vec_add1 (vec_elt_at_index (dcm->drv, drv_id)->devs, i);
697 
698  if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
699  continue;
700 
701  if ((error = crypto_dev_conf (i, dev->max_qp, dev->numa)))
702  {
703  clib_error_report (error);
704  continue;
705  }
706 
707  max_res_idx = (dev->max_qp / 2) - 1;
708 
709  vec_validate (dev->free_resources, max_res_idx);
710 
711  res_idx = vec_len (dcm->resource);
712  vec_validate_init_empty_aligned (dcm->resource, res_idx + max_res_idx,
713  (crypto_resource_t) EMPTY_STRUCT,
715 
716  for (j = 0; j <= max_res_idx; j++, res_idx++)
717  {
718  vec_elt (dev->free_resources, max_res_idx - j) = res_idx;
719  res = &dcm->resource[res_idx];
720  res->dev_id = i;
721  res->drv_id = drv_id;
722  res->qp_id = j * 2;
723  res->numa = dev->numa;
724  res->thread_idx = (u16) ~ 0;
725  }
726 
727  crypto_parse_capabilities (dev, info.capabilities, n_mains);
728  }
729 }
730 
731 void
733 {
735  crypto_resource_t *res;
737  crypto_dev_t *dev;
738  u32 thread_idx, skip_master;
739  u16 res_idx, *idx;
740  u8 used;
741  u16 i;
742 
743  skip_master = vlib_num_workers () > 0;
744 
745  /* *INDENT-OFF* */
746  vec_foreach (dev, dcm->dev)
747  {
748  vec_foreach_index (thread_idx, dcm->workers_main)
749  {
750  if (vec_len (dev->free_resources) == 0)
751  break;
752 
753  if (thread_idx < skip_master)
754  continue;
755 
756  /* Check thread is not already using the device */
757  vec_foreach (idx, dev->used_resources)
758  if (dcm->resource[idx[0]].thread_idx == thread_idx)
759  continue;
760 
761  cwm = vec_elt_at_index (dcm->workers_main, thread_idx);
762 
763  used = 0;
764  res_idx = vec_pop (dev->free_resources);
765 
766  /* Set device only for supported algos */
767  for (i = 0; i < IPSEC_CRYPTO_N_ALG; i++)
768  if (dev->cipher_support[i] &&
769  cwm->cipher_resource_idx[i] == (u16) ~0)
770  {
771  dcm->cipher_algs[i].disabled--;
772  cwm->cipher_resource_idx[i] = res_idx;
773  used = 1;
774  }
775 
776  for (i = 0; i < IPSEC_INTEG_N_ALG; i++)
777  if (dev->auth_support[i] &&
778  cwm->auth_resource_idx[i] == (u16) ~0)
779  {
780  dcm->auth_algs[i].disabled--;
781  cwm->auth_resource_idx[i] = res_idx;
782  used = 1;
783  }
784 
785  if (!used)
786  {
787  vec_add1 (dev->free_resources, res_idx);
788  continue;
789  }
790 
791  vec_add1 (dev->used_resources, res_idx);
792 
793  res = vec_elt_at_index (dcm->resource, res_idx);
794 
795  ASSERT (res->thread_idx == (u16) ~0);
796  res->thread_idx = thread_idx;
797 
798  /* Add device to vector of polling resources */
799  vec_add1 (cwm->resource_idx, res_idx);
800  }
801  }
802  /* *INDENT-ON* */
803 }
804 
805 static void
806 crypto_op_init (struct rte_mempool *mempool,
807  void *_arg __attribute__ ((unused)),
808  void *_obj, unsigned i __attribute__ ((unused)))
809 {
810  struct rte_crypto_op *op = _obj;
811 
812  op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
813  op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
814  op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
815  op->phys_addr = rte_mempool_virt2iova (_obj);
816  op->mempool = mempool;
817 }
818 
819 static clib_error_t *
821 {
824  crypto_data_t *data;
825  u8 *pool_name;
826  u32 pool_priv_size = sizeof (struct rte_crypto_op_pool_private);
827  struct rte_crypto_op_pool_private *priv;
828  struct rte_mempool *mp;
829 
830  data = vec_elt_at_index (dcm->data, numa);
831 
832  /* Already allocated */
833  if (data->crypto_op)
834  return NULL;
835 
836  pool_name = format (0, "crypto_pool_numa%u%c", numa, 0);
837 
838  mp =
839  rte_mempool_create ((char *) pool_name,
840  conf->num_mbufs,
841  crypto_op_len (), 512, pool_priv_size, NULL, NULL,
842  crypto_op_init, NULL, numa, 0);
843 
844  vec_free (pool_name);
845 
846  if (!mp)
847  return clib_error_return (0, "failed to create crypto op mempool");
848 
849  /* Initialize mempool private data */
850  priv = rte_mempool_get_priv (mp);
851  priv->priv_size = pool_priv_size;
852  priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
853 
854  data->crypto_op = mp;
855 
856  return NULL;
857 }
858 
859 static clib_error_t *
861 {
863  crypto_data_t *data;
864  u8 *pool_name;
865  struct rte_mempool *mp;
866  u32 elt_size;
867 
868  data = vec_elt_at_index (dcm->data, numa);
869 
870  if (data->session_h)
871  return NULL;
872 
873  pool_name = format (0, "session_h_pool_numa%u%c", numa, 0);
874 
875 
876  elt_size = rte_cryptodev_sym_get_header_session_size ();
877 
878  mp =
879  rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
880  elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
881 
882  vec_free (pool_name);
883 
884  if (!mp)
885  return clib_error_return (0, "failed to create crypto session mempool");
886 
887  data->session_h = mp;
888 
889  return NULL;
890 }
891 
892 static clib_error_t *
894 {
896  crypto_data_t *data;
897  u8 *pool_name;
898  struct rte_mempool *mp;
899  u32 elt_size;
900  u8 numa = dev->numa;
901 
902  data = vec_elt_at_index (dcm->data, numa);
903 
904  vec_validate (data->session_drv, dev->drv_id);
905  vec_validate (data->session_drv_failed, dev->drv_id);
908 
909  if (data->session_drv[dev->drv_id])
910  return NULL;
911 
912  pool_name = format (0, "session_drv%u_pool_numa%u%c", dev->drv_id, numa, 0);
913 
914  elt_size = rte_cryptodev_sym_get_private_session_size (dev->id);
915  mp =
916  rte_mempool_create ((char *) pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
917  elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
918 
919  vec_free (pool_name);
920 
921  if (!mp)
922  return clib_error_return (0, "failed to create session drv mempool");
923 
924  data->session_drv[dev->drv_id] = mp;
925  clib_spinlock_init (&data->lockp);
926 
927  return NULL;
928 }
929 
930 static clib_error_t *
932 {
934  clib_error_t *error = NULL;
935  crypto_dev_t *dev;
936 
937  /* *INDENT-OFF* */
938  vec_foreach (dev, dcm->dev)
939  {
941 
942  error = crypto_create_crypto_op_pool (vm, dev->numa);
943  if (error)
944  return error;
945 
946  error = crypto_create_session_h_pool (vm, dev->numa);
947  if (error)
948  return error;
949 
950  error = crypto_create_session_drv_pool (vm, dev);
951  if (error)
952  return error;
953  }
954  /* *INDENT-ON* */
955 
956  return NULL;
957 }
958 
959 static void
961 {
963  crypto_data_t *data;
964  u8 i;
965 
966  dcm->enabled = 0;
967 
968  /* *INDENT-OFF* */
969  vec_foreach (data, dcm->data)
970  {
971  rte_mempool_free (data->crypto_op);
972  rte_mempool_free (data->session_h);
973 
974  vec_foreach_index (i, data->session_drv)
975  rte_mempool_free (data->session_drv[i]);
976 
977  vec_free (data->session_drv);
978  clib_spinlock_free (&data->lockp);
979  }
980  /* *INDENT-ON* */
981 
982  vec_free (dcm->data);
983  vec_free (dcm->workers_main);
984  vec_free (dcm->dev);
985  vec_free (dcm->resource);
986  vec_free (dcm->cipher_algs);
987  vec_free (dcm->auth_algs);
988 }
989 
990 static uword
992  vlib_frame_t * f)
993 {
994  ipsec_main_t *im = &ipsec_main;
998  clib_error_t *error = NULL;
999  u32 i, skip_master, n_mains;
1000 
1001  n_mains = tm->n_vlib_mains;
1002  skip_master = vlib_num_workers () > 0;
1003 
1004  algos_init (n_mains - skip_master);
1005 
1006  crypto_scan_devs (n_mains - skip_master);
1007 
1008  if (!(dcm->enabled))
1009  {
1010  clib_warning ("not enough DPDK crypto resources, default to OpenSSL");
1011  crypto_disable ();
1012  return 0;
1013  }
1014 
1015  dcm->session_timeout = 10e9;
1016 
1017  vec_validate_init_empty_aligned (dcm->workers_main, n_mains - 1,
1020 
1021  /* *INDENT-OFF* */
1022  vec_foreach (cwm, dcm->workers_main)
1023  {
1026  clib_memset (cwm->cipher_resource_idx, ~0,
1027  IPSEC_CRYPTO_N_ALG * sizeof(*cwm->cipher_resource_idx));
1028  clib_memset (cwm->auth_resource_idx, ~0,
1029  IPSEC_INTEG_N_ALG * sizeof(*cwm->auth_resource_idx));
1030  }
1031  /* *INDENT-ON* */
1032 
1034 
1035  error = crypto_create_pools (vm);
1036  if (error)
1037  {
1038  clib_error_report (error);
1039  crypto_disable ();
1040  return 0;
1041  }
1042 
1043 
1044  u32 idx = ipsec_register_esp_backend (vm, im, "dpdk backend",
1045  "dpdk-esp4-encrypt",
1046  "dpdk-esp4-decrypt",
1047  "dpdk-esp6-encrypt",
1048  "dpdk-esp6-decrypt",
1051  int rv = ipsec_select_esp_backend (im, idx);
1052  ASSERT (rv == 0);
1053 
1054  vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "dpdk-crypto-input");
1055  ASSERT (node);
1056  for (i = skip_master; i < n_mains; i++)
1057  vlib_node_set_state (vlib_mains[i], node->index, VLIB_NODE_STATE_POLLING);
1058  return 0;
1059 }
1060 
1061 /* *INDENT-OFF* */
1062 VLIB_REGISTER_NODE (dpdk_ipsec_process_node,static) = {
1063  .function = dpdk_ipsec_process,
1064  .type = VLIB_NODE_TYPE_PROCESS,
1065  .name = "dpdk-ipsec-process",
1066  .process_log2_n_stack_bytes = 17,
1067 };
1068 /* *INDENT-ON* */
1069 
1070 /*
1071  * fd.io coding-style-patch-verification: ON
1072  *
1073  * Local Variables:
1074  * eval: (c-set-style "gnu")
1075  * End:
1076  */
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
Definition: vec.h:439
u32 alg
Definition: ipsec.h:80
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
Definition: hash.h:255
#define hash_unset(h, key)
Definition: hash.h:261
a
Definition: bitmap.h:538
u8 numa
Definition: ipsec.h:96
static u8 cipher_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:175
unsigned long u64
Definition: types.h:89
#define NULL
Definition: clib.h:58
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
u32 index
Definition: node.h:304
u64 session_h_failed
Definition: ipsec.h:147
#define DPDK_CRYPTO_NB_SESS_OBJS
Definition: ipsec.c:627
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
Definition: lock.h:98
ipsec_integ_alg_t integ_alg
Definition: ipsec.h:129
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
Definition: vec.h:525
static clib_error_t * crypto_create_session_drv_pool(vlib_main_t *vm, crypto_dev_t *dev)
Definition: ipsec.c:893
static u64 clib_cpu_time_now(void)
Definition: time.h:75
#define EMPTY_STRUCT
Definition: ipsec.c:26
static_always_inline u32 crypto_op_get_priv_offset(void)
Definition: ipsec.h:188
int i
struct rte_cryptodev_sym_session * session
Definition: ipsec.h:127
u8 disabled
Definition: ipsec.h:85
u8 * format(u8 *s, const char *fmt,...)
Definition: format.c:419
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:450
static void crypto_set_cipher_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:269
vlib_main_t ** vlib_mains
Definition: buffer.c:310
static clib_error_t * crypto_create_session_h_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:860
unsigned char u8
Definition: types.h:56
#define vec_pop(V)
Returns last element of a vector and decrements its length.
Definition: vec.h:619
u8 crypto_key[128]
Definition: ipsec.h:127
static crypto_alg_t * cipher_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 key_len)
Definition: ipsec.c:191
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
static void clib_spinlock_free(clib_spinlock_t *p)
Definition: lock.h:64
static clib_error_t * add_del_sa_session(u32 sa_index, u8 is_add)
Definition: ipsec.c:480
#define clib_memcpy(d, s, n)
Definition: string.h:180
u16 * resource_idx
Definition: ipsec.h:68
#define AES_GCM_TYPE
u8 integ_key[128]
Definition: ipsec.h:131
u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:70
static void algos_init(u32 n_mains)
Definition: ipsec.c:29
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:24
dpdk_config_main_t dpdk_config_main
Definition: init.c:44
ipsec_main_t ipsec_main
Definition: ipsec.c:30
u8 resources
Definition: ipsec.h:86
u8 use_esn
Definition: ipsec.h:133
int ipsec_select_esp_backend(ipsec_main_t *im, u32 backend_idx)
Definition: ipsec.c:693
crypto_drv_t * drv
Definition: ipsec.h:161
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 drv_id
Definition: ipsec.h:95
#define clib_error_return(e, args...)
Definition: error.h:99
static void crypto_parse_capabilities(crypto_dev_t *dev, const struct rte_cryptodev_capabilities *cap, u32 n_mains)
Definition: ipsec.c:576
u16 * free_resources
Definition: ipsec.h:91
unsigned int u32
Definition: types.h:88
#define vec_end(v)
End (last data address) of vector.
crypto_alg_t * auth_algs
Definition: ipsec.h:159
#define VLIB_FRAME_SIZE
Definition: node.h:401
u8 * format_ipsec_crypto_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:58
static clib_error_t * crypto_dev_conf(u8 dev, u16 n_qp, u8 numa)
Definition: ipsec.c:630
u8 trunc_size
Definition: ipsec.h:83
static_always_inline void add_session_by_drv_and_sa_idx(struct rte_cryptodev_sym_session *session, crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:207
static void crypto_disable(void)
Definition: ipsec.c:960
static void clib_spinlock_init(clib_spinlock_t *p)
Definition: lock.h:57
clib_spinlock_t lockp
Definition: ipsec.h:150
static void * get_session_private_data(const struct rte_cryptodev_sym_session *sess, uint8_t driver_id)
Definition: ipsec.c:417
u16 * used_resources
Definition: ipsec.h:92
#define hash_get(h, key)
Definition: hash.h:249
#define pool_elt_at_index(p, i)
Returns pointer to element at given index.
Definition: pool.h:511
crypto_alg_t * cipher_algs
Definition: ipsec.h:158
u32 salt
Definition: ipsec.h:143
unsigned short u16
Definition: types.h:57
u64 session_timeout
Definition: ipsec.h:162
static_always_inline u32 crypto_op_len(void)
Definition: ipsec.h:178
#define AES_GCM_ALG
#define foreach_ipsec_integ_alg
Definition: ipsec.h:96
static clib_error_t * crypto_create_crypto_op_pool(vlib_main_t *vm, u8 numa)
Definition: ipsec.c:820
static u8 auth_alg_index(const crypto_alg_t *alg)
Definition: ipsec.c:183
struct rte_mempool ** session_drv
Definition: ipsec.h:143
u8 len
Definition: ip_types.api:49
vlib node functions
#define foreach_ipsec_crypto_alg
Definition: ipsec.h:74
#define VLIB_REGISTER_NODE(x,...)
Definition: node.h:169
crypto_session_by_drv_t * session_by_drv_id_and_sa_index
Definition: ipsec.h:149
svmdb_client_t * c
vlib_main_t * vm
Definition: buffer.c:301
#define vec_free(V)
Free vector&#39;s memory (no header).
Definition: vec.h:341
#define clib_warning(format, args...)
Definition: error.h:59
static u64 unix_time_now_nsec(void)
Definition: time.h:255
crypto_session_disposal_t * session_disposal
Definition: ipsec.h:144
static void crypto_set_aead_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:241
vlib_node_t * vlib_get_node_by_name(vlib_main_t *vm, u8 *name)
Definition: node.c:45
u16 id
Definition: ipsec.h:97
static void set_session_private_data(struct rte_cryptodev_sym_session *sess, uint8_t driver_id, void *private_data)
Definition: ipsec.c:425
u8 iv_len
Definition: ipsec.h:82
signed int i32
Definition: types.h:77
u32 ipsec_register_esp_backend(vlib_main_t *vm, ipsec_main_t *im, const char *name, const char *esp4_encrypt_node_name, const char *esp4_decrypt_node_name, const char *esp6_encrypt_node_name, const char *esp6_decrypt_node_name, check_support_cb_t esp_check_support_cb, add_del_sa_sess_cb_t esp_add_del_sa_sess_cb)
Definition: ipsec.c:644
#define ASSERT(truth)
#define vec_delete(V, N, M)
Delete N elements starting at element M.
Definition: vec.h:788
struct rte_mempool * crypto_op
Definition: ipsec.h:141
ipsec_sa_t * sad
Definition: ipsec.h:353
static clib_error_t * dpdk_crypto_session_disposal(crypto_session_disposal_t *v, u64 ts)
Definition: ipsec.c:432
crypto_worker_main_t * workers_main
Definition: ipsec.h:155
#define clib_error_report(e)
Definition: error.h:113
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
Definition: node_funcs.h:148
crypto_resource_t * resource
Definition: ipsec.h:157
#define vec_elt(v, i)
Get vector value at index i.
u8 * format_ipsec_integ_alg(u8 *s, va_list *args)
Definition: ipsec_format.c:90
u8 crypto_key_len
Definition: ipsec.h:126
static clib_error_t * crypto_create_pools(vlib_main_t *vm)
Definition: ipsec.c:931
struct rte_mempool * session_h
Definition: ipsec.h:142
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
void crypto_auto_placement(void)
Definition: ipsec.c:732
u64 * session_drv_failed
Definition: ipsec.h:148
u64 uword
Definition: types.h:112
uword * session_by_sa_index
Definition: ipsec.h:145
static void crypto_scan_devs(u32 n_mains)
Definition: ipsec.c:663
static void crypto_op_init(struct rte_mempool *mempool, void *_arg, void *_obj, unsigned i)
Definition: ipsec.c:806
struct rte_crypto_op ** ops
Definition: ipsec.h:69
clib_error_t * create_sym_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.c:319
crypto_dev_t * dev
Definition: ipsec.h:156
enum rte_crypto_sym_xform_type type
Definition: ipsec.h:79
#define DPDK_CRYPTO_N_QUEUE_DESC
Definition: ipsec.c:626
crypto_data_t * data
Definition: ipsec.h:160
static u32 random_u32(u32 *seed)
32-bit random number generator
Definition: random.h:69
ipsec_crypto_alg_t crypto_alg
Definition: ipsec.h:125
static vlib_thread_main_t * vlib_get_thread_main()
Definition: global_funcs.h:32
static u32 vlib_num_workers()
Definition: threads.h:366
static uword dpdk_ipsec_process(vlib_main_t *vm, vlib_node_runtime_t *rt, vlib_frame_t *f)
Definition: ipsec.c:991
u16 auth_resource_idx[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:71
u8 auth_support[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:94
#define vec_validate_init_empty_aligned(V, I, INIT, A)
Make sure vector is long enough for given index and initialize empty space (no header, alignment alignment)
Definition: vec.h:501
#define vec_foreach(var, vec)
Vector iterator.
static_always_inline struct rte_cryptodev_sym_session * get_session_by_drv_and_sa_idx(crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:219
u8 key_len
Definition: ipsec.h:81
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
Definition: vec.h:488
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
Definition: lock.h:82
static void crypto_set_auth_xform(struct rte_crypto_sym_xform *xform, ipsec_sa_t *sa, u8 is_outbound)
Definition: ipsec.c:295
static clib_error_t * dpdk_ipsec_check_support(ipsec_sa_t *sa)
Definition: ipsec.c:543
u8 cipher_support[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:93
static void clear_and_free_obj(void *obj)
Definition: ipsec.c:406
static crypto_alg_t * auth_cap_to_alg(const struct rte_cryptodev_capabilities *cap, u8 trunc_size)
Definition: ipsec.c:219