24 #include <dpdk/ipsec/ipsec.h> 28 #define EMPTY_STRUCT {0} 29 #define NUM_CRYPTO_MBUFS 16384 41 dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].name = str; \ 42 dcm->cipher_algs[IPSEC_CRYPTO_ALG_##f].disabled = n_mains; 49 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
50 a->alg = RTE_CRYPTO_CIPHER_NULL;
55 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_128];
56 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
57 a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
62 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_192];
63 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
64 a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
69 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CBC_256];
70 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
71 a->alg = RTE_CRYPTO_CIPHER_AES_CBC;
76 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_128];
77 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
78 a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
83 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_192];
84 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
85 a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
90 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_CTR_256];
91 a->
type = RTE_CRYPTO_SYM_XFORM_CIPHER;
92 a->alg = RTE_CRYPTO_CIPHER_AES_CTR;
97 #define AES_GCM_TYPE RTE_CRYPTO_SYM_XFORM_AEAD 98 #define AES_GCM_ALG RTE_CRYPTO_AEAD_AES_GCM 100 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_128];
108 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_192];
116 a = &dcm->
cipher_algs[IPSEC_CRYPTO_ALG_AES_GCM_256];
128 dcm->auth_algs[IPSEC_INTEG_ALG_##f].name = str; \ 129 dcm->auth_algs[IPSEC_INTEG_ALG_##f].disabled = n_mains; 134 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_NONE];
135 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
136 a->alg = RTE_CRYPTO_AUTH_NULL;
140 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_MD5_96];
141 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
142 a->alg = RTE_CRYPTO_AUTH_MD5_HMAC;
146 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA1_96];
147 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
148 a->alg = RTE_CRYPTO_AUTH_SHA1_HMAC;
152 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA_256_96];
153 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
154 a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
158 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA_256_128];
159 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
160 a->alg = RTE_CRYPTO_AUTH_SHA256_HMAC;
164 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA_384_192];
165 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
166 a->alg = RTE_CRYPTO_AUTH_SHA384_HMAC;
170 a = &dcm->
auth_algs[IPSEC_INTEG_ALG_SHA_512_256];
171 a->
type = RTE_CRYPTO_SYM_XFORM_AUTH;
172 a->alg = RTE_CRYPTO_AUTH_SHA512_HMAC;
199 if (cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
205 if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
206 (alg->
type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
207 (cap->sym.cipher.algo == alg->
alg) &&
210 if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
211 (alg->
type == RTE_CRYPTO_SYM_XFORM_AEAD) &&
212 (cap->sym.aead.algo == alg->
alg) &&
227 if ((cap->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC) ||
228 (cap->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH))
234 if ((cap->sym.auth.algo == alg->
alg) &&
252 ASSERT (c->
type == RTE_CRYPTO_SYM_XFORM_AEAD);
254 xform->type = RTE_CRYPTO_SYM_XFORM_AEAD;
255 xform->aead.algo = c->
alg;
257 xform->aead.key.length = c->
key_len;
258 xform->aead.iv.offset =
260 xform->aead.iv.length = 12;
262 xform->aead.aad_length = ipsec_sa_is_set_USE_ESN (sa) ? 12 : 8;
266 xform->aead.op = RTE_CRYPTO_AEAD_OP_ENCRYPT;
268 xform->aead.op = RTE_CRYPTO_AEAD_OP_DECRYPT;
280 ASSERT (c->
type == RTE_CRYPTO_SYM_XFORM_CIPHER);
282 xform->type = RTE_CRYPTO_SYM_XFORM_CIPHER;
283 xform->cipher.algo = c->
alg;
285 xform->cipher.key.length = c->
key_len;
286 xform->cipher.iv.offset =
288 xform->cipher.iv.length = c->
iv_len;
292 xform->cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
294 xform->cipher.op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
306 ASSERT (a->
type == RTE_CRYPTO_SYM_XFORM_AUTH);
308 xform->type = RTE_CRYPTO_SYM_XFORM_AUTH;
309 xform->auth.algo = a->
alg;
311 xform->auth.key.length = a->
key_len;
316 xform->auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
318 xform->auth.op = RTE_CRYPTO_AUTH_OP_VERIFY;
330 struct rte_crypto_sym_xform cipher_xform = { 0 };
331 struct rte_crypto_sym_xform auth_xform = { 0 };
332 struct rte_crypto_sym_xform *xfs;
333 struct rte_cryptodev_sym_session **s;
338 if ((sa->
crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) |
339 (sa->
crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) |
340 (sa->
crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256))
352 cipher_xform.next = &auth_xform;
357 auth_xform.next = &cipher_xform;
374 session[0] = rte_cryptodev_sym_session_create (data->
session_h);
386 struct rte_mempool **mp;
391 rte_cryptodev_sym_session_init (res->
dev_id, session[0], xfs, mp[0]);
409 struct rte_mempool *mp = rte_mempool_from_obj (obj);
413 rte_mempool_put (mp, obj);
421 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0) 422 return sess->sess_private_data[driver_id];
424 if (unlikely (sess->nb_drivers <= driver_id))
427 return sess->sess_data[driver_id].data;
434 uint8_t driver_id,
void *private_data)
436 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0) 437 sess->sess_private_data[driver_id] = private_data;
439 if (unlikely (sess->nb_drivers <= driver_id))
441 sess->sess_data[driver_id].data = private_data;
477 if (rte_mempool_from_obj(s->
session))
479 ret = rte_cryptodev_sym_session_free (s->
session);
498 struct rte_cryptodev_sym_session *s;
512 s = (
struct rte_cryptodev_sym_session *) val[0];
543 if (sa->
integ_alg == IPSEC_INTEG_ALG_NONE)
546 case IPSEC_CRYPTO_ALG_NONE:
547 case IPSEC_CRYPTO_ALG_AES_GCM_128:
548 case IPSEC_CRYPTO_ALG_AES_GCM_192:
549 case IPSEC_CRYPTO_ALG_AES_GCM_256:
558 if (sa->
crypto_alg != IPSEC_CRYPTO_ALG_NONE &&
564 if (sa->
integ_alg != IPSEC_INTEG_ALG_NONE &&
573 const struct rte_cryptodev_capabilities *cap,
580 for (; cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; cap++)
583 switch (cap->sym.xform_type)
585 case RTE_CRYPTO_SYM_XFORM_AEAD:
586 case RTE_CRYPTO_SYM_XFORM_CIPHER:
587 inc = cap->sym.cipher.key_size.increment;
589 for (len = cap->sym.cipher.key_size.min;
590 len <= cap->sym.cipher.key_size.max; len += inc)
601 case RTE_CRYPTO_SYM_XFORM_AUTH:
602 inc = cap->sym.auth.digest_size.increment;
604 for (len = cap->sym.auth.digest_size.min;
605 len <= cap->sym.auth.digest_size.max; len += inc)
625 struct rte_cryptodev_config dev_conf = { 0 };
626 struct rte_cryptodev_qp_conf qp_conf = { 0 };
631 dev_conf.socket_id = numa;
632 dev_conf.nb_queue_pairs = n_qp;
634 error_str =
"failed to configure crypto device %u";
635 ret = rte_cryptodev_configure (dev, &dev_conf);
639 error_str =
"failed to setup crypto device %u queue pair %u";
641 for (qp = 0; qp < n_qp; qp++)
643 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0) 644 ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa, NULL);
646 ret = rte_cryptodev_queue_pair_setup (dev, qp, &qp_conf, numa);
652 error_str =
"failed to start crypto device %u";
653 if (rte_cryptodev_start (dev))
663 struct rte_cryptodev *cryptodev;
664 struct rte_cryptodev_info info = { 0 };
669 u16 max_res_idx, res_idx, j;
675 for (i = 0; i < rte_cryptodev_count (); i++)
679 cryptodev = &rte_cryptodevs[
i];
680 rte_cryptodev_info_get (i, &info);
683 dev->
name = cryptodev->data->name;
684 dev->
numa = rte_cryptodev_socket_id (i);
686 dev->
max_qp = info.max_nb_queue_pairs;
687 drv_id = info.driver_id;
695 if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
704 max_res_idx = dev->
max_qp - 1;
713 for (j = 0; j <= max_res_idx; j++)
750 if (thread_idx < skip_master)
804 void *_arg __attribute__ ((unused)),
805 void *_obj,
unsigned i __attribute__ ((unused)))
807 struct rte_crypto_op *op = _obj;
809 op->sess_type = RTE_CRYPTO_OP_WITH_SESSION;
810 op->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
811 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
812 op->phys_addr = rte_mempool_virt2iova (_obj);
813 op->mempool = mempool;
823 u32 pool_priv_size =
sizeof (
struct rte_crypto_op_pool_private);
824 struct rte_crypto_op_pool_private *priv;
825 struct rte_mempool *mp;
833 pool_name =
format (0,
"crypto_pool_numa%u%c", numa, 0);
848 priv = rte_mempool_get_priv (mp);
849 priv->priv_size = pool_priv_size;
850 priv->type = RTE_CRYPTO_OP_TYPE_SYMMETRIC;
863 struct rte_mempool *mp;
871 pool_name =
format (0,
"session_h_pool_numa%u%c", numa, 0);
874 elt_size = rte_cryptodev_sym_get_header_session_size ();
876 #if RTE_VERSION < RTE_VERSION_NUM(19, 2, 0, 0) 878 elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
881 mp = rte_cryptodev_sym_session_pool_create ((
char *) pool_name,
883 elt_size, 512, 0, numa);
901 struct rte_mempool *mp;
915 pool_name =
format (0,
"session_drv%u_pool_numa%u%c", dev->
drv_id, numa, 0);
917 elt_size = rte_cryptodev_sym_get_private_session_size (dev->
id);
920 elt_size, 512, 0, NULL, NULL, NULL, NULL, numa, 0);
1004 for (i = skip_master; i < n_mains; i++)
1006 VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED);
1019 u32 skip_master, n_mains;
1031 "not enough DPDK crypto resources");
1065 vm, im,
"dpdk backend",
"dpdk-esp4-encrypt",
"dpdk-esp4-encrypt-tun",
1066 "dpdk-esp4-decrypt",
"dpdk-esp4-decrypt",
"dpdk-esp6-encrypt",
1067 "dpdk-esp6-encrypt-tun",
"dpdk-esp6-decrypt",
"dpdk-esp6-decrypt",
#define vec_validate(V, I)
Make sure vector is long enough for given index (no header, unspecified alignment) ...
#define vec_foreach_index(var, v)
Iterate over vector indices.
#define hash_set(h, key, value)
#define hash_unset(h, key)
#define VLIB_MAIN_LOOP_ENTER_FUNCTION(x)
clib_memset(h->entries, 0, sizeof(h->entries[0]) *entries)
#define foreach_ipsec_crypto_alg
static_always_inline void clib_spinlock_unlock_if_init(clib_spinlock_t *p)
ipsec_integ_alg_t integ_alg
#define vec_add1(V, E)
Add 1 element to end of vector (unspecified alignment).
#define foreach_ipsec_integ_alg
struct rte_cryptodev_sym_session * session
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
#define vec_pop(V)
Returns last element of a vector and decrements its length.
#define vec_reset_length(v)
Reset vector length to zero NULL-pointer tolerant.
static void clib_spinlock_free(clib_spinlock_t *p)
u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG]
dpdk_config_main_t dpdk_config_main
description fragment has unexpected format
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
#define clib_error_return(e, args...)
#define vec_end(v)
End (last data address) of vector.
static void clib_spinlock_init(clib_spinlock_t *p)
vlib_node_t * vlib_get_node_by_name(vlib_main_t *vm, u8 *name)
crypto_alg_t * cipher_algs
vlib_main_t * vm
X-connect all packets from the HOST to the PHY.
struct rte_mempool ** session_drv
crypto_session_by_drv_t * session_by_drv_id_and_sa_index
sll srl srl sll sra u16x4 i
#define vec_free(V)
Free vector's memory (no header).
vnet_interface_main_t * im
static u64 unix_time_now_nsec(void)
crypto_session_disposal_t * session_disposal
u8 data[IPSEC_KEY_MAX_LEN]
#define vec_delete(V, N, M)
Delete N elements starting at element M.
struct rte_mempool * crypto_op
vlib_log_class_t log_default
crypto_worker_main_t * workers_main
#define clib_error_report(e)
u32 ipsec_register_esp_backend(vlib_main_t *vm, ipsec_main_t *im, const char *name, const char *esp4_encrypt_node_name, const char *esp4_encrypt_node_tun_name, const char *esp4_decrypt_node_name, const char *esp4_decrypt_tun_node_name, const char *esp6_encrypt_node_name, const char *esp6_encrypt_node_tun_name, const char *esp6_decrypt_node_name, const char *esp6_decrypt_tun_node_name, const char *esp_mpls_encrypt_node_tun_name, check_support_cb_t esp_check_support_cb, add_del_sa_sess_cb_t esp_add_del_sa_sess_cb, enable_disable_cb_t enable_disable_cb)
static void vlib_node_set_state(vlib_main_t *vm, u32 node_index, vlib_node_state_t new_state)
Set node dispatch state.
crypto_resource_t * resource
static vlib_main_t * vlib_get_main(void)
#define vec_elt(v, i)
Get vector value at index i.
struct rte_mempool * session_h
static ipsec_sa_t * ipsec_sa_get(u32 sa_index)
#define vec_len(v)
Number of elements in vector (rvalue-only, NULL tolerant)
vlib_main_t vlib_node_runtime_t * node
uword * session_by_sa_index
struct rte_crypto_op ** ops
int ipsec_select_esp_backend(ipsec_main_t *im, u32 backend_idx)
enum rte_crypto_sym_xform_type type
ipsec_crypto_alg_t crypto_alg
static vlib_thread_main_t * vlib_get_thread_main()
static u32 vlib_num_workers()
u16 auth_resource_idx[IPSEC_INTEG_N_ALG]
u8 auth_support[IPSEC_INTEG_N_ALG]
#define vec_validate_init_empty_aligned(V, I, INIT, A)
Make sure vector is long enough for given index and initialize empty space (no header, alignment alignment)
#define vec_foreach(var, vec)
Vector iterator.
#define vec_validate_init_empty(V, I, INIT)
Make sure vector is long enough for given index and initialize empty space (no header, unspecified alignment)
#define CLIB_CACHE_LINE_BYTES
static_always_inline void clib_spinlock_lock_if_init(clib_spinlock_t *p)
#define vlib_log_warn(...)
u8 cipher_support[IPSEC_CRYPTO_N_ALG]