FD.io VPP  v20.01-9-g24064d02a
Vector Packet Processing
ipsec.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Intel and/or its affiliates.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at:
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 #ifndef __DPDK_IPSEC_H__
16 #define __DPDK_IPSEC_H__
17 
18 #include <vnet/vnet.h>
19 #include <vppinfra/cache.h>
20 #include <vnet/ipsec/ipsec.h>
21 
22 #undef always_inline
23 #include <rte_config.h>
24 #include <rte_crypto.h>
25 #include <rte_cryptodev.h>
26 
27 #if CLIB_DEBUG > 0
28 #define always_inline static inline
29 #else
30 #define always_inline static inline __attribute__ ((__always_inline__))
31 #endif
32 
33 #define DPDK_CRYPTO_N_QUEUE_DESC 2048
34 #define DPDK_CRYPTO_NB_SESS_OBJS 20000
35 
36 #define foreach_dpdk_crypto_input_next \
37  _(DROP, "error-drop") \
38  _(IP4_LOOKUP, "ip4-lookup") \
39  _(IP6_LOOKUP, "ip6-lookup") \
40  _(INTERFACE_OUTPUT, "interface-output") \
41  _(MIDCHAIN, "adj-midchain-tx") \
42  _(DECRYPT4_POST, "dpdk-esp4-decrypt-post") \
43  _(DECRYPT6_POST, "dpdk-esp6-decrypt-post")
44 
45 typedef enum
46 {
47 #define _(f,s) DPDK_CRYPTO_INPUT_NEXT_##f,
49 #undef _
52 
53 #define MAX_QP_PER_LCORE 16
54 
55 typedef struct
56 {
58  u32 iv[2];
61 
62 typedef struct
63 {
67  CLIB_ALIGN_MARK (mark0, 16);
69  u8 aad[16];
70  u8 icv[32]; /* XXX last 16B in next cache line */
72 
73 typedef struct
74 {
76  struct rte_crypto_op **ops;
77  u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG];
78  u16 auth_resource_idx[IPSEC_INTEG_N_ALG];
79  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
81 
82 typedef struct
83 {
84  CLIB_ALIGN_MARK (pad, 8); /* align up to 8 bytes for 32bit builds */
85  char *name;
86  enum rte_crypto_sym_xform_type type;
94 } crypto_alg_t;
95 
96 typedef struct
97 {
100  u8 cipher_support[IPSEC_CRYPTO_N_ALG];
101  u8 auth_support[IPSEC_INTEG_N_ALG];
105  const char *name;
108 } crypto_dev_t;
109 
110 typedef struct
111 {
112  const char *name;
114 } crypto_drv_t;
115 
116 typedef struct
117 {
119  u8 remove;
124  u16 inflights[2];
126  u16 __unused;
127  struct rte_crypto_op *ops[VLIB_FRAME_SIZE];
129  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
131 
132 typedef struct
133 {
135  struct rte_cryptodev_sym_session *session;
137 
138 typedef struct
139 {
140  struct rte_cryptodev_sym_session *session;
142  CLIB_ALIGN_MARK (pad, 16); /* align up to 16 bytes for 32bit builds */
144 
145 typedef struct
146 {
147  struct rte_mempool *crypto_op;
148  struct rte_mempool *session_h;
149  struct rte_mempool **session_drv;
157  /* Required for vec_validate_aligned */
158  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
159 } crypto_data_t;
160 
161 typedef struct
162 {
170  u64 session_timeout; /* nsec */
173 
175 
176 static const u8 pad_data[] =
177  { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0 };
178 
179 void crypto_auto_placement (void);
180 
181 clib_error_t *create_sym_session (struct rte_cryptodev_sym_session **session,
182  u32 sa_idx, crypto_resource_t * res,
184 
187 {
188  const u32 align = 4;
189  u32 op_size =
190  sizeof (struct rte_crypto_op) + sizeof (struct rte_crypto_sym_op);
191 
192  return ((op_size + align - 1) & ~(align - 1)) + sizeof (dpdk_op_priv_t);
193 }
194 
197 {
198  const u32 align = 16;
199  u32 offset;
200 
201  offset = sizeof (struct rte_crypto_op) + sizeof (struct rte_crypto_sym_op);
202  offset = (offset + align - 1) & ~(align - 1);
203 
204  return offset;
205 }
206 
208 crypto_op_get_priv (struct rte_crypto_op * op)
209 {
210  return (dpdk_op_priv_t *) (((u8 *) op) + crypto_op_get_priv_offset ());
211 }
212 
213 
215 add_session_by_drv_and_sa_idx (struct rte_cryptodev_sym_session *session,
216  crypto_data_t * data, u32 drv_id, u32 sa_idx)
217 {
221  sbd = vec_elt_at_index (data->session_by_drv_id_and_sa_index, sa_idx);
222  sbd->dev_mask |= 1L << drv_id;
223  sbd->session = session;
224 }
225 
226 static_always_inline struct rte_cryptodev_sym_session *
228 {
229  crypto_session_by_drv_t *sess_by_sa;
230  if (_vec_len (data->session_by_drv_id_and_sa_index) <= sa_idx)
231  return NULL;
232  sess_by_sa =
234  return (sess_by_sa->dev_mask & (1L << drv_id)) ? sess_by_sa->session : NULL;
235 }
236 
238 crypto_get_session (struct rte_cryptodev_sym_session ** session,
239  u32 sa_idx,
240  crypto_resource_t * res,
242 {
245  struct rte_cryptodev_sym_session *sess;
246 
247  data = vec_elt_at_index (dcm->data, res->numa);
248  sess = get_session_by_drv_and_sa_idx (data, res->drv_id, sa_idx);
249 
250  if (PREDICT_FALSE (!sess))
251  return create_sym_session (session, sa_idx, res, cwm, is_outbound);
252 
253  session[0] = sess;
254 
255  return NULL;
256 }
257 
260 {
261  u16 cipher_res = cwm->cipher_resource_idx[sa->crypto_alg];
262  u16 auth_res = cwm->auth_resource_idx[sa->integ_alg];
263  u8 is_aead;
264 
265  /* Not allowed to setup SA with no-aead-cipher/NULL or NULL/NULL */
266 
267  is_aead = ((sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128) ||
268  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_192) ||
269  (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_256));
270 
271  if (sa->crypto_alg == IPSEC_CRYPTO_ALG_NONE)
272  return auth_res;
273 
274  if (cipher_res == auth_res)
275  return cipher_res;
276 
277  if (is_aead)
278  return cipher_res;
279 
280  return (u16) ~ 0;
281 }
282 
284 crypto_alloc_ops (u8 numa, struct rte_crypto_op ** ops, u32 n)
285 {
287  crypto_data_t *data = vec_elt_at_index (dcm->data, numa);
288  i32 ret;
289 
290  ret = rte_mempool_get_bulk (data->crypto_op, (void **) ops, n);
291 
292  /* *INDENT-OFF* */
293  data->crypto_op_get_failed += ! !ret;
294  /* *INDENT-ON* */
295 
296  return ret;
297 }
298 
300 crypto_free_ops (u8 numa, struct rte_crypto_op **ops, u32 n)
301 {
303  crypto_data_t *data = vec_elt_at_index (dcm->data, numa);
304 
305  if (!n)
306  return;
307 
308  rte_mempool_put_bulk (data->crypto_op, (void **) ops, n);
309 }
310 
313  u32 node_index, u32 error, u8 numa, u8 encrypt)
314 {
316  crypto_resource_t *res;
317  u16 *res_idx;
318 
319  /* *INDENT-OFF* */
320  vec_foreach (res_idx, cwm->resource_idx)
321  {
322  u16 enq, n_ops;
323  res = vec_elt_at_index (dcm->resource, res_idx[0]);
324 
325  if (!res->n_ops)
326  continue;
327 
328  n_ops = (DPDK_CRYPTO_N_QUEUE_DESC / 2) - res->inflights[encrypt];
329  n_ops = res->n_ops < n_ops ? res->n_ops : n_ops;
330  enq = rte_cryptodev_enqueue_burst (res->dev_id, res->qp_id,
331  res->ops, n_ops);
332  ASSERT (n_ops == enq);
333  res->inflights[encrypt] += enq;
334 
335  if (PREDICT_FALSE (enq < res->n_ops))
336  {
337  crypto_free_ops (numa, &res->ops[enq], res->n_ops - enq);
338  vlib_buffer_free (vm, &res->bi[enq], res->n_ops - enq);
339 
340  vlib_node_increment_counter (vm, node_index, error,
341  res->n_ops - enq);
342  }
343  res->n_ops = 0;
344  }
345  /* *INDENT-ON* */
346 }
347 
350 {
351  icb->salt = salt;
352  icb->iv[0] = seq;
353  icb->iv[1] = seq_hi;
354 }
355 
357 crypto_op_setup (u8 is_aead, struct rte_mbuf *mb0,
358  struct rte_crypto_op *op, void *session,
359  u32 cipher_off, u32 cipher_len,
360  u32 auth_off, u32 auth_len,
361  u8 * aad, u8 * digest, u64 digest_paddr)
362 {
363  struct rte_crypto_sym_op *sym_op;
364 
365  sym_op = (struct rte_crypto_sym_op *) (op + 1);
366 
367  sym_op->m_src = mb0;
368  sym_op->session = session;
369 
370  if (is_aead)
371  {
372  sym_op->aead.data.offset = cipher_off;
373  sym_op->aead.data.length = cipher_len;
374 
375  sym_op->aead.aad.data = aad;
376  sym_op->aead.aad.phys_addr =
377  op->phys_addr + (uintptr_t) aad - (uintptr_t) op;
378 
379  sym_op->aead.digest.data = digest;
380  sym_op->aead.digest.phys_addr = digest_paddr;
381  }
382  else
383  {
384  sym_op->cipher.data.offset = cipher_off;
385  sym_op->cipher.data.length = cipher_len;
386 
387  sym_op->auth.data.offset = auth_off;
388  sym_op->auth.data.length = auth_len;
389 
390  sym_op->auth.digest.data = digest;
391  sym_op->auth.digest.phys_addr = digest_paddr;
392  }
393 }
394 
395 #endif /* __DPDK_IPSEC_H__ */
396 
397 /*
398  * fd.io coding-style-patch-verification: ON
399  *
400  * Local Variables:
401  * eval: (c-set-style "gnu")
402  * End:
403  */
u32 alg
Definition: ipsec.h:87
static_always_inline void crypto_op_setup(u8 is_aead, struct rte_mbuf *mb0, struct rte_crypto_op *op, void *session, u32 cipher_off, u32 cipher_len, u32 auth_off, u32 auth_len, u8 *aad, u8 *digest, u64 digest_paddr)
Definition: ipsec.h:357
u8 pad[3]
log2 (size of the packing page block)
Definition: bihash_doc.h:61
#define CLIB_CACHE_LINE_ALIGN_MARK(mark)
Definition: cache.h:60
clib_error_t * create_sym_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.c:321
static void vlib_buffer_free(vlib_main_t *vm, u32 *buffers, u32 n_buffers)
Free buffers Frees the entire buffer chain for each buffer.
Definition: buffer_funcs.h:890
#define foreach_dpdk_crypto_input_next
Definition: ipsec.h:36
unsigned long u64
Definition: types.h:89
u64 crypto_op_get_failed
Definition: ipsec.h:152
#define NULL
Definition: clib.h:58
u64 session_h_failed
Definition: ipsec.h:153
static_always_inline i32 crypto_alloc_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:284
ipsec_integ_alg_t integ_alg
Definition: ipsec_sa.h:152
static_always_inline void crypto_set_icb(dpdk_gcm_cnt_blk *icb, u32 salt, u32 seq, u32 seq_hi)
Definition: ipsec.h:349
static_always_inline u32 crypto_op_get_priv_offset(void)
Definition: ipsec.h:196
struct rte_cryptodev_sym_session * session
Definition: ipsec.h:135
u8 disabled
Definition: ipsec.h:92
#define vec_validate_aligned(V, I, A)
Make sure vector is long enough for given index (no header, specified alignment)
Definition: vec.h:451
unsigned char u8
Definition: types.h:56
dpdk_crypto_main_t dpdk_crypto_main
Definition: ipsec.c:25
u32 bi[VLIB_FRAME_SIZE]
Definition: ipsec.h:128
u16 * devs
Definition: ipsec.h:113
u16 * resource_idx
Definition: ipsec.h:75
#define DPDK_CRYPTO_N_QUEUE_DESC
Definition: ipsec.h:33
static_always_inline clib_error_t * crypto_get_session(struct rte_cryptodev_sym_session **session, u32 sa_idx, crypto_resource_t *res, crypto_worker_main_t *cwm, u8 is_outbound)
Definition: ipsec.h:238
static_always_inline void crypto_free_ops(u8 numa, struct rte_crypto_op **ops, u32 n)
Definition: ipsec.h:300
#define static_always_inline
Definition: clib.h:99
u16 cipher_resource_idx[IPSEC_CRYPTO_N_ALG]
Definition: ipsec.h:77
u8 resources
Definition: ipsec.h:93
static_always_inline void crypto_enqueue_ops(vlib_main_t *vm, crypto_worker_main_t *cwm, u32 node_index, u32 error, u8 numa, u8 encrypt)
Definition: ipsec.h:312
crypto_drv_t * drv
Definition: ipsec.h:169
#define vec_elt_at_index(v, i)
Get vector value at index i checking that i is in bounds.
u8 drv_id
Definition: ipsec.h:102
u16 * free_resources
Definition: ipsec.h:98
u32 salt
unsigned int u32
Definition: types.h:88
crypto_alg_t * auth_algs
Definition: ipsec.h:167
static const u8 pad_data[]
Definition: ipsec.h:176
#define VLIB_FRAME_SIZE
Definition: node.h:378
u32 iv[2]
Definition: ipsec.h:58
u8 trunc_size
Definition: ipsec.h:90
static_always_inline void add_session_by_drv_and_sa_idx(struct rte_cryptodev_sym_session *session, crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:215
u32 next
Definition: ipsec.h:64
struct rte_cryptodev_sym_session * session
Definition: ipsec.h:140
void crypto_auto_placement(void)
Definition: ipsec.c:730
vl_api_fib_path_type_t type
Definition: fib_types.api:123
clib_spinlock_t lockp
Definition: ipsec.h:156
u16 * used_resources
Definition: ipsec.h:99
static u8 iv[]
Definition: aes_cbc.c:24
crypto_alg_t * cipher_algs
Definition: ipsec.h:166
unsigned short u16
Definition: types.h:57
const char * name
Definition: ipsec.h:112
u64 features
Definition: ipsec.h:107
#define PREDICT_FALSE(x)
Definition: clib.h:111
u64 session_timeout
Definition: ipsec.h:170
static_always_inline u32 crypto_op_len(void)
Definition: ipsec.h:186
vlib_main_t * vm
Definition: in2out_ed.c:1810
u8 encrypt
Definition: ipsec.h:66
static void vlib_node_increment_counter(vlib_main_t *vm, u32 node_index, u32 counter_index, u64 increment)
Definition: node_funcs.h:1150
struct rte_mempool ** session_drv
Definition: ipsec.h:149
crypto_session_by_drv_t * session_by_drv_id_and_sa_index
Definition: ipsec.h:155
dpdk_gcm_cnt_blk cb
Definition: ipsec.h:68
u8 is_outbound
Definition: ipsec.api:93
crypto_session_disposal_t * session_disposal
Definition: ipsec.h:150
dpdk_crypto_input_next_t
Definition: ipsec.h:45
u16 id
Definition: ipsec.h:104
u8 iv_len
Definition: ipsec.h:89
signed int i32
Definition: types.h:77
#define ASSERT(truth)
u8 data[128]
Definition: ipsec_types.api:87
struct rte_mempool * crypto_op
Definition: ipsec.h:147
u32 max_qp
Definition: ipsec.h:106
u16 inflights[2]
Definition: ipsec.h:124
crypto_worker_main_t * workers_main
Definition: ipsec.h:163
u8 boundary
Definition: ipsec.h:91
crypto_resource_t * resource
Definition: ipsec.h:165
struct rte_mempool * session_h
Definition: ipsec.h:148
char * name
Definition: ipsec.h:85
#define CLIB_ALIGN_MARK(name, alignment)
Definition: clib.h:85
u64 * session_drv_failed
Definition: ipsec.h:154
u64 uword
Definition: types.h:112
uword * session_by_sa_index
Definition: ipsec.h:151
const char * name
Definition: ipsec.h:105
struct rte_crypto_op ** ops
Definition: ipsec.h:76
struct clib_bihash_value offset
template key/value backing page structure
crypto_dev_t * dev
Definition: ipsec.h:164
static_always_inline u16 get_resource(crypto_worker_main_t *cwm, ipsec_sa_t *sa)
Definition: ipsec.h:259
crypto_data_t * data
Definition: ipsec.h:168
ipsec_crypto_alg_t crypto_alg
Definition: ipsec_sa.h:148
u16 auth_resource_idx[IPSEC_INTEG_N_ALG]
Definition: ipsec.h:78
#define vec_foreach(var, vec)
Vector iterator.
static_always_inline struct rte_cryptodev_sym_session * get_session_by_drv_and_sa_idx(crypto_data_t *data, u32 drv_id, u32 sa_idx)
Definition: ipsec.h:227
u8 key_len
Definition: ipsec.h:88
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
struct rte_crypto_op * ops[VLIB_FRAME_SIZE]
Definition: ipsec.h:127
static_always_inline dpdk_op_priv_t * crypto_op_get_priv(struct rte_crypto_op *op)
Definition: ipsec.h:208