FD.io VPP  v20.01-46-g7006026de
Vector Packet Processing
aes_gcm.c
Go to the documentation of this file.
1 /*
2  *------------------------------------------------------------------
3  * Copyright (c) 2019 Cisco and/or its affiliates.
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at:
7  *
8  * http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  *------------------------------------------------------------------
16  */
17 
18 #include <vlib/vlib.h>
19 #include <vnet/plugin/plugin.h>
20 #include <vnet/crypto/crypto.h>
21 #include <x86intrin.h>
23 #include <crypto_ia32/aesni.h>
24 #include <crypto_ia32/ghash.h>
25 
26 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
27 #pragma GCC optimize ("O3")
28 #endif
29 
30 typedef struct
31 {
32  /* pre-calculated hash key values */
33  const __m128i Hi[8];
34  /* extracted AES key */
35  const __m128i Ke[15];
37 
38 static const __m128i last_byte_one = { 0, 1ULL << 56 };
39 static const __m128i zero = { 0, 0 };
40 
41 static const u8x16 bswap_mask = {
42  15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
43 };
44 
45 static const u8x16 byte_mask_scale = {
46  0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
47 };
48 
50 aesni_gcm_bswap (__m128i x)
51 {
52  return _mm_shuffle_epi8 (x, (__m128i) bswap_mask);
53 }
54 
56 aesni_gcm_byte_mask (__m128i x, u8 n_bytes)
57 {
58  u8x16 mask = u8x16_is_greater (u8x16_splat (n_bytes), byte_mask_scale);
59 
60  return _mm_blendv_epi8 (zero, x, (__m128i) mask);
61 }
62 
64 aesni_gcm_load_partial (__m128i * p, int n_bytes)
65 {
66  ASSERT (n_bytes <= 16);
67 #ifdef __AVX512F__
68  return _mm_mask_loadu_epi8 (zero, (1 << n_bytes) - 1, p);
69 #else
70  return aesni_gcm_byte_mask (CLIB_MEM_OVERFLOW_LOAD (_mm_loadu_si128, p),
71  n_bytes);
72 #endif
73 }
74 
76 aesni_gcm_store_partial (void *p, __m128i r, int n_bytes)
77 {
78 #ifdef __AVX512F__
79  _mm_mask_storeu_epi8 (p, (1 << n_bytes) - 1, r);
80 #else
81  u8x16 mask = u8x16_is_greater (u8x16_splat (n_bytes), byte_mask_scale);
82  _mm_maskmoveu_si128 (r, (__m128i) mask, p);
83 #endif
84 }
85 
87 aesni_gcm_load (__m128i * d, __m128i * inv, int n, int n_bytes)
88 {
89  for (int i = 0; i < n - 1; i++)
90  d[i] = _mm_loadu_si128 (inv + i);
91  d[n - 1] = n_bytes ? aesni_gcm_load_partial (inv + n - 1, n_bytes) :
92  _mm_loadu_si128 (inv + n - 1);
93 }
94 
96 aesni_gcm_store (__m128i * d, __m128i * outv, int n, int n_bytes)
97 {
98  for (int i = 0; i < n - 1; i++)
99  _mm_storeu_si128 (outv + i, d[i]);
100  if (n_bytes & 0xf)
101  aesni_gcm_store_partial (outv + n - 1, d[n - 1], n_bytes);
102  else
103  _mm_storeu_si128 (outv + n - 1, d[n - 1]);
104 }
105 
107 aesni_gcm_enc_first_round (__m128i * r, __m128i * Y, u32 * ctr, __m128i k,
108  int n_blocks)
109 {
110  u32 i;
111 
112  if (PREDICT_TRUE ((u8) ctr[0] < (256 - n_blocks)))
113  {
114  for (i = 0; i < n_blocks; i++)
115  {
116  Y[0] = _mm_add_epi32 (Y[0], last_byte_one);
117  r[i] = k ^ Y[0];
118  }
119  ctr[0] += n_blocks;
120  }
121  else
122  {
123  for (i = 0; i < n_blocks; i++)
124  {
125  Y[0] = _mm_insert_epi32 (Y[0], clib_host_to_net_u32 (++ctr[0]), 3);
126  r[i] = k ^ Y[0];
127  }
128  }
129 }
130 
132 aesni_gcm_enc_round (__m128i * r, __m128i k, int n_blocks)
133 {
134  for (int i = 0; i < n_blocks; i++)
135  r[i] = _mm_aesenc_si128 (r[i], k);
136 }
137 
139 aesni_gcm_enc_last_round (__m128i * r, __m128i * d, const __m128i * k,
140  int rounds, int n_blocks)
141 {
142 
143  /* additional ronuds for AES-192 and AES-256 */
144  for (int i = 10; i < rounds; i++)
145  aesni_gcm_enc_round (r, k[i], n_blocks);
146 
147  for (int i = 0; i < n_blocks; i++)
148  d[i] ^= _mm_aesenclast_si128 (r[i], k[rounds]);
149 }
150 
151 static_always_inline __m128i
153  const __m128i * in, int n_blocks)
154 {
155  ghash_data_t _gd, *gd = &_gd;
156  const __m128i *Hi = kd->Hi + n_blocks - 1;
157  ghash_mul_first (gd, aesni_gcm_bswap (_mm_loadu_si128 (in)) ^ T, Hi[0]);
158  for (int i = 1; i < n_blocks; i++)
159  ghash_mul_next (gd, aesni_gcm_bswap (_mm_loadu_si128 (in + i)), Hi[-i]);
160  ghash_reduce (gd);
161  ghash_reduce2 (gd);
162  return ghash_final (gd);
163 }
164 
165 static_always_inline __m128i
166 aesni_gcm_ghash (__m128i T, aes_gcm_key_data_t * kd, const __m128i * in,
167  u32 n_left)
168 {
169 
170  while (n_left >= 128)
171  {
172  T = aesni_gcm_ghash_blocks (T, kd, in, 8);
173  n_left -= 128;
174  in += 8;
175  }
176 
177  if (n_left >= 64)
178  {
179  T = aesni_gcm_ghash_blocks (T, kd, in, 4);
180  n_left -= 64;
181  in += 4;
182  }
183 
184  if (n_left >= 32)
185  {
186  T = aesni_gcm_ghash_blocks (T, kd, in, 2);
187  n_left -= 32;
188  in += 2;
189  }
190 
191  if (n_left >= 16)
192  {
193  T = aesni_gcm_ghash_blocks (T, kd, in, 1);
194  n_left -= 16;
195  in += 1;
196  }
197 
198  if (n_left)
199  {
200  __m128i r = aesni_gcm_load_partial ((__m128i *) in, n_left);
201  T = ghash_mul (aesni_gcm_bswap (r) ^ T, kd->Hi[0]);
202  }
203  return T;
204 }
205 
206 static_always_inline __m128i
207 aesni_gcm_calc (__m128i T, aes_gcm_key_data_t * kd, __m128i * d,
208  __m128i * Y, u32 * ctr, __m128i * inv, __m128i * outv,
209  int rounds, int n, int last_block_bytes, int with_ghash,
210  int is_encrypt)
211 {
212  __m128i r[n];
213  ghash_data_t _gd = { }, *gd = &_gd;
214  const __m128i *k = kd->Ke;
215  int hidx = is_encrypt ? 4 : n, didx = 0;
216 
217  _mm_prefetch (inv + 4, _MM_HINT_T0);
218 
219  /* AES rounds 0 and 1 */
220  aesni_gcm_enc_first_round (r, Y, ctr, k[0], n);
221  aesni_gcm_enc_round (r, k[1], n);
222 
223  /* load data - decrypt round */
224  if (is_encrypt == 0)
225  aesni_gcm_load (d, inv, n, last_block_bytes);
226 
227  /* GHASH multiply block 1 */
228  if (with_ghash)
229  ghash_mul_first (gd, aesni_gcm_bswap (d[didx++]) ^ T, kd->Hi[--hidx]);
230 
231  /* AES rounds 2 and 3 */
232  aesni_gcm_enc_round (r, k[2], n);
233  aesni_gcm_enc_round (r, k[3], n);
234 
235  /* GHASH multiply block 2 */
236  if (with_ghash && hidx)
237  ghash_mul_next (gd, aesni_gcm_bswap (d[didx++]), kd->Hi[--hidx]);
238 
239  /* AES rounds 4 and 5 */
240  aesni_gcm_enc_round (r, k[4], n);
241  aesni_gcm_enc_round (r, k[5], n);
242 
243  /* GHASH multiply block 3 */
244  if (with_ghash && hidx)
245  ghash_mul_next (gd, aesni_gcm_bswap (d[didx++]), kd->Hi[--hidx]);
246 
247  /* AES rounds 6 and 7 */
248  aesni_gcm_enc_round (r, k[6], n);
249  aesni_gcm_enc_round (r, k[7], n);
250 
251  /* GHASH multiply block 4 */
252  if (with_ghash && hidx)
253  ghash_mul_next (gd, aesni_gcm_bswap (d[didx++]), kd->Hi[--hidx]);
254 
255  /* AES rounds 8 and 9 */
256  aesni_gcm_enc_round (r, k[8], n);
257  aesni_gcm_enc_round (r, k[9], n);
258 
259  /* GHASH reduce 1st step */
260  if (with_ghash)
261  ghash_reduce (gd);
262 
263  /* load data - encrypt round */
264  if (is_encrypt)
265  aesni_gcm_load (d, inv, n, last_block_bytes);
266 
267  /* GHASH reduce 2nd step */
268  if (with_ghash)
269  ghash_reduce2 (gd);
270 
271  /* AES last round(s) */
272  aesni_gcm_enc_last_round (r, d, k, rounds, n);
273 
274  /* store data */
275  aesni_gcm_store (d, outv, n, last_block_bytes);
276 
277  /* GHASH final step */
278  if (with_ghash)
279  T = ghash_final (gd);
280 
281  return T;
282 }
283 
284 static_always_inline __m128i
285 aesni_gcm_calc_double (__m128i T, aes_gcm_key_data_t * kd, __m128i * d,
286  __m128i * Y, u32 * ctr, __m128i * inv, __m128i * outv,
287  int rounds, int is_encrypt)
288 {
289  __m128i r[4];
290  ghash_data_t _gd, *gd = &_gd;
291  const __m128i *k = kd->Ke;
292 
293  /* AES rounds 0 and 1 */
294  aesni_gcm_enc_first_round (r, Y, ctr, k[0], 4);
295  aesni_gcm_enc_round (r, k[1], 4);
296 
297  /* load 4 blocks of data - decrypt round */
298  if (is_encrypt == 0)
299  aesni_gcm_load (d, inv, 4, 0);
300 
301  /* GHASH multiply block 0 */
302  ghash_mul_first (gd, aesni_gcm_bswap (d[0]) ^ T, kd->Hi[7]);
303 
304  /* AES rounds 2 and 3 */
305  aesni_gcm_enc_round (r, k[2], 4);
306  aesni_gcm_enc_round (r, k[3], 4);
307 
308  /* GHASH multiply block 1 */
309  ghash_mul_next (gd, aesni_gcm_bswap (d[1]), kd->Hi[6]);
310 
311  /* AES rounds 4 and 5 */
312  aesni_gcm_enc_round (r, k[4], 4);
313  aesni_gcm_enc_round (r, k[5], 4);
314 
315  /* GHASH multiply block 2 */
316  ghash_mul_next (gd, aesni_gcm_bswap (d[2]), kd->Hi[5]);
317 
318  /* AES rounds 6 and 7 */
319  aesni_gcm_enc_round (r, k[6], 4);
320  aesni_gcm_enc_round (r, k[7], 4);
321 
322  /* GHASH multiply block 3 */
323  ghash_mul_next (gd, aesni_gcm_bswap (d[3]), kd->Hi[4]);
324 
325  /* AES rounds 8 and 9 */
326  aesni_gcm_enc_round (r, k[8], 4);
327  aesni_gcm_enc_round (r, k[9], 4);
328 
329  /* load 4 blocks of data - encrypt round */
330  if (is_encrypt)
331  aesni_gcm_load (d, inv, 4, 0);
332 
333  /* AES last round(s) */
334  aesni_gcm_enc_last_round (r, d, k, rounds, 4);
335 
336  /* store 4 blocks of data */
337  aesni_gcm_store (d, outv, 4, 0);
338 
339  /* load next 4 blocks of data data - decrypt round */
340  if (is_encrypt == 0)
341  aesni_gcm_load (d, inv + 4, 4, 0);
342 
343  /* GHASH multiply block 4 */
344  ghash_mul_next (gd, aesni_gcm_bswap (d[0]), kd->Hi[3]);
345 
346  /* AES rounds 0, 1 and 2 */
347  aesni_gcm_enc_first_round (r, Y, ctr, k[0], 4);
348  aesni_gcm_enc_round (r, k[1], 4);
349  aesni_gcm_enc_round (r, k[2], 4);
350 
351  /* GHASH multiply block 5 */
352  ghash_mul_next (gd, aesni_gcm_bswap (d[1]), kd->Hi[2]);
353 
354  /* AES rounds 3 and 4 */
355  aesni_gcm_enc_round (r, k[3], 4);
356  aesni_gcm_enc_round (r, k[4], 4);
357 
358  /* GHASH multiply block 6 */
359  ghash_mul_next (gd, aesni_gcm_bswap (d[2]), kd->Hi[1]);
360 
361  /* AES rounds 5 and 6 */
362  aesni_gcm_enc_round (r, k[5], 4);
363  aesni_gcm_enc_round (r, k[6], 4);
364 
365  /* GHASH multiply block 7 */
366  ghash_mul_next (gd, aesni_gcm_bswap (d[3]), kd->Hi[0]);
367 
368  /* AES rounds 7 and 8 */
369  aesni_gcm_enc_round (r, k[7], 4);
370  aesni_gcm_enc_round (r, k[8], 4);
371 
372  /* GHASH reduce 1st step */
373  ghash_reduce (gd);
374 
375  /* AES round 9 */
376  aesni_gcm_enc_round (r, k[9], 4);
377 
378  /* load data - encrypt round */
379  if (is_encrypt)
380  aesni_gcm_load (d, inv + 4, 4, 0);
381 
382  /* GHASH reduce 2nd step */
383  ghash_reduce2 (gd);
384 
385  /* AES last round(s) */
386  aesni_gcm_enc_last_round (r, d, k, rounds, 4);
387 
388  /* store data */
389  aesni_gcm_store (d, outv + 4, 4, 0);
390 
391  /* GHASH final step */
392  return ghash_final (gd);
393 }
394 
395 static_always_inline __m128i
396 aesni_gcm_ghash_last (__m128i T, aes_gcm_key_data_t * kd, __m128i * d,
397  int n_blocks, int n_bytes)
398 {
399  ghash_data_t _gd, *gd = &_gd;
400 
401  if (n_bytes)
402  d[n_blocks - 1] = aesni_gcm_byte_mask (d[n_blocks - 1], n_bytes);
403 
404  ghash_mul_first (gd, aesni_gcm_bswap (d[0]) ^ T, kd->Hi[n_blocks - 1]);
405  if (n_blocks > 1)
406  ghash_mul_next (gd, aesni_gcm_bswap (d[1]), kd->Hi[n_blocks - 2]);
407  if (n_blocks > 2)
408  ghash_mul_next (gd, aesni_gcm_bswap (d[2]), kd->Hi[n_blocks - 3]);
409  if (n_blocks > 3)
410  ghash_mul_next (gd, aesni_gcm_bswap (d[3]), kd->Hi[n_blocks - 4]);
411  ghash_reduce (gd);
412  ghash_reduce2 (gd);
413  return ghash_final (gd);
414 }
415 
416 
417 static_always_inline __m128i
418 aesni_gcm_enc (__m128i T, aes_gcm_key_data_t * kd, __m128i Y, const u8 * in,
419  const u8 * out, u32 n_left, int rounds)
420 {
421  __m128i *inv = (__m128i *) in, *outv = (__m128i *) out;
422  __m128i d[4];
423  u32 ctr = 1;
424 
425  if (n_left == 0)
426  return T;
427 
428  if (n_left < 64)
429  {
430  if (n_left > 48)
431  {
432  n_left &= 0x0f;
433  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, n_left,
434  /* with_ghash */ 0, /* is_encrypt */ 1);
435  return aesni_gcm_ghash_last (T, kd, d, 4, n_left);
436  }
437  else if (n_left > 32)
438  {
439  n_left &= 0x0f;
440  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 3, n_left,
441  /* with_ghash */ 0, /* is_encrypt */ 1);
442  return aesni_gcm_ghash_last (T, kd, d, 3, n_left);
443  }
444  else if (n_left > 16)
445  {
446  n_left &= 0x0f;
447  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 2, n_left,
448  /* with_ghash */ 0, /* is_encrypt */ 1);
449  return aesni_gcm_ghash_last (T, kd, d, 2, n_left);
450  }
451  else
452  {
453  n_left &= 0x0f;
454  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 1, n_left,
455  /* with_ghash */ 0, /* is_encrypt */ 1);
456  return aesni_gcm_ghash_last (T, kd, d, 1, n_left);
457  }
458  }
459 
460  aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, 0,
461  /* with_ghash */ 0, /* is_encrypt */ 1);
462 
463  /* next */
464  n_left -= 64;
465  outv += 4;
466  inv += 4;
467 
468  while (n_left >= 128)
469  {
470  T = aesni_gcm_calc_double (T, kd, d, &Y, &ctr, inv, outv, rounds,
471  /* is_encrypt */ 1);
472 
473  /* next */
474  n_left -= 128;
475  outv += 8;
476  inv += 8;
477  }
478 
479  if (n_left >= 64)
480  {
481  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, 0,
482  /* with_ghash */ 1, /* is_encrypt */ 1);
483 
484  /* next */
485  n_left -= 64;
486  outv += 4;
487  inv += 4;
488  }
489 
490  if (n_left == 0)
491  return aesni_gcm_ghash_last (T, kd, d, 4, 0);
492 
493  if (n_left > 48)
494  {
495  n_left &= 0x0f;
496  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, n_left,
497  /* with_ghash */ 1, /* is_encrypt */ 1);
498  return aesni_gcm_ghash_last (T, kd, d, 4, n_left);
499  }
500 
501  if (n_left > 32)
502  {
503  n_left &= 0x0f;
504  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 3, n_left,
505  /* with_ghash */ 1, /* is_encrypt */ 1);
506  return aesni_gcm_ghash_last (T, kd, d, 3, n_left);
507  }
508 
509  if (n_left > 16)
510  {
511  n_left &= 0x0f;
512  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 2, n_left,
513  /* with_ghash */ 1, /* is_encrypt */ 1);
514  return aesni_gcm_ghash_last (T, kd, d, 2, n_left);
515  }
516 
517  n_left &= 0x0f;
518  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 1, n_left,
519  /* with_ghash */ 1, /* is_encrypt */ 1);
520  return aesni_gcm_ghash_last (T, kd, d, 1, n_left);
521 }
522 
523 static_always_inline __m128i
524 aesni_gcm_dec (__m128i T, aes_gcm_key_data_t * kd, __m128i Y, const u8 * in,
525  const u8 * out, u32 n_left, int rounds)
526 {
527  __m128i *inv = (__m128i *) in, *outv = (__m128i *) out;
528  __m128i d[8];
529  u32 ctr = 1;
530 
531  while (n_left >= 128)
532  {
533  T = aesni_gcm_calc_double (T, kd, d, &Y, &ctr, inv, outv, rounds,
534  /* is_encrypt */ 0);
535 
536  /* next */
537  n_left -= 128;
538  outv += 8;
539  inv += 8;
540  }
541 
542  if (n_left >= 64)
543  {
544  T = aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4, 0, 1, 0);
545 
546  /* next */
547  n_left -= 64;
548  outv += 4;
549  inv += 4;
550  }
551 
552  if (n_left == 0)
553  return T;
554 
555  if (n_left > 48)
556  return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 4,
557  n_left - 48,
558  /* with_ghash */ 1, /* is_encrypt */ 0);
559 
560  if (n_left > 32)
561  return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 3,
562  n_left - 32,
563  /* with_ghash */ 1, /* is_encrypt */ 0);
564 
565  if (n_left > 16)
566  return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 2,
567  n_left - 16,
568  /* with_ghash */ 1, /* is_encrypt */ 0);
569 
570  return aesni_gcm_calc (T, kd, d, &Y, &ctr, inv, outv, rounds, 1, n_left,
571  /* with_ghash */ 1, /* is_encrypt */ 0);
572 }
573 
575 aes_gcm (const u8 * in, u8 * out, const u8 * addt, const u8 * iv, u8 * tag,
576  u32 data_bytes, u32 aad_bytes, u8 tag_len, aes_gcm_key_data_t * kd,
577  int aes_rounds, int is_encrypt)
578 {
579  int i;
580  __m128i r, Y0, T = { };
581  ghash_data_t _gd, *gd = &_gd;
582 
583  _mm_prefetch (iv, _MM_HINT_T0);
584  _mm_prefetch (in, _MM_HINT_T0);
585  _mm_prefetch (in + CLIB_CACHE_LINE_BYTES, _MM_HINT_T0);
586 
587  /* calculate ghash for AAD - optimized for ipsec common cases */
588  if (aad_bytes == 8)
589  T = aesni_gcm_ghash (T, kd, (__m128i *) addt, 8);
590  else if (aad_bytes == 12)
591  T = aesni_gcm_ghash (T, kd, (__m128i *) addt, 12);
592  else
593  T = aesni_gcm_ghash (T, kd, (__m128i *) addt, aad_bytes);
594 
595  /* initalize counter */
596  Y0 = CLIB_MEM_OVERFLOW_LOAD (_mm_loadu_si128, (__m128i *) iv);
597  Y0 = _mm_insert_epi32 (Y0, clib_host_to_net_u32 (1), 3);
598 
599  /* ghash and encrypt/edcrypt */
600  if (is_encrypt)
601  T = aesni_gcm_enc (T, kd, Y0, in, out, data_bytes, aes_rounds);
602  else
603  T = aesni_gcm_dec (T, kd, Y0, in, out, data_bytes, aes_rounds);
604 
605  _mm_prefetch (tag, _MM_HINT_T0);
606 
607  /* Finalize ghash */
608  r[0] = data_bytes;
609  r[1] = aad_bytes;
610 
611  /* bytes to bits */
612  r <<= 3;
613 
614  /* interleaved computation of final ghash and E(Y0, k) */
615  ghash_mul_first (gd, r ^ T, kd->Hi[0]);
616  r = kd->Ke[0] ^ Y0;
617  for (i = 1; i < 5; i += 1)
618  r = _mm_aesenc_si128 (r, kd->Ke[i]);
619  ghash_reduce (gd);
620  ghash_reduce2 (gd);
621  for (; i < 9; i += 1)
622  r = _mm_aesenc_si128 (r, kd->Ke[i]);
623  T = ghash_final (gd);
624  for (; i < aes_rounds; i += 1)
625  r = _mm_aesenc_si128 (r, kd->Ke[i]);
626  r = _mm_aesenclast_si128 (r, kd->Ke[aes_rounds]);
627  T = aesni_gcm_bswap (T) ^ r;
628 
629  /* tag_len 16 -> 0 */
630  tag_len &= 0xf;
631 
632  if (is_encrypt)
633  {
634  /* store tag */
635  if (tag_len)
636  aesni_gcm_store_partial ((__m128i *) tag, T, (1 << tag_len) - 1);
637  else
638  _mm_storeu_si128 ((__m128i *) tag, T);
639  }
640  else
641  {
642  /* check tag */
643  u16 tag_mask = tag_len ? (1 << tag_len) - 1 : 0xffff;
644  r = _mm_loadu_si128 ((__m128i *) tag);
645  if (_mm_movemask_epi8 (r == T) != tag_mask)
646  return 0;
647  }
648  return 1;
649 }
650 
653  u32 n_ops, aesni_key_size_t ks)
654 {
656  vnet_crypto_op_t *op = ops[0];
657  aes_gcm_key_data_t *kd;
658  u32 n_left = n_ops;
659 
660 
661 next:
662  kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
663  aes_gcm (op->src, op->dst, op->aad, op->iv, op->tag, op->len, op->aad_len,
664  op->tag_len, kd, AESNI_KEY_ROUNDS (ks), /* is_encrypt */ 1);
665  op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
666 
667  if (--n_left)
668  {
669  op += 1;
670  goto next;
671  }
672 
673  return n_ops;
674 }
675 
678  u32 n_ops, aesni_key_size_t ks)
679 {
681  vnet_crypto_op_t *op = ops[0];
682  aes_gcm_key_data_t *kd;
683  u32 n_left = n_ops;
684  int rv;
685 
686 next:
687  kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
688  rv = aes_gcm (op->src, op->dst, op->aad, op->iv, op->tag, op->len,
689  op->aad_len, op->tag_len, kd, AESNI_KEY_ROUNDS (ks),
690  /* is_encrypt */ 0);
691 
692  if (rv)
693  {
694  op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
695  }
696  else
697  {
698  op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
699  n_ops--;
700  }
701 
702  if (--n_left)
703  {
704  op += 1;
705  goto next;
706  }
707 
708  return n_ops;
709 }
710 
713 {
714  aes_gcm_key_data_t *kd;
715  __m128i H;
716  int i;
717 
718  kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
719 
720  /* expand AES key */
721  aes_key_expand ((__m128i *) kd->Ke, key->data, ks);
722 
723  /* pre-calculate H */
724  H = kd->Ke[0];
725  for (i = 1; i < AESNI_KEY_ROUNDS (ks); i += 1)
726  H = _mm_aesenc_si128 (H, kd->Ke[i]);
727  H = _mm_aesenclast_si128 (H, kd->Ke[i]);
728  H = aesni_gcm_bswap (H);
729  ghash_precompute (H, (__m128i *) kd->Hi, 8);
730  return kd;
731 }
732 
733 #define foreach_aesni_gcm_handler_type _(128) _(192) _(256)
734 
735 #define _(x) \
736 static u32 aesni_ops_dec_aes_gcm_##x \
737 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
738 { return aesni_ops_dec_aes_gcm (vm, ops, n_ops, AESNI_KEY_##x); } \
739 static u32 aesni_ops_enc_aes_gcm_##x \
740 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
741 { return aesni_ops_enc_aes_gcm (vm, ops, n_ops, AESNI_KEY_##x); } \
742 static void * aesni_gcm_key_exp_##x (vnet_crypto_key_t *key) \
743 { return aesni_gcm_key_exp (key, AESNI_KEY_##x); }
744 
746 #undef _
747 
748 clib_error_t *
749 #ifdef __AVX512F__
751 #elif __AVX2__
753 #else
755 #endif
756 {
758 
759 #define _(x) \
760  vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
761  VNET_CRYPTO_OP_AES_##x##_GCM_ENC, \
762  aesni_ops_enc_aes_gcm_##x); \
763  vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
764  VNET_CRYPTO_OP_AES_##x##_GCM_DEC, \
765  aesni_ops_dec_aes_gcm_##x); \
766  cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_GCM] = aesni_gcm_key_exp_##x;
768 #undef _
769  return 0;
770 }
771 
772 /*
773  * fd.io coding-style-patch-verification: ON
774  *
775  * Local Variables:
776  * eval: (c-set-style "gnu")
777  * End:
778  */
crypto_ia32_main_t crypto_ia32_main
Definition: main.c:23
static_always_inline __m128i aesni_gcm_ghash(__m128i T, aes_gcm_key_data_t *kd, const __m128i *in, u32 n_left)
Definition: aes_gcm.c:166
static_always_inline void aesni_gcm_store_partial(void *p, __m128i r, int n_bytes)
Definition: aes_gcm.c:76
static const __m128i last_byte_one
Definition: aes_gcm.c:38
#define PREDICT_TRUE(x)
Definition: clib.h:112
static_always_inline void aesni_gcm_load(__m128i *d, __m128i *inv, int n, int n_bytes)
Definition: aes_gcm.c:87
static_always_inline void aesni_gcm_store(__m128i *d, __m128i *outv, int n, int n_bytes)
Definition: aes_gcm.c:96
static_always_inline void aesni_gcm_enc_round(__m128i *r, __m128i k, int n_blocks)
Definition: aes_gcm.c:132
static_always_inline __m128i aesni_gcm_calc(__m128i T, aes_gcm_key_data_t *kd, __m128i *d, __m128i *Y, u32 *ctr, __m128i *inv, __m128i *outv, int rounds, int n, int last_block_bytes, int with_ghash, int is_encrypt)
Definition: aes_gcm.c:207
int i
clib_error_t * crypto_ia32_aesni_gcm_init_sse42(vlib_main_t *vm)
Definition: aes_gcm.c:754
#define AESNI_KEY_ROUNDS(x)
Definition: aesni.h:28
static const u8x16 bswap_mask
Definition: aes_gcm.c:41
static_always_inline void aes_key_expand(__m128i *k, u8 *key, aesni_key_size_t ks)
Definition: aesni.h:181
unsigned char u8
Definition: types.h:56
static_always_inline void ghash_precompute(__m128i H, __m128i *Hi, int count)
Definition: ghash.h:226
const __m128i Hi[8]
Definition: aes_gcm.c:33
#define static_always_inline
Definition: clib.h:99
static const u8x16 byte_mask_scale
Definition: aes_gcm.c:45
aesni_key_size_t
Definition: aesni.h:21
static_always_inline u32 aesni_ops_enc_aes_gcm(vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, aesni_key_size_t ks)
Definition: aes_gcm.c:652
static_always_inline void ghash_reduce(ghash_data_t *gd)
Definition: ghash.h:176
static_always_inline int aes_gcm(const u8 *in, u8 *out, const u8 *addt, const u8 *iv, u8 *tag, u32 data_bytes, u32 aad_bytes, u8 tag_len, aes_gcm_key_data_t *kd, int aes_rounds, int is_encrypt)
Definition: aes_gcm.c:575
unsigned int u32
Definition: types.h:88
static_always_inline void ghash_reduce2(ghash_data_t *gd)
Definition: ghash.h:202
static_always_inline __m128i aesni_gcm_ghash_blocks(__m128i T, aes_gcm_key_data_t *kd, const __m128i *in, int n_blocks)
Definition: aes_gcm.c:152
vnet_crypto_main_t * cm
Definition: quic_crypto.c:41
static u8 iv[]
Definition: aes_cbc.c:24
const __m128i Ke[15]
Definition: aes_gcm.c:35
static const __m128i zero
Definition: aes_gcm.c:39
unsigned short u16
Definition: types.h:57
static_always_inline __m128i aesni_gcm_load_partial(__m128i *p, int n_bytes)
Definition: aes_gcm.c:64
clib_error_t * crypto_ia32_aesni_gcm_init_avx512(vlib_main_t *vm)
static_always_inline __m128i aesni_gcm_bswap(__m128i x)
Definition: aes_gcm.c:50
static_always_inline __m128i ghash_final(ghash_data_t *gd)
Definition: ghash.h:209
static_always_inline __m128i aesni_gcm_calc_double(__m128i T, aes_gcm_key_data_t *kd, __m128i *d, __m128i *Y, u32 *ctr, __m128i *inv, __m128i *outv, int rounds, int is_encrypt)
Definition: aes_gcm.c:285
static_always_inline __m128i aesni_gcm_enc(__m128i T, aes_gcm_key_data_t *kd, __m128i Y, const u8 *in, const u8 *out, u32 n_left, int rounds)
Definition: aes_gcm.c:418
#define ASSERT(truth)
static_always_inline void aesni_gcm_enc_last_round(__m128i *r, __m128i *d, const __m128i *k, int rounds, int n_blocks)
Definition: aes_gcm.c:139
static_always_inline __m128i aesni_gcm_byte_mask(__m128i x, u8 n_bytes)
Definition: aes_gcm.c:56
static_always_inline u8x16 u8x16_is_greater(u8x16 v1, u8x16 v2)
Definition: vector_sse42.h:729
typedef key
Definition: ipsec_types.api:83
static_always_inline __m128i aesni_gcm_dec(__m128i T, aes_gcm_key_data_t *kd, __m128i Y, const u8 *in, const u8 *out, u32 n_left, int rounds)
Definition: aes_gcm.c:524
clib_error_t * crypto_ia32_aesni_gcm_init_avx2(vlib_main_t *vm)
static_always_inline u32 aesni_ops_dec_aes_gcm(vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops, aesni_key_size_t ks)
Definition: aes_gcm.c:677
#define foreach_aesni_gcm_handler_type
Definition: aes_gcm.c:733
static_always_inline void * aesni_gcm_key_exp(vnet_crypto_key_t *key, aesni_key_size_t ks)
Definition: aes_gcm.c:712
static_always_inline void aesni_gcm_enc_first_round(__m128i *r, __m128i *Y, u32 *ctr, __m128i k, int n_blocks)
Definition: aes_gcm.c:107
vnet_crypto_op_status_t status
Definition: crypto.h:129
static void * clib_mem_alloc_aligned(uword size, uword align)
Definition: mem.h:161
static_always_inline __m128i ghash_mul(__m128i a, __m128i b)
Definition: ghash.h:216
#define CLIB_MEM_OVERFLOW_LOAD(f, src)
Definition: sanitizer.h:49
static_always_inline __m128i aesni_gcm_ghash_last(__m128i T, aes_gcm_key_data_t *kd, __m128i *d, int n_blocks, int n_bytes)
Definition: aes_gcm.c:396
static_always_inline void ghash_mul_next(ghash_data_t *gd, __m128i a, __m128i b)
Definition: ghash.h:145
#define CLIB_CACHE_LINE_BYTES
Definition: cache.h:59
static_always_inline void ghash_mul_first(ghash_data_t *gd, __m128i a, __m128i b)
Definition: ghash.h:129