Skip to content

Commit 4860620

Browse files
Ard Biesheuvelherbertx
Ard Biesheuvel
authored and
herbertx
committed
crypto: arm64/aes - add NEON/Crypto Extensions CBCMAC/CMAC/XCBC driver
On ARMv8 implementations that do not support the Crypto Extensions, such as the Raspberry Pi 3, the CCM driver falls back to the generic table based AES implementation to perform the MAC part of the algorithm, which is slow and not time invariant. So add a CBCMAC implementation to the shared glue code between NEON AES and Crypto Extensions AES, so that it can be used instead now that the CCM driver has been updated to look for CBCMAC implementations other than the one it supplies itself. Also, given how these algorithms mostly only differ in the way the key handling and the final encryption are implemented, expose CMAC and XCBC algorithms as well based on the same core update code. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
1 parent f15f05b commit 4860620

File tree

2 files changed

+267
-2
lines changed

2 files changed

+267
-2
lines changed

‎arch/arm64/crypto/aes-glue.c

Lines changed: 239 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/*
22
* linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
33
*
4-
* Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
4+
* Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
55
*
66
* This program is free software; you can redistribute it and/or modify
77
* it under the terms of the GNU General Public License version 2 as
@@ -11,6 +11,7 @@
1111
#include <asm/neon.h>
1212
#include <asm/hwcap.h>
1313
#include <crypto/aes.h>
14+
#include <crypto/internal/hash.h>
1415
#include <crypto/internal/simd.h>
1516
#include <crypto/internal/skcipher.h>
1617
#include <linux/module.h>
@@ -31,6 +32,7 @@
3132
#define aes_ctr_encrypt ce_aes_ctr_encrypt
3233
#define aes_xts_encrypt ce_aes_xts_encrypt
3334
#define aes_xts_decrypt ce_aes_xts_decrypt
35+
#define aes_mac_update ce_aes_mac_update
3436
MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
3537
#else
3638
#define MODE "neon"
@@ -44,11 +46,15 @@ MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
4446
#define aes_ctr_encrypt neon_aes_ctr_encrypt
4547
#define aes_xts_encrypt neon_aes_xts_encrypt
4648
#define aes_xts_decrypt neon_aes_xts_decrypt
49+
#define aes_mac_update neon_aes_mac_update
4750
MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
4851
MODULE_ALIAS_CRYPTO("ecb(aes)");
4952
MODULE_ALIAS_CRYPTO("cbc(aes)");
5053
MODULE_ALIAS_CRYPTO("ctr(aes)");
5154
MODULE_ALIAS_CRYPTO("xts(aes)");
55+
MODULE_ALIAS_CRYPTO("cmac(aes)");
56+
MODULE_ALIAS_CRYPTO("xcbc(aes)");
57+
MODULE_ALIAS_CRYPTO("cbcmac(aes)");
5258
#endif
5359

5460
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
@@ -75,11 +81,25 @@ asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
7581
int rounds, int blocks, u8 const rk2[], u8 iv[],
7682
int first);
7783

84+
asmlinkage void aes_mac_update(u8 const in[], u32 const rk[], int rounds,
85+
int blocks, u8 dg[], int enc_before,
86+
int enc_after);
87+
7888
struct crypto_aes_xts_ctx {
7989
struct crypto_aes_ctx key1;
8090
struct crypto_aes_ctx __aligned(8) key2;
8191
};
8292

93+
struct mac_tfm_ctx {
94+
struct crypto_aes_ctx key;
95+
u8 __aligned(8) consts[];
96+
};
97+
98+
struct mac_desc_ctx {
99+
unsigned int len;
100+
u8 dg[AES_BLOCK_SIZE];
101+
};
102+
83103
static int skcipher_aes_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
84104
unsigned int key_len)
85105
{
@@ -357,6 +377,217 @@ static struct skcipher_alg aes_algs[] = { {
357377
.decrypt = xts_decrypt,
358378
} };
359379

380+
static int cbcmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
381+
unsigned int key_len)
382+
{
383+
struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
384+
int err;
385+
386+
err = aes_expandkey(&ctx->key, in_key, key_len);
387+
if (err)
388+
crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
389+
390+
return err;
391+
}
392+
393+
static void cmac_gf128_mul_by_x(be128 *y, const be128 *x)
394+
{
395+
u64 a = be64_to_cpu(x->a);
396+
u64 b = be64_to_cpu(x->b);
397+
398+
y->a = cpu_to_be64((a << 1) | (b >> 63));
399+
y->b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0));
400+
}
401+
402+
static int cmac_setkey(struct crypto_shash *tfm, const u8 *in_key,
403+
unsigned int key_len)
404+
{
405+
struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
406+
be128 *consts = (be128 *)ctx->consts;
407+
u8 *rk = (u8 *)ctx->key.key_enc;
408+
int rounds = 6 + key_len / 4;
409+
int err;
410+
411+
err = cbcmac_setkey(tfm, in_key, key_len);
412+
if (err)
413+
return err;
414+
415+
/* encrypt the zero vector */
416+
kernel_neon_begin();
417+
aes_ecb_encrypt(ctx->consts, (u8[AES_BLOCK_SIZE]){}, rk, rounds, 1, 1);
418+
kernel_neon_end();
419+
420+
cmac_gf128_mul_by_x(consts, consts);
421+
cmac_gf128_mul_by_x(consts + 1, consts);
422+
423+
return 0;
424+
}
425+
426+
static int xcbc_setkey(struct crypto_shash *tfm, const u8 *in_key,
427+
unsigned int key_len)
428+
{
429+
static u8 const ks[3][AES_BLOCK_SIZE] = {
430+
{ [0 ... AES_BLOCK_SIZE - 1] = 0x1 },
431+
{ [0 ... AES_BLOCK_SIZE - 1] = 0x2 },
432+
{ [0 ... AES_BLOCK_SIZE - 1] = 0x3 },
433+
};
434+
435+
struct mac_tfm_ctx *ctx = crypto_shash_ctx(tfm);
436+
u8 *rk = (u8 *)ctx->key.key_enc;
437+
int rounds = 6 + key_len / 4;
438+
u8 key[AES_BLOCK_SIZE];
439+
int err;
440+
441+
err = cbcmac_setkey(tfm, in_key, key_len);
442+
if (err)
443+
return err;
444+
445+
kernel_neon_begin();
446+
aes_ecb_encrypt(key, ks[0], rk, rounds, 1, 1);
447+
aes_ecb_encrypt(ctx->consts, ks[1], rk, rounds, 2, 0);
448+
kernel_neon_end();
449+
450+
return cbcmac_setkey(tfm, key, sizeof(key));
451+
}
452+
453+
static int mac_init(struct shash_desc *desc)
454+
{
455+
struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
456+
457+
memset(ctx->dg, 0, AES_BLOCK_SIZE);
458+
ctx->len = 0;
459+
460+
return 0;
461+
}
462+
463+
static int mac_update(struct shash_desc *desc, const u8 *p, unsigned int len)
464+
{
465+
struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
466+
struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
467+
int rounds = 6 + tctx->key.key_length / 4;
468+
469+
while (len > 0) {
470+
unsigned int l;
471+
472+
if ((ctx->len % AES_BLOCK_SIZE) == 0 &&
473+
(ctx->len + len) > AES_BLOCK_SIZE) {
474+
475+
int blocks = len / AES_BLOCK_SIZE;
476+
477+
len %= AES_BLOCK_SIZE;
478+
479+
kernel_neon_begin();
480+
aes_mac_update(p, tctx->key.key_enc, rounds, blocks,
481+
ctx->dg, (ctx->len != 0), (len != 0));
482+
kernel_neon_end();
483+
484+
p += blocks * AES_BLOCK_SIZE;
485+
486+
if (!len) {
487+
ctx->len = AES_BLOCK_SIZE;
488+
break;
489+
}
490+
ctx->len = 0;
491+
}
492+
493+
l = min(len, AES_BLOCK_SIZE - ctx->len);
494+
495+
if (l <= AES_BLOCK_SIZE) {
496+
crypto_xor(ctx->dg + ctx->len, p, l);
497+
ctx->len += l;
498+
len -= l;
499+
p += l;
500+
}
501+
}
502+
503+
return 0;
504+
}
505+
506+
static int cbcmac_final(struct shash_desc *desc, u8 *out)
507+
{
508+
struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
509+
struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
510+
int rounds = 6 + tctx->key.key_length / 4;
511+
512+
kernel_neon_begin();
513+
aes_mac_update(NULL, tctx->key.key_enc, rounds, 0, ctx->dg, 1, 0);
514+
kernel_neon_end();
515+
516+
memcpy(out, ctx->dg, AES_BLOCK_SIZE);
517+
518+
return 0;
519+
}
520+
521+
static int cmac_final(struct shash_desc *desc, u8 *out)
522+
{
523+
struct mac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
524+
struct mac_desc_ctx *ctx = shash_desc_ctx(desc);
525+
int rounds = 6 + tctx->key.key_length / 4;
526+
u8 *consts = tctx->consts;
527+
528+
if (ctx->len != AES_BLOCK_SIZE) {
529+
ctx->dg[ctx->len] ^= 0x80;
530+
consts += AES_BLOCK_SIZE;
531+
}
532+
533+
kernel_neon_begin();
534+
aes_mac_update(consts, tctx->key.key_enc, rounds, 1, ctx->dg, 0, 1);
535+
kernel_neon_end();
536+
537+
memcpy(out, ctx->dg, AES_BLOCK_SIZE);
538+
539+
return 0;
540+
}
541+
542+
static struct shash_alg mac_algs[] = { {
543+
.base.cra_name = "cmac(aes)",
544+
.base.cra_driver_name = "cmac-aes-" MODE,
545+
.base.cra_priority = PRIO,
546+
.base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
547+
.base.cra_blocksize = AES_BLOCK_SIZE,
548+
.base.cra_ctxsize = sizeof(struct mac_tfm_ctx) +
549+
2 * AES_BLOCK_SIZE,
550+
.base.cra_module = THIS_MODULE,
551+
552+
.digestsize = AES_BLOCK_SIZE,
553+
.init = mac_init,
554+
.update = mac_update,
555+
.final = cmac_final,
556+
.setkey = cmac_setkey,
557+
.descsize = sizeof(struct mac_desc_ctx),
558+
}, {
559+
.base.cra_name = "xcbc(aes)",
560+
.base.cra_driver_name = "xcbc-aes-" MODE,
561+
.base.cra_priority = PRIO,
562+
.base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
563+
.base.cra_blocksize = AES_BLOCK_SIZE,
564+
.base.cra_ctxsize = sizeof(struct mac_tfm_ctx) +
565+
2 * AES_BLOCK_SIZE,
566+
.base.cra_module = THIS_MODULE,
567+
568+
.digestsize = AES_BLOCK_SIZE,
569+
.init = mac_init,
570+
.update = mac_update,
571+
.final = cmac_final,
572+
.setkey = xcbc_setkey,
573+
.descsize = sizeof(struct mac_desc_ctx),
574+
}, {
575+
.base.cra_name = "cbcmac(aes)",
576+
.base.cra_driver_name = "cbcmac-aes-" MODE,
577+
.base.cra_priority = PRIO,
578+
.base.cra_flags = CRYPTO_ALG_TYPE_SHASH,
579+
.base.cra_blocksize = 1,
580+
.base.cra_ctxsize = sizeof(struct mac_tfm_ctx),
581+
.base.cra_module = THIS_MODULE,
582+
583+
.digestsize = AES_BLOCK_SIZE,
584+
.init = mac_init,
585+
.update = mac_update,
586+
.final = cbcmac_final,
587+
.setkey = cbcmac_setkey,
588+
.descsize = sizeof(struct mac_desc_ctx),
589+
} };
590+
360591
static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];
361592

362593
static void aes_exit(void)
@@ -367,6 +598,7 @@ static void aes_exit(void)
367598
if (aes_simd_algs[i])
368599
simd_skcipher_free(aes_simd_algs[i]);
369600

601+
crypto_unregister_shashes(mac_algs, ARRAY_SIZE(mac_algs));
370602
crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
371603
}
372604

@@ -383,6 +615,10 @@ static int __init aes_init(void)
383615
if (err)
384616
return err;
385617

618+
err = crypto_register_shashes(mac_algs, ARRAY_SIZE(mac_algs));
619+
if (err)
620+
goto unregister_ciphers;
621+
386622
for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
387623
if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
388624
continue;
@@ -402,6 +638,8 @@ static int __init aes_init(void)
402638

403639
unregister_simds:
404640
aes_exit();
641+
unregister_ciphers:
642+
crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
405643
return err;
406644
}
407645

‎arch/arm64/crypto/aes-modes.S

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/*
22
* linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES
33
*
4-
* Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
4+
* Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
55
*
66
* This program is free software; you can redistribute it and/or modify
77
* it under the terms of the GNU General Public License version 2 as
@@ -525,3 +525,30 @@ AES_ENTRY(aes_xts_decrypt)
525525
FRAME_POP
526526
ret
527527
AES_ENDPROC(aes_xts_decrypt)
528+
529+
/*
530+
* aes_mac_update(u8 const in[], u32 const rk[], int rounds,
531+
* int blocks, u8 dg[], int enc_before, int enc_after)
532+
*/
533+
AES_ENTRY(aes_mac_update)
534+
ld1 {v0.16b}, [x4] /* get dg */
535+
enc_prepare w2, x1, x7
536+
cbnz w5, .Lmacenc
537+
538+
.Lmacloop:
539+
cbz w3, .Lmacout
540+
ld1 {v1.16b}, [x0], #16 /* get next pt block */
541+
eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */
542+
543+
subs w3, w3, #1
544+
csinv x5, x6, xzr, eq
545+
cbz w5, .Lmacout
546+
547+
.Lmacenc:
548+
encrypt_block v0, w2, x1, x7, w8
549+
b .Lmacloop
550+
551+
.Lmacout:
552+
st1 {v0.16b}, [x4] /* return dg */
553+
ret
554+
AES_ENDPROC(aes_mac_update)

0 commit comments

Comments
 (0)