Skip to content

Commit 49788fe

Browse files
author
Ard Biesheuvel
committed
arm64/crypto: AES-ECB/CBC/CTR/XTS using ARMv8 NEON and Crypto Extensions
This adds ARMv8 implementations of AES in ECB, CBC, CTR and XTS modes, both for ARMv8 with Crypto Extensions and for plain ARMv8 NEON. The Crypto Extensions version can only run on ARMv8 implementations that have support for these optional extensions. The plain NEON version is a table based yet time invariant implementation. All S-box substitutions are performed in parallel, leveraging the wide range of ARMv8's tbl/tbx instructions, and the huge NEON register file, which can comfortably hold the entire S-box and still have room to spare for doing the actual computations. The key expansion routines were borrowed from aes_generic. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
1 parent 2ca10f8 commit 49788fe

File tree

6 files changed

+1521
-0
lines changed

6 files changed

+1521
-0
lines changed

‎arch/arm64/crypto/Kconfig

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,4 +36,18 @@ config CRYPTO_AES_ARM64_CE_CCM
3636
select CRYPTO_AES
3737
select CRYPTO_AEAD
3838

39+
config CRYPTO_AES_ARM64_CE_BLK
40+
tristate "AES in ECB/CBC/CTR/XTS modes using ARMv8 Crypto Extensions"
41+
depends on ARM64 && KERNEL_MODE_NEON
42+
select CRYPTO_BLKCIPHER
43+
select CRYPTO_AES
44+
select CRYPTO_ABLK_HELPER
45+
46+
config CRYPTO_AES_ARM64_NEON_BLK
47+
tristate "AES in ECB/CBC/CTR/XTS modes using NEON instructions"
48+
depends on ARM64 && KERNEL_MODE_NEON
49+
select CRYPTO_BLKCIPHER
50+
select CRYPTO_AES
51+
select CRYPTO_ABLK_HELPER
52+
3953
endif

‎arch/arm64/crypto/Makefile

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,17 @@ CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto
2222

2323
obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o
2424
aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o
25+
26+
obj-$(CONFIG_CRYPTO_AES_ARM64_CE_BLK) += aes-ce-blk.o
27+
aes-ce-blk-y := aes-glue-ce.o aes-ce.o
28+
29+
obj-$(CONFIG_CRYPTO_AES_ARM64_NEON_BLK) += aes-neon-blk.o
30+
aes-neon-blk-y := aes-glue-neon.o aes-neon.o
31+
32+
AFLAGS_aes-ce.o := -DINTERLEAVE=2 -DINTERLEAVE_INLINE
33+
AFLAGS_aes-neon.o := -DINTERLEAVE=4
34+
35+
CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS
36+
37+
$(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE
38+
$(call if_changed_dep,cc_o_c)

‎arch/arm64/crypto/aes-ce.S

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
/*
2+
* linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with
3+
* Crypto Extensions
4+
*
5+
* Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
6+
*
7+
* This program is free software; you can redistribute it and/or modify
8+
* it under the terms of the GNU General Public License version 2 as
9+
* published by the Free Software Foundation.
10+
*/
11+
12+
#include <linux/linkage.h>
13+
14+
#define AES_ENTRY(func) ENTRY(ce_ ## func)
15+
#define AES_ENDPROC(func) ENDPROC(ce_ ## func)
16+
17+
.arch armv8-a+crypto
18+
19+
/* preload all round keys */
20+
.macro load_round_keys, rounds, rk
21+
cmp \rounds, #12
22+
blo 2222f /* 128 bits */
23+
beq 1111f /* 192 bits */
24+
ld1 {v17.16b-v18.16b}, [\rk], #32
25+
1111: ld1 {v19.16b-v20.16b}, [\rk], #32
26+
2222: ld1 {v21.16b-v24.16b}, [\rk], #64
27+
ld1 {v25.16b-v28.16b}, [\rk], #64
28+
ld1 {v29.16b-v31.16b}, [\rk]
29+
.endm
30+
31+
/* prepare for encryption with key in rk[] */
32+
.macro enc_prepare, rounds, rk, ignore
33+
load_round_keys \rounds, \rk
34+
.endm
35+
36+
/* prepare for encryption (again) but with new key in rk[] */
37+
.macro enc_switch_key, rounds, rk, ignore
38+
load_round_keys \rounds, \rk
39+
.endm
40+
41+
/* prepare for decryption with key in rk[] */
42+
.macro dec_prepare, rounds, rk, ignore
43+
load_round_keys \rounds, \rk
44+
.endm
45+
46+
.macro do_enc_Nx, de, mc, k, i0, i1, i2, i3
47+
aes\de \i0\().16b, \k\().16b
48+
.ifnb \i1
49+
aes\de \i1\().16b, \k\().16b
50+
.ifnb \i3
51+
aes\de \i2\().16b, \k\().16b
52+
aes\de \i3\().16b, \k\().16b
53+
.endif
54+
.endif
55+
aes\mc \i0\().16b, \i0\().16b
56+
.ifnb \i1
57+
aes\mc \i1\().16b, \i1\().16b
58+
.ifnb \i3
59+
aes\mc \i2\().16b, \i2\().16b
60+
aes\mc \i3\().16b, \i3\().16b
61+
.endif
62+
.endif
63+
.endm
64+
65+
/* up to 4 interleaved encryption rounds with the same round key */
66+
.macro round_Nx, enc, k, i0, i1, i2, i3
67+
.ifc \enc, e
68+
do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3
69+
.else
70+
do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3
71+
.endif
72+
.endm
73+
74+
/* up to 4 interleaved final rounds */
75+
.macro fin_round_Nx, de, k, k2, i0, i1, i2, i3
76+
aes\de \i0\().16b, \k\().16b
77+
.ifnb \i1
78+
aes\de \i1\().16b, \k\().16b
79+
.ifnb \i3
80+
aes\de \i2\().16b, \k\().16b
81+
aes\de \i3\().16b, \k\().16b
82+
.endif
83+
.endif
84+
eor \i0\().16b, \i0\().16b, \k2\().16b
85+
.ifnb \i1
86+
eor \i1\().16b, \i1\().16b, \k2\().16b
87+
.ifnb \i3
88+
eor \i2\().16b, \i2\().16b, \k2\().16b
89+
eor \i3\().16b, \i3\().16b, \k2\().16b
90+
.endif
91+
.endif
92+
.endm
93+
94+
/* up to 4 interleaved blocks */
95+
.macro do_block_Nx, enc, rounds, i0, i1, i2, i3
96+
cmp \rounds, #12
97+
blo 2222f /* 128 bits */
98+
beq 1111f /* 192 bits */
99+
round_Nx \enc, v17, \i0, \i1, \i2, \i3
100+
round_Nx \enc, v18, \i0, \i1, \i2, \i3
101+
1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3
102+
round_Nx \enc, v20, \i0, \i1, \i2, \i3
103+
2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29
104+
round_Nx \enc, \key, \i0, \i1, \i2, \i3
105+
.endr
106+
fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3
107+
.endm
108+
109+
.macro encrypt_block, in, rounds, t0, t1, t2
110+
do_block_Nx e, \rounds, \in
111+
.endm
112+
113+
.macro encrypt_block2x, i0, i1, rounds, t0, t1, t2
114+
do_block_Nx e, \rounds, \i0, \i1
115+
.endm
116+
117+
.macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
118+
do_block_Nx e, \rounds, \i0, \i1, \i2, \i3
119+
.endm
120+
121+
.macro decrypt_block, in, rounds, t0, t1, t2
122+
do_block_Nx d, \rounds, \in
123+
.endm
124+
125+
.macro decrypt_block2x, i0, i1, rounds, t0, t1, t2
126+
do_block_Nx d, \rounds, \i0, \i1
127+
.endm
128+
129+
.macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
130+
do_block_Nx d, \rounds, \i0, \i1, \i2, \i3
131+
.endm
132+
133+
#include "aes-modes.S"

0 commit comments

Comments
 (0)