Skip to content

Commit a3fd821

Browse files
author
Ard Biesheuvel
committed
arm64/crypto: AES in CCM mode using ARMv8 Crypto Extensions
This patch adds support for the AES-CCM encryption algorithm for CPUs that have support for the AES part of the ARM v8 Crypto Extensions. Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Acked-by: Herbert Xu <herbert@gondor.apana.org.au>
1 parent 317f2f7 commit a3fd821

File tree

4 files changed

+529
-0
lines changed

4 files changed

+529
-0
lines changed

‎arch/arm64/crypto/Kconfig

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,11 @@ config CRYPTO_AES_ARM64_CE
2929
select CRYPTO_ALGAPI
3030
select CRYPTO_AES
3131

32+
config CRYPTO_AES_ARM64_CE_CCM
33+
tristate "AES in CCM mode using ARMv8 Crypto Extensions"
34+
depends on ARM64 && KERNEL_MODE_NEON
35+
select CRYPTO_ALGAPI
36+
select CRYPTO_AES
37+
select CRYPTO_AEAD
38+
3239
endif

‎arch/arm64/crypto/Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,6 @@ ghash-ce-y := ghash-ce-glue.o ghash-ce-core.o
1919

2020
obj-$(CONFIG_CRYPTO_AES_ARM64_CE) += aes-ce-cipher.o
2121
CFLAGS_aes-ce-cipher.o += -march=armv8-a+crypto
22+
23+
obj-$(CONFIG_CRYPTO_AES_ARM64_CE_CCM) += aes-ce-ccm.o
24+
aes-ce-ccm-y := aes-ce-ccm-glue.o aes-ce-ccm-core.o

‎arch/arm64/crypto/aes-ce-ccm-core.S

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
/*
2+
* aesce-ccm-core.S - AES-CCM transform for ARMv8 with Crypto Extensions
3+
*
4+
* Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
5+
*
6+
* This program is free software; you can redistribute it and/or modify
7+
* it under the terms of the GNU General Public License version 2 as
8+
* published by the Free Software Foundation.
9+
*/
10+
11+
#include <linux/linkage.h>
12+
13+
.text
14+
.arch armv8-a+crypto
15+
16+
/*
17+
* void ce_aes_ccm_auth_data(u8 mac[], u8 const in[], u32 abytes,
18+
* u32 *macp, u8 const rk[], u32 rounds);
19+
*/
20+
ENTRY(ce_aes_ccm_auth_data)
21+
ldr w8, [x3] /* leftover from prev round? */
22+
ld1 {v0.2d}, [x0] /* load mac */
23+
cbz w8, 1f
24+
sub w8, w8, #16
25+
eor v1.16b, v1.16b, v1.16b
26+
0: ldrb w7, [x1], #1 /* get 1 byte of input */
27+
subs w2, w2, #1
28+
add w8, w8, #1
29+
ins v1.b[0], w7
30+
ext v1.16b, v1.16b, v1.16b, #1 /* rotate in the input bytes */
31+
beq 8f /* out of input? */
32+
cbnz w8, 0b
33+
eor v0.16b, v0.16b, v1.16b
34+
1: ld1 {v3.2d}, [x4] /* load first round key */
35+
prfm pldl1strm, [x1]
36+
cmp w5, #12 /* which key size? */
37+
add x6, x4, #16
38+
sub w7, w5, #2 /* modified # of rounds */
39+
bmi 2f
40+
bne 5f
41+
mov v5.16b, v3.16b
42+
b 4f
43+
2: mov v4.16b, v3.16b
44+
ld1 {v5.2d}, [x6], #16 /* load 2nd round key */
45+
3: aese v0.16b, v4.16b
46+
aesmc v0.16b, v0.16b
47+
4: ld1 {v3.2d}, [x6], #16 /* load next round key */
48+
aese v0.16b, v5.16b
49+
aesmc v0.16b, v0.16b
50+
5: ld1 {v4.2d}, [x6], #16 /* load next round key */
51+
subs w7, w7, #3
52+
aese v0.16b, v3.16b
53+
aesmc v0.16b, v0.16b
54+
ld1 {v5.2d}, [x6], #16 /* load next round key */
55+
bpl 3b
56+
aese v0.16b, v4.16b
57+
subs w2, w2, #16 /* last data? */
58+
eor v0.16b, v0.16b, v5.16b /* final round */
59+
bmi 6f
60+
ld1 {v1.16b}, [x1], #16 /* load next input block */
61+
eor v0.16b, v0.16b, v1.16b /* xor with mac */
62+
bne 1b
63+
6: st1 {v0.2d}, [x0] /* store mac */
64+
beq 10f
65+
adds w2, w2, #16
66+
beq 10f
67+
mov w8, w2
68+
7: ldrb w7, [x1], #1
69+
umov w6, v0.b[0]
70+
eor w6, w6, w7
71+
strb w6, [x0], #1
72+
subs w2, w2, #1
73+
beq 10f
74+
ext v0.16b, v0.16b, v0.16b, #1 /* rotate out the mac bytes */
75+
b 7b
76+
8: mov w7, w8
77+
add w8, w8, #16
78+
9: ext v1.16b, v1.16b, v1.16b, #1
79+
adds w7, w7, #1
80+
bne 9b
81+
eor v0.16b, v0.16b, v1.16b
82+
st1 {v0.2d}, [x0]
83+
10: str w8, [x3]
84+
ret
85+
ENDPROC(ce_aes_ccm_auth_data)
86+
87+
/*
88+
* void ce_aes_ccm_final(u8 mac[], u8 const ctr[], u8 const rk[],
89+
* u32 rounds);
90+
*/
91+
ENTRY(ce_aes_ccm_final)
92+
ld1 {v3.2d}, [x2], #16 /* load first round key */
93+
ld1 {v0.2d}, [x0] /* load mac */
94+
cmp w3, #12 /* which key size? */
95+
sub w3, w3, #2 /* modified # of rounds */
96+
ld1 {v1.2d}, [x1] /* load 1st ctriv */
97+
bmi 0f
98+
bne 3f
99+
mov v5.16b, v3.16b
100+
b 2f
101+
0: mov v4.16b, v3.16b
102+
1: ld1 {v5.2d}, [x2], #16 /* load next round key */
103+
aese v0.16b, v4.16b
104+
aese v1.16b, v4.16b
105+
aesmc v0.16b, v0.16b
106+
aesmc v1.16b, v1.16b
107+
2: ld1 {v3.2d}, [x2], #16 /* load next round key */
108+
aese v0.16b, v5.16b
109+
aese v1.16b, v5.16b
110+
aesmc v0.16b, v0.16b
111+
aesmc v1.16b, v1.16b
112+
3: ld1 {v4.2d}, [x2], #16 /* load next round key */
113+
subs w3, w3, #3
114+
aese v0.16b, v3.16b
115+
aese v1.16b, v3.16b
116+
aesmc v0.16b, v0.16b
117+
aesmc v1.16b, v1.16b
118+
bpl 1b
119+
aese v0.16b, v4.16b
120+
aese v1.16b, v4.16b
121+
/* final round key cancels out */
122+
eor v0.16b, v0.16b, v1.16b /* en-/decrypt the mac */
123+
st1 {v0.2d}, [x0] /* store result */
124+
ret
125+
ENDPROC(ce_aes_ccm_final)
126+
127+
.macro aes_ccm_do_crypt,enc
128+
ldr x8, [x6, #8] /* load lower ctr */
129+
ld1 {v0.2d}, [x5] /* load mac */
130+
rev x8, x8 /* keep swabbed ctr in reg */
131+
0: /* outer loop */
132+
ld1 {v1.1d}, [x6] /* load upper ctr */
133+
prfm pldl1strm, [x1]
134+
add x8, x8, #1
135+
rev x9, x8
136+
cmp w4, #12 /* which key size? */
137+
sub w7, w4, #2 /* get modified # of rounds */
138+
ins v1.d[1], x9 /* no carry in lower ctr */
139+
ld1 {v3.2d}, [x3] /* load first round key */
140+
add x10, x3, #16
141+
bmi 1f
142+
bne 4f
143+
mov v5.16b, v3.16b
144+
b 3f
145+
1: mov v4.16b, v3.16b
146+
ld1 {v5.2d}, [x10], #16 /* load 2nd round key */
147+
2: /* inner loop: 3 rounds, 2x interleaved */
148+
aese v0.16b, v4.16b
149+
aese v1.16b, v4.16b
150+
aesmc v0.16b, v0.16b
151+
aesmc v1.16b, v1.16b
152+
3: ld1 {v3.2d}, [x10], #16 /* load next round key */
153+
aese v0.16b, v5.16b
154+
aese v1.16b, v5.16b
155+
aesmc v0.16b, v0.16b
156+
aesmc v1.16b, v1.16b
157+
4: ld1 {v4.2d}, [x10], #16 /* load next round key */
158+
subs w7, w7, #3
159+
aese v0.16b, v3.16b
160+
aese v1.16b, v3.16b
161+
aesmc v0.16b, v0.16b
162+
aesmc v1.16b, v1.16b
163+
ld1 {v5.2d}, [x10], #16 /* load next round key */
164+
bpl 2b
165+
aese v0.16b, v4.16b
166+
aese v1.16b, v4.16b
167+
subs w2, w2, #16
168+
bmi 6f /* partial block? */
169+
ld1 {v2.16b}, [x1], #16 /* load next input block */
170+
.if \enc == 1
171+
eor v2.16b, v2.16b, v5.16b /* final round enc+mac */
172+
eor v1.16b, v1.16b, v2.16b /* xor with crypted ctr */
173+
.else
174+
eor v2.16b, v2.16b, v1.16b /* xor with crypted ctr */
175+
eor v1.16b, v2.16b, v5.16b /* final round enc */
176+
.endif
177+
eor v0.16b, v0.16b, v2.16b /* xor mac with pt ^ rk[last] */
178+
st1 {v1.16b}, [x0], #16 /* write output block */
179+
bne 0b
180+
rev x8, x8
181+
st1 {v0.2d}, [x5] /* store mac */
182+
str x8, [x6, #8] /* store lsb end of ctr (BE) */
183+
5: ret
184+
185+
6: eor v0.16b, v0.16b, v5.16b /* final round mac */
186+
eor v1.16b, v1.16b, v5.16b /* final round enc */
187+
st1 {v0.2d}, [x5] /* store mac */
188+
add w2, w2, #16 /* process partial tail block */
189+
7: ldrb w9, [x1], #1 /* get 1 byte of input */
190+
umov w6, v1.b[0] /* get top crypted ctr byte */
191+
umov w7, v0.b[0] /* get top mac byte */
192+
.if \enc == 1
193+
eor w7, w7, w9
194+
eor w9, w9, w6
195+
.else
196+
eor w9, w9, w6
197+
eor w7, w7, w9
198+
.endif
199+
strb w9, [x0], #1 /* store out byte */
200+
strb w7, [x5], #1 /* store mac byte */
201+
subs w2, w2, #1
202+
beq 5b
203+
ext v0.16b, v0.16b, v0.16b, #1 /* shift out mac byte */
204+
ext v1.16b, v1.16b, v1.16b, #1 /* shift out ctr byte */
205+
b 7b
206+
.endm
207+
208+
/*
209+
* void ce_aes_ccm_encrypt(u8 out[], u8 const in[], u32 cbytes,
210+
* u8 const rk[], u32 rounds, u8 mac[],
211+
* u8 ctr[]);
212+
* void ce_aes_ccm_decrypt(u8 out[], u8 const in[], u32 cbytes,
213+
* u8 const rk[], u32 rounds, u8 mac[],
214+
* u8 ctr[]);
215+
*/
216+
ENTRY(ce_aes_ccm_encrypt)
217+
aes_ccm_do_crypt 1
218+
ENDPROC(ce_aes_ccm_encrypt)
219+
220+
ENTRY(ce_aes_ccm_decrypt)
221+
aes_ccm_do_crypt 0
222+
ENDPROC(ce_aes_ccm_decrypt)

0 commit comments

Comments
 (0)