Skip to content

Commit 95a34b7

Browse files
ebiggersherbertx
ebiggers
authored and
herbertx
committed
crypto: arm64/chacha20 - refactor to allow varying number of rounds
In preparation for adding XChaCha12 support, rename/refactor the ARM64 NEON implementation of ChaCha20 to support different numbers of rounds. Reviewed-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> Signed-off-by: Eric Biggers <ebiggers@google.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
1 parent cc7cf99 commit 95a34b7

File tree

3 files changed

+57
-49
lines changed

3 files changed

+57
-49
lines changed

‎arch/arm64/crypto/Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ sha256-arm64-y := sha256-glue.o sha256-core.o
5050
obj-$(CONFIG_CRYPTO_SHA512_ARM64) += sha512-arm64.o
5151
sha512-arm64-y := sha512-glue.o sha512-core.o
5252

53-
obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha20-neon.o
54-
chacha20-neon-y := chacha20-neon-core.o chacha20-neon-glue.o
53+
obj-$(CONFIG_CRYPTO_CHACHA20_NEON) += chacha-neon.o
54+
chacha-neon-y := chacha-neon-core.o chacha-neon-glue.o
5555

5656
obj-$(CONFIG_CRYPTO_NHPOLY1305_NEON) += nhpoly1305-neon.o
5757
nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o

‎arch/arm64/crypto/chacha20-neon-core.S renamed to ‎arch/arm64/crypto/chacha-neon-core.S

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* ChaCha20 256-bit cipher algorithm, RFC7539, arm64 NEON functions
2+
* ChaCha/XChaCha NEON helper functions
33
*
44
* Copyright (C) 2016 Linaro, Ltd. <ard.biesheuvel@linaro.org>
55
*
@@ -24,17 +24,18 @@
2424
.align 6
2525

2626
/*
27-
* chacha20_permute - permute one block
27+
* chacha_permute - permute one block
2828
*
2929
* Permute one 64-byte block where the state matrix is stored in the four NEON
3030
* registers v0-v3. It performs matrix operations on four words in parallel,
3131
* but requires shuffling to rearrange the words after each round.
3232
*
33-
* Clobbers: x3, x10, v4, v12
33+
* The round count is given in w3.
34+
*
35+
* Clobbers: w3, x10, v4, v12
3436
*/
35-
chacha20_permute:
37+
chacha_permute:
3638

37-
mov x3, #10
3839
adr x10, ROT8
3940
ld1 {v12.4s}, [x10]
4041

@@ -97,16 +98,17 @@ chacha20_permute:
9798
// x3 = shuffle32(x3, MASK(0, 3, 2, 1))
9899
ext v3.16b, v3.16b, v3.16b, #4
99100

100-
subs x3, x3, #1
101+
subs w3, w3, #2
101102
b.ne .Ldoubleround
102103

103104
ret
104-
ENDPROC(chacha20_permute)
105+
ENDPROC(chacha_permute)
105106

106-
ENTRY(chacha20_block_xor_neon)
107+
ENTRY(chacha_block_xor_neon)
107108
// x0: Input state matrix, s
108109
// x1: 1 data block output, o
109110
// x2: 1 data block input, i
111+
// w3: nrounds
110112

111113
stp x29, x30, [sp, #-16]!
112114
mov x29, sp
@@ -115,7 +117,7 @@ ENTRY(chacha20_block_xor_neon)
115117
ld1 {v0.4s-v3.4s}, [x0]
116118
ld1 {v8.4s-v11.4s}, [x0]
117119

118-
bl chacha20_permute
120+
bl chacha_permute
119121

120122
ld1 {v4.16b-v7.16b}, [x2]
121123

@@ -139,42 +141,45 @@ ENTRY(chacha20_block_xor_neon)
139141

140142
ldp x29, x30, [sp], #16
141143
ret
142-
ENDPROC(chacha20_block_xor_neon)
144+
ENDPROC(chacha_block_xor_neon)
143145

144-
ENTRY(hchacha20_block_neon)
146+
ENTRY(hchacha_block_neon)
145147
// x0: Input state matrix, s
146148
// x1: output (8 32-bit words)
149+
// w2: nrounds
147150

148151
stp x29, x30, [sp, #-16]!
149152
mov x29, sp
150153

151154
ld1 {v0.4s-v3.4s}, [x0]
152155

153-
bl chacha20_permute
156+
mov w3, w2
157+
bl chacha_permute
154158

155159
st1 {v0.16b}, [x1], #16
156160
st1 {v3.16b}, [x1]
157161

158162
ldp x29, x30, [sp], #16
159163
ret
160-
ENDPROC(hchacha20_block_neon)
164+
ENDPROC(hchacha_block_neon)
161165

162166
.align 6
163-
ENTRY(chacha20_4block_xor_neon)
167+
ENTRY(chacha_4block_xor_neon)
164168
// x0: Input state matrix, s
165169
// x1: 4 data blocks output, o
166170
// x2: 4 data blocks input, i
171+
// w3: nrounds
167172

168173
//
169-
// This function encrypts four consecutive ChaCha20 blocks by loading
174+
// This function encrypts four consecutive ChaCha blocks by loading
170175
// the state matrix in NEON registers four times. The algorithm performs
171176
// each operation on the corresponding word of each state matrix, hence
172177
// requires no word shuffling. For final XORing step we transpose the
173178
// matrix by interleaving 32- and then 64-bit words, which allows us to
174179
// do XOR in NEON registers.
175180
//
176-
adr x3, CTRINC // ... and ROT8
177-
ld1 {v30.4s-v31.4s}, [x3]
181+
adr x9, CTRINC // ... and ROT8
182+
ld1 {v30.4s-v31.4s}, [x9]
178183

179184
// x0..15[0-3] = s0..3[0..3]
180185
mov x4, x0
@@ -186,8 +191,6 @@ ENTRY(chacha20_4block_xor_neon)
186191
// x12 += counter values 0-3
187192
add v12.4s, v12.4s, v30.4s
188193

189-
mov x3, #10
190-
191194
.Ldoubleround4:
192195
// x0 += x4, x12 = rotl32(x12 ^ x0, 16)
193196
// x1 += x5, x13 = rotl32(x13 ^ x1, 16)
@@ -361,7 +364,7 @@ ENTRY(chacha20_4block_xor_neon)
361364
sri v7.4s, v18.4s, #25
362365
sri v4.4s, v19.4s, #25
363366

364-
subs x3, x3, #1
367+
subs w3, w3, #2
365368
b.ne .Ldoubleround4
366369

367370
ld4r {v16.4s-v19.4s}, [x0], #16
@@ -475,7 +478,7 @@ ENTRY(chacha20_4block_xor_neon)
475478
st1 {v28.16b-v31.16b}, [x1]
476479

477480
ret
478-
ENDPROC(chacha20_4block_xor_neon)
481+
ENDPROC(chacha_4block_xor_neon)
479482

480483
CTRINC: .word 0, 1, 2, 3
481484
ROT8: .word 0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f

‎arch/arm64/crypto/chacha20-neon-glue.c renamed to ‎arch/arm64/crypto/chacha-neon-glue.c

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/*
2-
* ChaCha20 256-bit cipher algorithm, RFC7539, arm64 NEON functions
2+
* ARM NEON accelerated ChaCha and XChaCha stream ciphers,
3+
* including ChaCha20 (RFC7539)
34
*
45
* Copyright (C) 2016 - 2017 Linaro, Ltd. <ard.biesheuvel@linaro.org>
56
*
@@ -28,18 +29,20 @@
2829
#include <asm/neon.h>
2930
#include <asm/simd.h>
3031

31-
asmlinkage void chacha20_block_xor_neon(u32 *state, u8 *dst, const u8 *src);
32-
asmlinkage void chacha20_4block_xor_neon(u32 *state, u8 *dst, const u8 *src);
33-
asmlinkage void hchacha20_block_neon(const u32 *state, u32 *out);
32+
asmlinkage void chacha_block_xor_neon(u32 *state, u8 *dst, const u8 *src,
33+
int nrounds);
34+
asmlinkage void chacha_4block_xor_neon(u32 *state, u8 *dst, const u8 *src,
35+
int nrounds);
36+
asmlinkage void hchacha_block_neon(const u32 *state, u32 *out, int nrounds);
3437

35-
static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src,
36-
unsigned int bytes)
38+
static void chacha_doneon(u32 *state, u8 *dst, const u8 *src,
39+
unsigned int bytes, int nrounds)
3740
{
3841
u8 buf[CHACHA_BLOCK_SIZE];
3942

4043
while (bytes >= CHACHA_BLOCK_SIZE * 4) {
4144
kernel_neon_begin();
42-
chacha20_4block_xor_neon(state, dst, src);
45+
chacha_4block_xor_neon(state, dst, src, nrounds);
4346
kernel_neon_end();
4447
bytes -= CHACHA_BLOCK_SIZE * 4;
4548
src += CHACHA_BLOCK_SIZE * 4;
@@ -52,22 +55,22 @@ static void chacha20_doneon(u32 *state, u8 *dst, const u8 *src,
5255

5356
kernel_neon_begin();
5457
while (bytes >= CHACHA_BLOCK_SIZE) {
55-
chacha20_block_xor_neon(state, dst, src);
58+
chacha_block_xor_neon(state, dst, src, nrounds);
5659
bytes -= CHACHA_BLOCK_SIZE;
5760
src += CHACHA_BLOCK_SIZE;
5861
dst += CHACHA_BLOCK_SIZE;
5962
state[12]++;
6063
}
6164
if (bytes) {
6265
memcpy(buf, src, bytes);
63-
chacha20_block_xor_neon(state, buf, buf);
66+
chacha_block_xor_neon(state, buf, buf, nrounds);
6467
memcpy(dst, buf, bytes);
6568
}
6669
kernel_neon_end();
6770
}
6871

69-
static int chacha20_neon_stream_xor(struct skcipher_request *req,
70-
struct chacha_ctx *ctx, u8 *iv)
72+
static int chacha_neon_stream_xor(struct skcipher_request *req,
73+
struct chacha_ctx *ctx, u8 *iv)
7174
{
7275
struct skcipher_walk walk;
7376
u32 state[16];
@@ -83,26 +86,26 @@ static int chacha20_neon_stream_xor(struct skcipher_request *req,
8386
if (nbytes < walk.total)
8487
nbytes = round_down(nbytes, walk.stride);
8588

86-
chacha20_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
87-
nbytes);
89+
chacha_doneon(state, walk.dst.virt.addr, walk.src.virt.addr,
90+
nbytes, ctx->nrounds);
8891
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
8992
}
9093

9194
return err;
9295
}
9396

94-
static int chacha20_neon(struct skcipher_request *req)
97+
static int chacha_neon(struct skcipher_request *req)
9598
{
9699
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
97100
struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
98101

99102
if (req->cryptlen <= CHACHA_BLOCK_SIZE || !may_use_simd())
100103
return crypto_chacha_crypt(req);
101104

102-
return chacha20_neon_stream_xor(req, ctx, req->iv);
105+
return chacha_neon_stream_xor(req, ctx, req->iv);
103106
}
104107

105-
static int xchacha20_neon(struct skcipher_request *req)
108+
static int xchacha_neon(struct skcipher_request *req)
106109
{
107110
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
108111
struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
@@ -116,12 +119,13 @@ static int xchacha20_neon(struct skcipher_request *req)
116119
crypto_chacha_init(state, ctx, req->iv);
117120

118121
kernel_neon_begin();
119-
hchacha20_block_neon(state, subctx.key);
122+
hchacha_block_neon(state, subctx.key, ctx->nrounds);
120123
kernel_neon_end();
124+
subctx.nrounds = ctx->nrounds;
121125

122126
memcpy(&real_iv[0], req->iv + 24, 8);
123127
memcpy(&real_iv[8], req->iv + 16, 8);
124-
return chacha20_neon_stream_xor(req, &subctx, real_iv);
128+
return chacha_neon_stream_xor(req, &subctx, real_iv);
125129
}
126130

127131
static struct skcipher_alg algs[] = {
@@ -139,8 +143,8 @@ static struct skcipher_alg algs[] = {
139143
.chunksize = CHACHA_BLOCK_SIZE,
140144
.walksize = 4 * CHACHA_BLOCK_SIZE,
141145
.setkey = crypto_chacha20_setkey,
142-
.encrypt = chacha20_neon,
143-
.decrypt = chacha20_neon,
146+
.encrypt = chacha_neon,
147+
.decrypt = chacha_neon,
144148
}, {
145149
.base.cra_name = "xchacha20",
146150
.base.cra_driver_name = "xchacha20-neon",
@@ -155,27 +159,28 @@ static struct skcipher_alg algs[] = {
155159
.chunksize = CHACHA_BLOCK_SIZE,
156160
.walksize = 4 * CHACHA_BLOCK_SIZE,
157161
.setkey = crypto_chacha20_setkey,
158-
.encrypt = xchacha20_neon,
159-
.decrypt = xchacha20_neon,
162+
.encrypt = xchacha_neon,
163+
.decrypt = xchacha_neon,
160164
}
161165
};
162166

163-
static int __init chacha20_simd_mod_init(void)
167+
static int __init chacha_simd_mod_init(void)
164168
{
165169
if (!(elf_hwcap & HWCAP_ASIMD))
166170
return -ENODEV;
167171

168172
return crypto_register_skciphers(algs, ARRAY_SIZE(algs));
169173
}
170174

171-
static void __exit chacha20_simd_mod_fini(void)
175+
static void __exit chacha_simd_mod_fini(void)
172176
{
173177
crypto_unregister_skciphers(algs, ARRAY_SIZE(algs));
174178
}
175179

176-
module_init(chacha20_simd_mod_init);
177-
module_exit(chacha20_simd_mod_fini);
180+
module_init(chacha_simd_mod_init);
181+
module_exit(chacha_simd_mod_fini);
178182

183+
MODULE_DESCRIPTION("ChaCha and XChaCha stream ciphers (NEON accelerated)");
179184
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
180185
MODULE_LICENSE("GPL v2");
181186
MODULE_ALIAS_CRYPTO("chacha20");

0 commit comments

Comments
 (0)