Skip to content

Commit a76c1c2

Browse files
Chuck Ebbertherbertx
Chuck Ebbert
authored and
herbertx
committed
crypto: padlock-aes - work around Nano CPU errata in ECB mode
The VIA Nano processor has a bug that makes it prefetch extra data during encryption operations, causing spurious page faults. Extend existing workarounds for ECB mode to copy the data to an temporary buffer to avoid the problem. Signed-off-by: Chuck Ebbert <cebbert@redhat.com> Acked-by: Harald Welte <HaraldWelte@viatech.com> Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
1 parent 1d89b30 commit a76c1c2

File tree

1 file changed

+46
-35
lines changed

1 file changed

+46
-35
lines changed

‎drivers/crypto/padlock-aes.c

Lines changed: 46 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,17 @@
1818
#include <linux/percpu.h>
1919
#include <linux/smp.h>
2020
#include <asm/byteorder.h>
21+
#include <asm/processor.h>
2122
#include <asm/i387.h>
2223
#include "padlock.h"
2324

25+
/* number of data blocks actually fetched for each xcrypt insn */
26+
static unsigned int ecb_fetch_blocks = 2;
27+
static unsigned int cbc_fetch_blocks = 1;
28+
29+
#define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)
30+
#define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)
31+
2432
/* Control word. */
2533
struct cword {
2634
unsigned int __attribute__ ((__packed__))
@@ -173,63 +181,59 @@ static inline void padlock_store_cword(struct cword *cword)
173181
*/
174182

175183
static inline void padlock_xcrypt(const u8 *input, u8 *output, void *key,
176-
struct cword *control_word)
184+
struct cword *control_word, int count)
177185
{
178186
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
179187
: "+S"(input), "+D"(output)
180-
: "d"(control_word), "b"(key), "c"(1));
188+
: "d"(control_word), "b"(key), "c"(count));
181189
}
182190

183-
static void aes_crypt_copy(const u8 *in, u8 *out, u32 *key, struct cword *cword)
191+
static void aes_crypt_copy(const u8 *in, u8 *out, u32 *key,
192+
struct cword *cword, int count)
184193
{
185-
u8 buf[AES_BLOCK_SIZE * 2 + PADLOCK_ALIGNMENT - 1];
194+
/*
195+
* Padlock prefetches extra data so we must provide mapped input buffers.
196+
* Assume there are at least 16 bytes of stack already in use.
197+
*/
198+
u8 buf[AES_BLOCK_SIZE * 7 + PADLOCK_ALIGNMENT - 1];
186199
u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
187200

188-
memcpy(tmp, in, AES_BLOCK_SIZE);
189-
padlock_xcrypt(tmp, out, key, cword);
201+
memcpy(tmp, in, count * AES_BLOCK_SIZE);
202+
padlock_xcrypt(tmp, out, key, cword, count);
190203
}
191204

192205
static inline void aes_crypt(const u8 *in, u8 *out, u32 *key,
193-
struct cword *cword)
206+
struct cword *cword, int count)
194207
{
195-
/* padlock_xcrypt requires at least two blocks of data. */
196-
if (unlikely(!(((unsigned long)in ^ (PAGE_SIZE - AES_BLOCK_SIZE)) &
197-
(PAGE_SIZE - 1)))) {
198-
aes_crypt_copy(in, out, key, cword);
208+
/* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
209+
* We could avoid some copying here but it's probably not worth it.
210+
*/
211+
if (unlikely(((unsigned long)in & PAGE_SIZE) + ecb_fetch_bytes > PAGE_SIZE)) {
212+
aes_crypt_copy(in, out, key, cword, count);
199213
return;
200214
}
201215

202-
padlock_xcrypt(in, out, key, cword);
216+
padlock_xcrypt(in, out, key, cword, count);
203217
}
204218

205219
static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
206220
void *control_word, u32 count)
207221
{
208-
if (count == 1) {
209-
aes_crypt(input, output, key, control_word);
222+
u32 initial = count & (ecb_fetch_blocks - 1);
223+
224+
if (count < ecb_fetch_blocks) {
225+
aes_crypt(input, output, key, control_word, count);
210226
return;
211227
}
212228

213-
asm volatile ("test $1, %%cl;"
214-
"je 1f;"
215-
#ifndef CONFIG_X86_64
216-
"lea -1(%%ecx), %%eax;"
217-
"mov $1, %%ecx;"
218-
#else
219-
"lea -1(%%rcx), %%rax;"
220-
"mov $1, %%rcx;"
221-
#endif
222-
".byte 0xf3,0x0f,0xa7,0xc8;" /* rep xcryptecb */
223-
#ifndef CONFIG_X86_64
224-
"mov %%eax, %%ecx;"
225-
#else
226-
"mov %%rax, %%rcx;"
227-
#endif
228-
"1:"
229-
".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
229+
if (initial)
230+
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
231+
: "+S"(input), "+D"(output)
232+
: "d"(control_word), "b"(key), "c"(initial));
233+
234+
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
230235
: "+S"(input), "+D"(output)
231-
: "d"(control_word), "b"(key), "c"(count)
232-
: "ax");
236+
: "d"(control_word), "b"(key), "c"(count - initial));
233237
}
234238

235239
static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
@@ -249,7 +253,7 @@ static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
249253

250254
padlock_reset_key(&ctx->cword.encrypt);
251255
ts_state = irq_ts_save();
252-
aes_crypt(in, out, ctx->E, &ctx->cword.encrypt);
256+
aes_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
253257
irq_ts_restore(ts_state);
254258
padlock_store_cword(&ctx->cword.encrypt);
255259
}
@@ -261,7 +265,7 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
261265

262266
padlock_reset_key(&ctx->cword.encrypt);
263267
ts_state = irq_ts_save();
264-
aes_crypt(in, out, ctx->D, &ctx->cword.decrypt);
268+
aes_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
265269
irq_ts_restore(ts_state);
266270
padlock_store_cword(&ctx->cword.encrypt);
267271
}
@@ -454,6 +458,7 @@ static struct crypto_alg cbc_aes_alg = {
454458
static int __init padlock_init(void)
455459
{
456460
int ret;
461+
struct cpuinfo_x86 *c = &cpu_data(0);
457462

458463
if (!cpu_has_xcrypt) {
459464
printk(KERN_NOTICE PFX "VIA PadLock not detected.\n");
@@ -476,6 +481,12 @@ static int __init padlock_init(void)
476481

477482
printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
478483

484+
if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) {
485+
ecb_fetch_blocks = 8;
486+
cbc_fetch_blocks = 4; /* NOTE: notused */
487+
printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
488+
}
489+
479490
out:
480491
return ret;
481492

0 commit comments

Comments
 (0)