Skip to content

Commit f01af9f

Browse files
torvaldstorvalds
committed
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc
Pull sparc fixes from David Miller: "Please pull to get these sparc AES/DES/CAMELLIA crypto bug fixes as well as an addition of a pte_accessible() define for sparc64 and a hugetlb fix from Dave Kleikamp." * git://git.kernel.org/pub/scm/linux/kernel/git/davem/sparc: sparc64: Set CRYPTO_TFM_REQ_MAY_SLEEP consistently in CAMELLIA code. sparc64: Set CRYPTO_TFM_REQ_MAY_SLEEP consistently in DES code. sparc64: Fix ECB looping constructs in AES code. sparc64: Set CRYPTO_TFM_REQ_MAY_SLEEP consistently in AES code. sparc64: Fix AES ctr mode block size. sparc64: Fix unrolled AES 256-bit key loops. sparc64: Define pte_accessible() sparc: huge_ptep_set_* functions need to call set_huge_pte_at()
2 parents 9eb127c + 62ba63d commit f01af9f

File tree

7 files changed

+67
-12
lines changed

7 files changed

+67
-12
lines changed

‎arch/sparc/crypto/aes_asm.S

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1024,7 +1024,11 @@ ENTRY(aes_sparc64_ecb_encrypt_256)
10241024
add %o2, 0x20, %o2
10251025
brlz,pt %o3, 11f
10261026
nop
1027-
10: ldx [%o1 + 0x00], %g3
1027+
10: ldd [%o0 + 0xd0], %f56
1028+
ldd [%o0 + 0xd8], %f58
1029+
ldd [%o0 + 0xe0], %f60
1030+
ldd [%o0 + 0xe8], %f62
1031+
ldx [%o1 + 0x00], %g3
10281032
ldx [%o1 + 0x08], %g7
10291033
xor %g1, %g3, %g3
10301034
xor %g2, %g7, %g7
@@ -1128,9 +1132,9 @@ ENTRY(aes_sparc64_ecb_decrypt_256)
11281132
/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
11291133
ldx [%o0 - 0x10], %g1
11301134
subcc %o3, 0x10, %o3
1135+
ldx [%o0 - 0x08], %g2
11311136
be 10f
1132-
ldx [%o0 - 0x08], %g2
1133-
sub %o0, 0xf0, %o0
1137+
sub %o0, 0xf0, %o0
11341138
1: ldx [%o1 + 0x00], %g3
11351139
ldx [%o1 + 0x08], %g7
11361140
ldx [%o1 + 0x10], %o4
@@ -1154,7 +1158,11 @@ ENTRY(aes_sparc64_ecb_decrypt_256)
11541158
add %o2, 0x20, %o2
11551159
brlz,pt %o3, 11f
11561160
nop
1157-
10: ldx [%o1 + 0x00], %g3
1161+
10: ldd [%o0 + 0x18], %f56
1162+
ldd [%o0 + 0x10], %f58
1163+
ldd [%o0 + 0x08], %f60
1164+
ldd [%o0 + 0x00], %f62
1165+
ldx [%o1 + 0x00], %g3
11581166
ldx [%o1 + 0x08], %g7
11591167
xor %g1, %g3, %g3
11601168
xor %g2, %g7, %g7
@@ -1511,11 +1519,11 @@ ENTRY(aes_sparc64_ctr_crypt_256)
15111519
add %o2, 0x20, %o2
15121520
brlz,pt %o3, 11f
15131521
nop
1514-
ldd [%o0 + 0xd0], %f56
1522+
10: ldd [%o0 + 0xd0], %f56
15151523
ldd [%o0 + 0xd8], %f58
15161524
ldd [%o0 + 0xe0], %f60
15171525
ldd [%o0 + 0xe8], %f62
1518-
10: xor %g1, %g3, %o5
1526+
xor %g1, %g3, %o5
15191527
MOVXTOD_O5_F0
15201528
xor %g2, %g7, %o5
15211529
MOVXTOD_O5_F2

‎arch/sparc/crypto/aes_glue.c

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,7 @@ static int ecb_encrypt(struct blkcipher_desc *desc,
222222

223223
blkcipher_walk_init(&walk, dst, src, nbytes);
224224
err = blkcipher_walk_virt(desc, &walk);
225+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
225226

226227
ctx->ops->load_encrypt_keys(&ctx->key[0]);
227228
while ((nbytes = walk.nbytes)) {
@@ -251,6 +252,7 @@ static int ecb_decrypt(struct blkcipher_desc *desc,
251252

252253
blkcipher_walk_init(&walk, dst, src, nbytes);
253254
err = blkcipher_walk_virt(desc, &walk);
255+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
254256

255257
ctx->ops->load_decrypt_keys(&ctx->key[0]);
256258
key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
@@ -280,6 +282,7 @@ static int cbc_encrypt(struct blkcipher_desc *desc,
280282

281283
blkcipher_walk_init(&walk, dst, src, nbytes);
282284
err = blkcipher_walk_virt(desc, &walk);
285+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
283286

284287
ctx->ops->load_encrypt_keys(&ctx->key[0]);
285288
while ((nbytes = walk.nbytes)) {
@@ -309,6 +312,7 @@ static int cbc_decrypt(struct blkcipher_desc *desc,
309312

310313
blkcipher_walk_init(&walk, dst, src, nbytes);
311314
err = blkcipher_walk_virt(desc, &walk);
315+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
312316

313317
ctx->ops->load_decrypt_keys(&ctx->key[0]);
314318
key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
@@ -329,6 +333,22 @@ static int cbc_decrypt(struct blkcipher_desc *desc,
329333
return err;
330334
}
331335

336+
static void ctr_crypt_final(struct crypto_sparc64_aes_ctx *ctx,
337+
struct blkcipher_walk *walk)
338+
{
339+
u8 *ctrblk = walk->iv;
340+
u64 keystream[AES_BLOCK_SIZE / sizeof(u64)];
341+
u8 *src = walk->src.virt.addr;
342+
u8 *dst = walk->dst.virt.addr;
343+
unsigned int nbytes = walk->nbytes;
344+
345+
ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
346+
keystream, AES_BLOCK_SIZE);
347+
crypto_xor((u8 *) keystream, src, nbytes);
348+
memcpy(dst, keystream, nbytes);
349+
crypto_inc(ctrblk, AES_BLOCK_SIZE);
350+
}
351+
332352
static int ctr_crypt(struct blkcipher_desc *desc,
333353
struct scatterlist *dst, struct scatterlist *src,
334354
unsigned int nbytes)
@@ -338,10 +358,11 @@ static int ctr_crypt(struct blkcipher_desc *desc,
338358
int err;
339359

340360
blkcipher_walk_init(&walk, dst, src, nbytes);
341-
err = blkcipher_walk_virt(desc, &walk);
361+
err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
362+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
342363

343364
ctx->ops->load_encrypt_keys(&ctx->key[0]);
344-
while ((nbytes = walk.nbytes)) {
365+
while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
345366
unsigned int block_len = nbytes & AES_BLOCK_MASK;
346367

347368
if (likely(block_len)) {
@@ -353,6 +374,10 @@ static int ctr_crypt(struct blkcipher_desc *desc,
353374
nbytes &= AES_BLOCK_SIZE - 1;
354375
err = blkcipher_walk_done(desc, &walk, nbytes);
355376
}
377+
if (walk.nbytes) {
378+
ctr_crypt_final(ctx, &walk);
379+
err = blkcipher_walk_done(desc, &walk, 0);
380+
}
356381
fprs_write(0);
357382
return err;
358383
}
@@ -418,7 +443,7 @@ static struct crypto_alg algs[] = { {
418443
.cra_driver_name = "ctr-aes-sparc64",
419444
.cra_priority = SPARC_CR_OPCODE_PRIORITY,
420445
.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
421-
.cra_blocksize = AES_BLOCK_SIZE,
446+
.cra_blocksize = 1,
422447
.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
423448
.cra_alignmask = 7,
424449
.cra_type = &crypto_blkcipher_type,

‎arch/sparc/crypto/camellia_glue.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ static int __ecb_crypt(struct blkcipher_desc *desc,
9898

9999
blkcipher_walk_init(&walk, dst, src, nbytes);
100100
err = blkcipher_walk_virt(desc, &walk);
101+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
101102

102103
if (encrypt)
103104
key = &ctx->encrypt_key[0];
@@ -160,6 +161,7 @@ static int cbc_encrypt(struct blkcipher_desc *desc,
160161

161162
blkcipher_walk_init(&walk, dst, src, nbytes);
162163
err = blkcipher_walk_virt(desc, &walk);
164+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
163165

164166
key = &ctx->encrypt_key[0];
165167
camellia_sparc64_load_keys(key, ctx->key_len);
@@ -198,6 +200,7 @@ static int cbc_decrypt(struct blkcipher_desc *desc,
198200

199201
blkcipher_walk_init(&walk, dst, src, nbytes);
200202
err = blkcipher_walk_virt(desc, &walk);
203+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
201204

202205
key = &ctx->decrypt_key[0];
203206
camellia_sparc64_load_keys(key, ctx->key_len);

‎arch/sparc/crypto/des_asm.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,7 @@ ENTRY(des3_ede_sparc64_ecb_crypt)
376376
1: ldd [%o1 + 0x00], %f60
377377
DES3_LOOP_BODY(60)
378378
std %f60, [%o2 + 0x00]
379+
add %o1, 0x08, %o1
379380
subcc %o3, 0x08, %o3
380381
bne,pt %icc, 1b
381382
add %o2, 0x08, %o2

‎arch/sparc/crypto/des_glue.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@ static int __ecb_crypt(struct blkcipher_desc *desc,
100100

101101
blkcipher_walk_init(&walk, dst, src, nbytes);
102102
err = blkcipher_walk_virt(desc, &walk);
103+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
103104

104105
if (encrypt)
105106
des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
@@ -147,6 +148,7 @@ static int cbc_encrypt(struct blkcipher_desc *desc,
147148

148149
blkcipher_walk_init(&walk, dst, src, nbytes);
149150
err = blkcipher_walk_virt(desc, &walk);
151+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
150152

151153
des_sparc64_load_keys(&ctx->encrypt_expkey[0]);
152154
while ((nbytes = walk.nbytes)) {
@@ -177,6 +179,7 @@ static int cbc_decrypt(struct blkcipher_desc *desc,
177179

178180
blkcipher_walk_init(&walk, dst, src, nbytes);
179181
err = blkcipher_walk_virt(desc, &walk);
182+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
180183

181184
des_sparc64_load_keys(&ctx->decrypt_expkey[0]);
182185
while ((nbytes = walk.nbytes)) {
@@ -266,6 +269,7 @@ static int __ecb3_crypt(struct blkcipher_desc *desc,
266269

267270
blkcipher_walk_init(&walk, dst, src, nbytes);
268271
err = blkcipher_walk_virt(desc, &walk);
272+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
269273

270274
if (encrypt)
271275
K = &ctx->encrypt_expkey[0];
@@ -317,6 +321,7 @@ static int cbc3_encrypt(struct blkcipher_desc *desc,
317321

318322
blkcipher_walk_init(&walk, dst, src, nbytes);
319323
err = blkcipher_walk_virt(desc, &walk);
324+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
320325

321326
K = &ctx->encrypt_expkey[0];
322327
des3_ede_sparc64_load_keys(K);
@@ -352,6 +357,7 @@ static int cbc3_decrypt(struct blkcipher_desc *desc,
352357

353358
blkcipher_walk_init(&walk, dst, src, nbytes);
354359
err = blkcipher_walk_virt(desc, &walk);
360+
desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
355361

356362
K = &ctx->decrypt_expkey[0];
357363
des3_ede_sparc64_load_keys(K);

‎arch/sparc/include/asm/hugetlb.h

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,14 +61,20 @@ static inline pte_t huge_pte_wrprotect(pte_t pte)
6161
static inline void huge_ptep_set_wrprotect(struct mm_struct *mm,
6262
unsigned long addr, pte_t *ptep)
6363
{
64-
ptep_set_wrprotect(mm, addr, ptep);
64+
pte_t old_pte = *ptep;
65+
set_huge_pte_at(mm, addr, ptep, pte_wrprotect(old_pte));
6566
}
6667

6768
static inline int huge_ptep_set_access_flags(struct vm_area_struct *vma,
6869
unsigned long addr, pte_t *ptep,
6970
pte_t pte, int dirty)
7071
{
71-
return ptep_set_access_flags(vma, addr, ptep, pte, dirty);
72+
int changed = !pte_same(*ptep, pte);
73+
if (changed) {
74+
set_huge_pte_at(vma->vm_mm, addr, ptep, pte);
75+
flush_tlb_page(vma, addr);
76+
}
77+
return changed;
7278
}
7379

7480
static inline pte_t huge_ptep_get(pte_t *ptep)

‎arch/sparc/include/asm/pgtable_64.h

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -617,6 +617,12 @@ static inline unsigned long pte_present(pte_t pte)
617617
return val;
618618
}
619619

620+
#define pte_accessible pte_accessible
621+
static inline unsigned long pte_accessible(pte_t a)
622+
{
623+
return pte_val(a) & _PAGE_VALID;
624+
}
625+
620626
static inline unsigned long pte_special(pte_t pte)
621627
{
622628
return pte_val(pte) & _PAGE_SPECIAL;
@@ -802,7 +808,7 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
802808
* SUN4V NOTE: _PAGE_VALID is the same value in both the SUN4U
803809
* and SUN4V pte layout, so this inline test is fine.
804810
*/
805-
if (likely(mm != &init_mm) && (pte_val(orig) & _PAGE_VALID))
811+
if (likely(mm != &init_mm) && pte_accessible(orig))
806812
tlb_batch_add(mm, addr, ptep, orig, fullmm);
807813
}
808814

0 commit comments

Comments
 (0)