Skip to content

Commit d04c56f

Browse files
paulusmackpaulusmack
committed
[POWERPC] Lazy interrupt disabling for 64-bit machines
This implements a lazy strategy for disabling interrupts. This means that local_irq_disable() et al. just clear the 'interrupts are enabled' flag in the paca. If an interrupt comes along, the interrupt entry code notices that interrupts are supposed to be disabled, and clears the EE bit in SRR1, clears the 'interrupts are hard-enabled' flag in the paca, and returns. This means that interrupts only actually get disabled in the processor when an interrupt comes along. When interrupts are enabled by local_irq_enable() et al., the code sets the interrupts-enabled flag in the paca, and then checks whether interrupts got hard-disabled. If so, it also sets the EE bit in the MSR to hard-enable the interrupts. This has the potential to improve performance, and also makes it easier to make a kernel that can boot on iSeries and on other 64-bit machines, since this lazy-disable strategy is very similar to the soft-disable strategy that iSeries already uses. This version renames paca->proc_enabled to paca->soft_enabled, and changes a couple of soft-disables in the kexec code to hard-disables, which should fix the crash that Michael Ellerman saw. This doesn't yet use a reserved CR field for the soft_enabled and hard_enabled flags. This applies on top of Stephen Rothwell's patches to make it possible to build a combined iSeries/other kernel. Signed-off-by: Paul Mackerras <paulus@samba.org>
1 parent 284a940 commit d04c56f

File tree

12 files changed

+160
-111
lines changed

12 files changed

+160
-111
lines changed

‎arch/powerpc/kernel/asm-offsets.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,8 @@ int main(void)
118118
DEFINE(PACASTABRR, offsetof(struct paca_struct, stab_rr));
119119
DEFINE(PACAR1, offsetof(struct paca_struct, saved_r1));
120120
DEFINE(PACATOC, offsetof(struct paca_struct, kernel_toc));
121-
DEFINE(PACAPROCENABLED, offsetof(struct paca_struct, proc_enabled));
121+
DEFINE(PACASOFTIRQEN, offsetof(struct paca_struct, soft_enabled));
122+
DEFINE(PACAHARDIRQEN, offsetof(struct paca_struct, hard_enabled));
122123
DEFINE(PACASLBCACHE, offsetof(struct paca_struct, slb_cache));
123124
DEFINE(PACASLBCACHEPTR, offsetof(struct paca_struct, slb_cache_ptr));
124125
DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));

‎arch/powerpc/kernel/crash.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ void crash_ipi_callback(struct pt_regs *regs)
111111
if (!cpu_online(cpu))
112112
return;
113113

114-
local_irq_disable();
114+
hard_irq_disable();
115115
if (!cpu_isset(cpu, cpus_in_crash))
116116
crash_save_this_cpu(regs, cpu);
117117
cpu_set(cpu, cpus_in_crash);
@@ -289,7 +289,7 @@ void default_machine_crash_shutdown(struct pt_regs *regs)
289289
* an SMP system.
290290
* The kernel is broken so disable interrupts.
291291
*/
292-
local_irq_disable();
292+
hard_irq_disable();
293293

294294
for_each_irq(irq) {
295295
struct irq_desc *desc = irq_desc + irq;

‎arch/powerpc/kernel/entry_64.S

Lines changed: 18 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -87,15 +87,17 @@ system_call_common:
8787
addi r9,r1,STACK_FRAME_OVERHEAD
8888
ld r11,exception_marker@toc(r2)
8989
std r11,-16(r9) /* "regshere" marker */
90+
li r10,1
91+
stb r10,PACASOFTIRQEN(r13)
92+
stb r10,PACAHARDIRQEN(r13)
93+
std r10,SOFTE(r1)
9094
#ifdef CONFIG_PPC_ISERIES
9195
BEGIN_FW_FTR_SECTION
9296
/* Hack for handling interrupts when soft-enabling on iSeries */
9397
cmpdi cr1,r0,0x5555 /* syscall 0x5555 */
9498
andi. r10,r12,MSR_PR /* from kernel */
9599
crand 4*cr0+eq,4*cr1+eq,4*cr0+eq
96100
beq hardware_interrupt_entry
97-
lbz r10,PACAPROCENABLED(r13)
98-
std r10,SOFTE(r1)
99101
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
100102
#endif
101103
mfmsr r11
@@ -460,9 +462,9 @@ _GLOBAL(ret_from_except_lite)
460462
#endif
461463

462464
restore:
465+
ld r5,SOFTE(r1)
463466
#ifdef CONFIG_PPC_ISERIES
464467
BEGIN_FW_FTR_SECTION
465-
ld r5,SOFTE(r1)
466468
cmpdi 0,r5,0
467469
beq 4f
468470
/* Check for pending interrupts (iSeries) */
@@ -472,16 +474,16 @@ BEGIN_FW_FTR_SECTION
472474
beq+ 4f /* skip do_IRQ if no interrupts */
473475

474476
li r3,0
475-
stb r3,PACAPROCENABLED(r13) /* ensure we are soft-disabled */
477+
stb r3,PACASOFTIRQEN(r13) /* ensure we are soft-disabled */
476478
ori r10,r10,MSR_EE
477479
mtmsrd r10 /* hard-enable again */
478480
addi r3,r1,STACK_FRAME_OVERHEAD
479481
bl .do_IRQ
480482
b .ret_from_except_lite /* loop back and handle more */
481-
482-
4: stb r5,PACAPROCENABLED(r13)
483+
4:
483484
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
484485
#endif
486+
stb r5,PACASOFTIRQEN(r13)
485487

486488
ld r3,_MSR(r1)
487489
andi. r0,r3,MSR_RI
@@ -538,25 +540,15 @@ do_work:
538540
/* Check that preempt_count() == 0 and interrupts are enabled */
539541
lwz r8,TI_PREEMPT(r9)
540542
cmpwi cr1,r8,0
541-
#ifdef CONFIG_PPC_ISERIES
542-
BEGIN_FW_FTR_SECTION
543543
ld r0,SOFTE(r1)
544544
cmpdi r0,0
545-
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
546-
#endif
547-
BEGIN_FW_FTR_SECTION
548-
andi. r0,r3,MSR_EE
549-
END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
550545
crandc eq,cr1*4+eq,eq
551546
bne restore
552547
/* here we are preempting the current task */
553548
1:
554-
#ifdef CONFIG_PPC_ISERIES
555-
BEGIN_FW_FTR_SECTION
556549
li r0,1
557-
stb r0,PACAPROCENABLED(r13)
558-
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
559-
#endif
550+
stb r0,PACASOFTIRQEN(r13)
551+
stb r0,PACAHARDIRQEN(r13)
560552
ori r10,r10,MSR_EE
561553
mtmsrd r10,1 /* reenable interrupts */
562554
bl .preempt_schedule
@@ -639,8 +631,7 @@ _GLOBAL(enter_rtas)
639631
/* There is no way it is acceptable to get here with interrupts enabled,
640632
* check it with the asm equivalent of WARN_ON
641633
*/
642-
mfmsr r6
643-
andi. r0,r6,MSR_EE
634+
lbz r0,PACASOFTIRQEN(r13)
644635
1: tdnei r0,0
645636
.section __bug_table,"a"
646637
.llong 1b,__LINE__ + 0x1000000, 1f, 2f
@@ -649,7 +640,13 @@ _GLOBAL(enter_rtas)
649640
1: .asciz __FILE__
650641
2: .asciz "enter_rtas"
651642
.previous
652-
643+
644+
/* Hard-disable interrupts */
645+
mfmsr r6
646+
rldicl r7,r6,48,1
647+
rotldi r7,r7,16
648+
mtmsrd r7,1
649+
653650
/* Unfortunately, the stack pointer and the MSR are also clobbered,
654651
* so they are saved in the PACA which allows us to restore
655652
* our original state after RTAS returns.

‎arch/powerpc/kernel/head_64.S

Lines changed: 77 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,7 @@
3535
#include <asm/thread_info.h>
3636
#include <asm/firmware.h>
3737

38-
#ifdef CONFIG_PPC_ISERIES
3938
#define DO_SOFT_DISABLE
40-
#endif
4139

4240
/*
4341
* We layout physical memory as follows:
@@ -308,7 +306,9 @@ exception_marker:
308306
std r9,_LINK(r1); \
309307
mfctr r10; /* save CTR in stackframe */ \
310308
std r10,_CTR(r1); \
309+
lbz r10,PACASOFTIRQEN(r13); \
311310
mfspr r11,SPRN_XER; /* save XER in stackframe */ \
311+
std r10,SOFTE(r1); \
312312
std r11,_XER(r1); \
313313
li r9,(n)+1; \
314314
std r9,_TRAP(r1); /* set trap number */ \
@@ -343,6 +343,34 @@ label##_pSeries: \
343343
EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
344344

345345

346+
#define MASKABLE_EXCEPTION_PSERIES(n, label) \
347+
. = n; \
348+
.globl label##_pSeries; \
349+
label##_pSeries: \
350+
HMT_MEDIUM; \
351+
mtspr SPRN_SPRG1,r13; /* save r13 */ \
352+
mfspr r13,SPRN_SPRG3; /* get paca address into r13 */ \
353+
std r9,PACA_EXGEN+EX_R9(r13); /* save r9, r10 */ \
354+
std r10,PACA_EXGEN+EX_R10(r13); \
355+
lbz r10,PACASOFTIRQEN(r13); \
356+
mfcr r9; \
357+
cmpwi r10,0; \
358+
beq masked_interrupt; \
359+
mfspr r10,SPRN_SPRG1; \
360+
std r10,PACA_EXGEN+EX_R13(r13); \
361+
std r11,PACA_EXGEN+EX_R11(r13); \
362+
std r12,PACA_EXGEN+EX_R12(r13); \
363+
clrrdi r12,r13,32; /* get high part of &label */ \
364+
mfmsr r10; \
365+
mfspr r11,SPRN_SRR0; /* save SRR0 */ \
366+
LOAD_HANDLER(r12,label##_common) \
367+
ori r10,r10,MSR_IR|MSR_DR|MSR_RI; \
368+
mtspr SPRN_SRR0,r12; \
369+
mfspr r12,SPRN_SRR1; /* and SRR1 */ \
370+
mtspr SPRN_SRR1,r10; \
371+
rfid; \
372+
b . /* prevent speculative execution */
373+
346374
#define STD_EXCEPTION_ISERIES(n, label, area) \
347375
.globl label##_iSeries; \
348376
label##_iSeries: \
@@ -358,49 +386,39 @@ label##_iSeries: \
358386
HMT_MEDIUM; \
359387
mtspr SPRN_SPRG1,r13; /* save r13 */ \
360388
EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN); \
361-
lbz r10,PACAPROCENABLED(r13); \
389+
lbz r10,PACASOFTIRQEN(r13); \
362390
cmpwi 0,r10,0; \
363391
beq- label##_iSeries_masked; \
364392
EXCEPTION_PROLOG_ISERIES_2; \
365393
b label##_common; \
366394

367-
#ifdef DO_SOFT_DISABLE
395+
#ifdef CONFIG_PPC_ISERIES
368396
#define DISABLE_INTS \
369-
BEGIN_FW_FTR_SECTION; \
370-
lbz r10,PACAPROCENABLED(r13); \
371397
li r11,0; \
372-
std r10,SOFTE(r1); \
398+
stb r11,PACASOFTIRQEN(r13); \
399+
BEGIN_FW_FTR_SECTION; \
400+
stb r11,PACAHARDIRQEN(r13); \
401+
END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES); \
402+
BEGIN_FW_FTR_SECTION; \
373403
mfmsr r10; \
374-
stb r11,PACAPROCENABLED(r13); \
375404
ori r10,r10,MSR_EE; \
376405
mtmsrd r10,1; \
377406
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
378407

379-
#define ENABLE_INTS \
380-
BEGIN_FW_FTR_SECTION; \
381-
lbz r10,PACAPROCENABLED(r13); \
382-
mfmsr r11; \
383-
std r10,SOFTE(r1); \
384-
ori r11,r11,MSR_EE; \
385-
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES); \
386-
BEGIN_FW_FTR_SECTION; \
387-
ld r12,_MSR(r1); \
388-
mfmsr r11; \
389-
rlwimi r11,r12,0,MSR_EE; \
390-
END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES); \
391-
mtmsrd r11,1
408+
#else
409+
#define DISABLE_INTS \
410+
li r11,0; \
411+
stb r11,PACASOFTIRQEN(r13); \
412+
stb r11,PACAHARDIRQEN(r13)
392413

393-
#else /* hard enable/disable interrupts */
394-
#define DISABLE_INTS
414+
#endif /* CONFIG_PPC_ISERIES */
395415

396416
#define ENABLE_INTS \
397417
ld r12,_MSR(r1); \
398418
mfmsr r11; \
399419
rlwimi r11,r12,0,MSR_EE; \
400420
mtmsrd r11,1
401421

402-
#endif
403-
404422
#define STD_EXCEPTION_COMMON(trap, label, hdlr) \
405423
.align 7; \
406424
.globl label##_common; \
@@ -541,11 +559,11 @@ instruction_access_slb_pSeries:
541559
mfspr r12,SPRN_SRR1 /* and SRR1 */
542560
b .slb_miss_realmode /* Rel. branch works in real mode */
543561

544-
STD_EXCEPTION_PSERIES(0x500, hardware_interrupt)
562+
MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)
545563
STD_EXCEPTION_PSERIES(0x600, alignment)
546564
STD_EXCEPTION_PSERIES(0x700, program_check)
547565
STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
548-
STD_EXCEPTION_PSERIES(0x900, decrementer)
566+
MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)
549567
STD_EXCEPTION_PSERIES(0xa00, trap_0a)
550568
STD_EXCEPTION_PSERIES(0xb00, trap_0b)
551569

@@ -597,7 +615,24 @@ system_call_pSeries:
597615
/*** pSeries interrupt support ***/
598616

599617
/* moved from 0xf00 */
600-
STD_EXCEPTION_PSERIES(., performance_monitor)
618+
MASKABLE_EXCEPTION_PSERIES(., performance_monitor)
619+
620+
/*
621+
* An interrupt came in while soft-disabled; clear EE in SRR1,
622+
* clear paca->hard_enabled and return.
623+
*/
624+
masked_interrupt:
625+
stb r10,PACAHARDIRQEN(r13)
626+
mtcrf 0x80,r9
627+
ld r9,PACA_EXGEN+EX_R9(r13)
628+
mfspr r10,SPRN_SRR1
629+
rldicl r10,r10,48,1 /* clear MSR_EE */
630+
rotldi r10,r10,16
631+
mtspr SPRN_SRR1,r10
632+
ld r10,PACA_EXGEN+EX_R10(r13)
633+
mfspr r13,SPRN_SPRG1
634+
rfid
635+
b .
601636

602637
.align 7
603638
_GLOBAL(do_stab_bolted_pSeries)
@@ -952,7 +987,8 @@ fast_exception_return:
952987
REST_8GPRS(2, r1)
953988

954989
mfmsr r10
955-
clrrdi r10,r10,2 /* clear RI (LE is 0 already) */
990+
rldicl r10,r10,48,1 /* clear EE */
991+
rldicr r10,r10,16,61 /* clear RI (LE is 0 already) */
956992
mtmsrd r10,1
957993

958994
mtspr SPRN_SRR1,r12
@@ -1877,11 +1913,16 @@ _GLOBAL(__secondary_start)
18771913
/* enable MMU and jump to start_secondary */
18781914
LOAD_REG_ADDR(r3, .start_secondary_prolog)
18791915
LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
1880-
#ifdef DO_SOFT_DISABLE
1916+
#ifdef CONFIG_PPC_ISERIES
18811917
BEGIN_FW_FTR_SECTION
18821918
ori r4,r4,MSR_EE
18831919
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
18841920
#endif
1921+
BEGIN_FW_FTR_SECTION
1922+
stb r7,PACASOFTIRQEN(r13)
1923+
stb r7,PACAHARDIRQEN(r13)
1924+
END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
1925+
18851926
mtspr SPRN_SRR0,r3
18861927
mtspr SPRN_SRR1,r4
18871928
rfid
@@ -2019,15 +2060,18 @@ _STATIC(start_here_common)
20192060

20202061
/* Load up the kernel context */
20212062
5:
2022-
#ifdef DO_SOFT_DISABLE
2023-
BEGIN_FW_FTR_SECTION
20242063
li r5,0
2025-
stb r5,PACAPROCENABLED(r13) /* Soft Disabled */
2064+
stb r5,PACASOFTIRQEN(r13) /* Soft Disabled */
2065+
#ifdef CONFIG_PPC_ISERIES
2066+
BEGIN_FW_FTR_SECTION
20262067
mfmsr r5
20272068
ori r5,r5,MSR_EE /* Hard Enabled */
20282069
mtmsrd r5
20292070
END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
20302071
#endif
2072+
BEGIN_FW_FTR_SECTION
2073+
stb r5,PACAHARDIRQEN(r13)
2074+
END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
20312075

20322076
bl .start_kernel
20332077

‎arch/powerpc/kernel/idle_power4.S

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,13 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CAN_NAP)
3030
beqlr
3131

3232
/* Go to NAP now */
33+
mfmsr r7
34+
rldicl r0,r7,48,1
35+
rotldi r0,r0,16
36+
mtmsrd r0,1 /* hard-disable interrupts */
37+
li r0,1
38+
stb r0,PACASOFTIRQEN(r13) /* we'll hard-enable shortly */
39+
stb r0,PACAHARDIRQEN(r13)
3340
BEGIN_FTR_SECTION
3441
DSSALL
3542
sync
@@ -38,7 +45,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
3845
ld r8,TI_LOCAL_FLAGS(r9) /* set napping bit */
3946
ori r8,r8,_TLF_NAPPING /* so when we take an exception */
4047
std r8,TI_LOCAL_FLAGS(r9) /* it will return to our caller */
41-
mfmsr r7
4248
ori r7,r7,MSR_EE
4349
oris r7,r7,MSR_POW@h
4450
1: sync

0 commit comments

Comments
 (0)