Skip to content

Commit 152d523

Browse files
antonblanchardmpe
antonblanchard
authored and
mpe
committed
powerpc: Create context switch helpers save_sprs() and restore_sprs()
Move all our context switch SPR save and restore code into two helpers. We do a few optimisations: - Group all mfsprs and all mtsprs. In many cases an mtspr sets a scoreboarding bit that an mfspr waits on, so the current practise of mfspr A; mtspr A; mfpsr B; mtspr B is the worst scheduling we can do. - SPR writes are slow, so check that the value is changing before writing it. A context switch microbenchmark using yield(): http://ozlabs.org/~anton/junkcode/context_switch2.c ./context_switch2 --test=yield 0 0 shows an improvement of almost 10% on POWER8. Signed-off-by: Anton Blanchard <anton@samba.org> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
1 parent af72ab6 commit 152d523

File tree

4 files changed

+82
-82
lines changed

4 files changed

+82
-82
lines changed

‎arch/powerpc/include/asm/processor.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ struct thread_struct {
294294
#endif
295295
#ifdef CONFIG_PPC64
296296
unsigned long dscr;
297+
unsigned long fscr;
297298
/*
298299
* This member element dscr_inherit indicates that the process
299300
* has explicitly attempted and changed the DSCR register value

‎arch/powerpc/include/asm/switch_to.h

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,6 @@ extern struct task_struct *__switch_to(struct task_struct *,
1515
struct thread_struct;
1616
extern struct task_struct *_switch(struct thread_struct *prev,
1717
struct thread_struct *next);
18-
#ifdef CONFIG_PPC_BOOK3S_64
19-
static inline void save_early_sprs(struct thread_struct *prev)
20-
{
21-
if (cpu_has_feature(CPU_FTR_ARCH_207S))
22-
prev->tar = mfspr(SPRN_TAR);
23-
if (cpu_has_feature(CPU_FTR_DSCR))
24-
prev->dscr = mfspr(SPRN_DSCR);
25-
}
26-
#else
27-
static inline void save_early_sprs(struct thread_struct *prev) {}
28-
#endif
2918

3019
extern void enable_kernel_fp(void);
3120
extern void enable_kernel_altivec(void);

‎arch/powerpc/kernel/entry_64.S

Lines changed: 1 addition & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -453,29 +453,12 @@ _GLOBAL(_switch)
453453
SAVE_8GPRS(14, r1)
454454
SAVE_10GPRS(22, r1)
455455
mflr r20 /* Return to switch caller */
456-
#ifdef CONFIG_ALTIVEC
457-
BEGIN_FTR_SECTION
458-
mfspr r24,SPRN_VRSAVE /* save vrsave register value */
459-
std r24,THREAD_VRSAVE(r3)
460-
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
461-
#endif /* CONFIG_ALTIVEC */
456+
462457
std r20,_NIP(r1)
463458
mfcr r23
464459
std r23,_CCR(r1)
465460
std r1,KSP(r3) /* Set old stack pointer */
466461

467-
#ifdef CONFIG_PPC_BOOK3S_64
468-
BEGIN_FTR_SECTION
469-
/* Event based branch registers */
470-
mfspr r0, SPRN_BESCR
471-
std r0, THREAD_BESCR(r3)
472-
mfspr r0, SPRN_EBBHR
473-
std r0, THREAD_EBBHR(r3)
474-
mfspr r0, SPRN_EBBRR
475-
std r0, THREAD_EBBRR(r3)
476-
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
477-
#endif
478-
479462
#ifdef CONFIG_SMP
480463
/* We need a sync somewhere here to make sure that if the
481464
* previous task gets rescheduled on another CPU, it sees all
@@ -563,47 +546,6 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
563546
mr r1,r8 /* start using new stack pointer */
564547
std r7,PACAKSAVE(r13)
565548

566-
#ifdef CONFIG_PPC_BOOK3S_64
567-
BEGIN_FTR_SECTION
568-
/* Event based branch registers */
569-
ld r0, THREAD_BESCR(r4)
570-
mtspr SPRN_BESCR, r0
571-
ld r0, THREAD_EBBHR(r4)
572-
mtspr SPRN_EBBHR, r0
573-
ld r0, THREAD_EBBRR(r4)
574-
mtspr SPRN_EBBRR, r0
575-
576-
ld r0,THREAD_TAR(r4)
577-
mtspr SPRN_TAR,r0
578-
END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
579-
#endif
580-
581-
#ifdef CONFIG_ALTIVEC
582-
BEGIN_FTR_SECTION
583-
ld r0,THREAD_VRSAVE(r4)
584-
mtspr SPRN_VRSAVE,r0 /* if G4, restore VRSAVE reg */
585-
END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
586-
#endif /* CONFIG_ALTIVEC */
587-
#ifdef CONFIG_PPC64
588-
BEGIN_FTR_SECTION
589-
lwz r6,THREAD_DSCR_INHERIT(r4)
590-
ld r0,THREAD_DSCR(r4)
591-
cmpwi r6,0
592-
bne 1f
593-
ld r0,PACA_DSCR_DEFAULT(r13)
594-
1:
595-
BEGIN_FTR_SECTION_NESTED(70)
596-
mfspr r8, SPRN_FSCR
597-
rldimi r8, r6, FSCR_DSCR_LG, (63 - FSCR_DSCR_LG)
598-
mtspr SPRN_FSCR, r8
599-
END_FTR_SECTION_NESTED(CPU_FTR_ARCH_207S, CPU_FTR_ARCH_207S, 70)
600-
cmpd r0,r25
601-
beq 2f
602-
mtspr SPRN_DSCR,r0
603-
2:
604-
END_FTR_SECTION_IFSET(CPU_FTR_DSCR)
605-
#endif
606-
607549
ld r6,_CCR(r1)
608550
mtcrf 0xFF,r6
609551

‎arch/powerpc/kernel/process.c

Lines changed: 80 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -742,6 +742,73 @@ void restore_tm_state(struct pt_regs *regs)
742742
#define __switch_to_tm(prev)
743743
#endif /* CONFIG_PPC_TRANSACTIONAL_MEM */
744744

745+
static inline void save_sprs(struct thread_struct *t)
746+
{
747+
#ifdef CONFIG_ALTIVEC
748+
if (cpu_has_feature(cpu_has_feature(CPU_FTR_ALTIVEC)))
749+
t->vrsave = mfspr(SPRN_VRSAVE);
750+
#endif
751+
#ifdef CONFIG_PPC_BOOK3S_64
752+
if (cpu_has_feature(CPU_FTR_DSCR))
753+
t->dscr = mfspr(SPRN_DSCR);
754+
755+
if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
756+
t->bescr = mfspr(SPRN_BESCR);
757+
t->ebbhr = mfspr(SPRN_EBBHR);
758+
t->ebbrr = mfspr(SPRN_EBBRR);
759+
760+
t->fscr = mfspr(SPRN_FSCR);
761+
762+
/*
763+
* Note that the TAR is not available for use in the kernel.
764+
* (To provide this, the TAR should be backed up/restored on
765+
* exception entry/exit instead, and be in pt_regs. FIXME,
766+
* this should be in pt_regs anyway (for debug).)
767+
*/
768+
t->tar = mfspr(SPRN_TAR);
769+
}
770+
#endif
771+
}
772+
773+
static inline void restore_sprs(struct thread_struct *old_thread,
774+
struct thread_struct *new_thread)
775+
{
776+
#ifdef CONFIG_ALTIVEC
777+
if (cpu_has_feature(CPU_FTR_ALTIVEC) &&
778+
old_thread->vrsave != new_thread->vrsave)
779+
mtspr(SPRN_VRSAVE, new_thread->vrsave);
780+
#endif
781+
#ifdef CONFIG_PPC_BOOK3S_64
782+
if (cpu_has_feature(CPU_FTR_DSCR)) {
783+
u64 dscr = get_paca()->dscr_default;
784+
u64 fscr = old_thread->fscr & ~FSCR_DSCR;
785+
786+
if (new_thread->dscr_inherit) {
787+
dscr = new_thread->dscr;
788+
fscr |= FSCR_DSCR;
789+
}
790+
791+
if (old_thread->dscr != dscr)
792+
mtspr(SPRN_DSCR, dscr);
793+
794+
if (old_thread->fscr != fscr)
795+
mtspr(SPRN_FSCR, fscr);
796+
}
797+
798+
if (cpu_has_feature(CPU_FTR_ARCH_207S)) {
799+
if (old_thread->bescr != new_thread->bescr)
800+
mtspr(SPRN_BESCR, new_thread->bescr);
801+
if (old_thread->ebbhr != new_thread->ebbhr)
802+
mtspr(SPRN_EBBHR, new_thread->ebbhr);
803+
if (old_thread->ebbrr != new_thread->ebbrr)
804+
mtspr(SPRN_EBBRR, new_thread->ebbrr);
805+
806+
if (old_thread->tar != new_thread->tar)
807+
mtspr(SPRN_TAR, new_thread->tar);
808+
}
809+
#endif
810+
}
811+
745812
struct task_struct *__switch_to(struct task_struct *prev,
746813
struct task_struct *new)
747814
{
@@ -751,17 +818,16 @@ struct task_struct *__switch_to(struct task_struct *prev,
751818
struct ppc64_tlb_batch *batch;
752819
#endif
753820

821+
new_thread = &new->thread;
822+
old_thread = &current->thread;
823+
754824
WARN_ON(!irqs_disabled());
755825

756-
/* Back up the TAR and DSCR across context switches.
757-
* Note that the TAR is not available for use in the kernel. (To
758-
* provide this, the TAR should be backed up/restored on exception
759-
* entry/exit instead, and be in pt_regs. FIXME, this should be in
760-
* pt_regs anyway (for debug).)
761-
* Save the TAR and DSCR here before we do treclaim/trecheckpoint as
762-
* these will change them.
826+
/*
827+
* We need to save SPRs before treclaim/trecheckpoint as these will
828+
* change a number of them.
763829
*/
764-
save_early_sprs(&prev->thread);
830+
save_sprs(&prev->thread);
765831

766832
__switch_to_tm(prev);
767833

@@ -844,10 +910,6 @@ struct task_struct *__switch_to(struct task_struct *prev,
844910
#endif /* CONFIG_HAVE_HW_BREAKPOINT */
845911
#endif
846912

847-
848-
new_thread = &new->thread;
849-
old_thread = &current->thread;
850-
851913
#ifdef CONFIG_PPC64
852914
/*
853915
* Collect processor utilization data per process
@@ -883,6 +945,10 @@ struct task_struct *__switch_to(struct task_struct *prev,
883945

884946
last = _switch(old_thread, new_thread);
885947

948+
/* Need to recalculate these after calling _switch() */
949+
old_thread = &last->thread;
950+
new_thread = &current->thread;
951+
886952
#ifdef CONFIG_PPC_BOOK3S_64
887953
if (current_thread_info()->local_flags & _TLF_LAZY_MMU) {
888954
current_thread_info()->local_flags &= ~_TLF_LAZY_MMU;
@@ -891,6 +957,8 @@ struct task_struct *__switch_to(struct task_struct *prev,
891957
}
892958
#endif /* CONFIG_PPC_BOOK3S_64 */
893959

960+
restore_sprs(old_thread, new_thread);
961+
894962
return last;
895963
}
896964

0 commit comments

Comments
 (0)