Skip to content

Commit 375074c

Browse files
amlutoIngo Molnar
amluto
authored and
Ingo Molnar
committed
x86: Clean up cr4 manipulation
CR4 manipulation was split, seemingly at random, between direct (write_cr4) and using a helper (set/clear_in_cr4). Unfortunately, the set_in_cr4 and clear_in_cr4 helpers also poke at the boot code, which only a small subset of users actually wanted. This patch replaces all cr4 access in functions that don't leave cr4 exactly the way they found it with new helpers cr4_set_bits, cr4_clear_bits, and cr4_set_bits_and_update_boot. Signed-off-by: Andy Lutomirski <luto@amacapital.net> Reviewed-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Andrea Arcangeli <aarcange@redhat.com> Cc: Vince Weaver <vince@deater.net> Cc: "hillf.zj" <hillf.zj@alibaba-inc.com> Cc: Valdis Kletnieks <Valdis.Kletnieks@vt.edu> Cc: Paul Mackerras <paulus@samba.org> Cc: Arnaldo Carvalho de Melo <acme@kernel.org> Cc: Kees Cook <keescook@chromium.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Link: http://lkml.kernel.org/r/495a10bdc9e67016b8fd3945700d46cfd5c12c2f.1414190806.git.luto@amacapital.net Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent 0967160 commit 375074c

File tree

15 files changed

+70
-57
lines changed

15 files changed

+70
-57
lines changed

‎arch/x86/include/asm/processor.h

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -579,39 +579,6 @@ static inline void load_sp0(struct tss_struct *tss,
579579
#define set_iopl_mask native_set_iopl_mask
580580
#endif /* CONFIG_PARAVIRT */
581581

582-
/*
583-
* Save the cr4 feature set we're using (ie
584-
* Pentium 4MB enable and PPro Global page
585-
* enable), so that any CPU's that boot up
586-
* after us can get the correct flags.
587-
*/
588-
extern unsigned long mmu_cr4_features;
589-
extern u32 *trampoline_cr4_features;
590-
591-
static inline void set_in_cr4(unsigned long mask)
592-
{
593-
unsigned long cr4;
594-
595-
mmu_cr4_features |= mask;
596-
if (trampoline_cr4_features)
597-
*trampoline_cr4_features = mmu_cr4_features;
598-
cr4 = read_cr4();
599-
cr4 |= mask;
600-
write_cr4(cr4);
601-
}
602-
603-
static inline void clear_in_cr4(unsigned long mask)
604-
{
605-
unsigned long cr4;
606-
607-
mmu_cr4_features &= ~mask;
608-
if (trampoline_cr4_features)
609-
*trampoline_cr4_features = mmu_cr4_features;
610-
cr4 = read_cr4();
611-
cr4 &= ~mask;
612-
write_cr4(cr4);
613-
}
614-
615582
typedef struct {
616583
unsigned long seg;
617584
} mm_segment_t;

‎arch/x86/include/asm/tlbflush.h

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,43 @@
1515
#define __flush_tlb_single(addr) __native_flush_tlb_single(addr)
1616
#endif
1717

18+
/* Set in this cpu's CR4. */
19+
static inline void cr4_set_bits(unsigned long mask)
20+
{
21+
unsigned long cr4;
22+
23+
cr4 = read_cr4();
24+
cr4 |= mask;
25+
write_cr4(cr4);
26+
}
27+
28+
/* Clear in this cpu's CR4. */
29+
static inline void cr4_clear_bits(unsigned long mask)
30+
{
31+
unsigned long cr4;
32+
33+
cr4 = read_cr4();
34+
cr4 &= ~mask;
35+
write_cr4(cr4);
36+
}
37+
38+
/*
39+
* Save some of cr4 feature set we're using (e.g. Pentium 4MB
40+
* enable and PPro Global page enable), so that any CPU's that boot
41+
* up after us can get the correct flags. This should only be used
42+
* during boot on the boot cpu.
43+
*/
44+
extern unsigned long mmu_cr4_features;
45+
extern u32 *trampoline_cr4_features;
46+
47+
static inline void cr4_set_bits_and_update_boot(unsigned long mask)
48+
{
49+
mmu_cr4_features |= mask;
50+
if (trampoline_cr4_features)
51+
*trampoline_cr4_features = mmu_cr4_features;
52+
cr4_set_bits(mask);
53+
}
54+
1855
static inline void __native_flush_tlb(void)
1956
{
2057
native_write_cr3(native_read_cr3());

‎arch/x86/include/asm/virtext.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
#include <asm/vmx.h>
2121
#include <asm/svm.h>
22+
#include <asm/tlbflush.h>
2223

2324
/*
2425
* VMX functions:
@@ -40,7 +41,7 @@ static inline int cpu_has_vmx(void)
4041
static inline void cpu_vmxoff(void)
4142
{
4243
asm volatile (ASM_VMX_VMXOFF : : : "cc");
43-
write_cr4(read_cr4() & ~X86_CR4_VMXE);
44+
cr4_clear_bits(X86_CR4_VMXE);
4445
}
4546

4647
static inline int cpu_vmx_enabled(void)

‎arch/x86/kernel/cpu/common.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@ __setup("nosmep", setup_disable_smep);
278278
static __always_inline void setup_smep(struct cpuinfo_x86 *c)
279279
{
280280
if (cpu_has(c, X86_FEATURE_SMEP))
281-
set_in_cr4(X86_CR4_SMEP);
281+
cr4_set_bits(X86_CR4_SMEP);
282282
}
283283

284284
static __init int setup_disable_smap(char *arg)
@@ -298,9 +298,9 @@ static __always_inline void setup_smap(struct cpuinfo_x86 *c)
298298

299299
if (cpu_has(c, X86_FEATURE_SMAP)) {
300300
#ifdef CONFIG_X86_SMAP
301-
set_in_cr4(X86_CR4_SMAP);
301+
cr4_set_bits(X86_CR4_SMAP);
302302
#else
303-
clear_in_cr4(X86_CR4_SMAP);
303+
cr4_clear_bits(X86_CR4_SMAP);
304304
#endif
305305
}
306306
}
@@ -1312,7 +1312,7 @@ void cpu_init(void)
13121312

13131313
pr_debug("Initializing CPU#%d\n", cpu);
13141314

1315-
clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1315+
cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
13161316

13171317
/*
13181318
* Initialize the per-CPU GDT with the boot GDT,
@@ -1393,7 +1393,7 @@ void cpu_init(void)
13931393
printk(KERN_INFO "Initializing CPU#%d\n", cpu);
13941394

13951395
if (cpu_feature_enabled(X86_FEATURE_VME) || cpu_has_tsc || cpu_has_de)
1396-
clear_in_cr4(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
1396+
cr4_clear_bits(X86_CR4_VME|X86_CR4_PVI|X86_CR4_TSD|X86_CR4_DE);
13971397

13981398
load_current_idt();
13991399
switch_to_new_gdt(cpu);

‎arch/x86/kernel/cpu/mcheck/mce.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444

4545
#include <asm/processor.h>
4646
#include <asm/traps.h>
47+
#include <asm/tlbflush.h>
4748
#include <asm/mce.h>
4849
#include <asm/msr.h>
4950

@@ -1449,7 +1450,7 @@ static void __mcheck_cpu_init_generic(void)
14491450
bitmap_fill(all_banks, MAX_NR_BANKS);
14501451
machine_check_poll(MCP_UC | m_fl, &all_banks);
14511452

1452-
set_in_cr4(X86_CR4_MCE);
1453+
cr4_set_bits(X86_CR4_MCE);
14531454

14541455
rdmsrl(MSR_IA32_MCG_CAP, cap);
14551456
if (cap & MCG_CTL_P)

‎arch/x86/kernel/cpu/mcheck/p5.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
#include <asm/processor.h>
1111
#include <asm/traps.h>
12+
#include <asm/tlbflush.h>
1213
#include <asm/mce.h>
1314
#include <asm/msr.h>
1415

@@ -65,7 +66,7 @@ void intel_p5_mcheck_init(struct cpuinfo_x86 *c)
6566
"Intel old style machine check architecture supported.\n");
6667

6768
/* Enable MCE: */
68-
set_in_cr4(X86_CR4_MCE);
69+
cr4_set_bits(X86_CR4_MCE);
6970
printk(KERN_INFO
7071
"Intel old style machine check reporting enabled on CPU#%d.\n",
7172
smp_processor_id());

‎arch/x86/kernel/cpu/mcheck/winchip.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
#include <asm/processor.h>
1010
#include <asm/traps.h>
11+
#include <asm/tlbflush.h>
1112
#include <asm/mce.h>
1213
#include <asm/msr.h>
1314

@@ -36,7 +37,7 @@ void winchip_mcheck_init(struct cpuinfo_x86 *c)
3637
lo &= ~(1<<4); /* Enable MCE */
3738
wrmsr(MSR_IDT_FCR1, lo, hi);
3839

39-
set_in_cr4(X86_CR4_MCE);
40+
cr4_set_bits(X86_CR4_MCE);
4041

4142
printk(KERN_INFO
4243
"Winchip machine check reporting enabled on CPU#0.\n");

‎arch/x86/kernel/cpu/perf_event.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include <asm/nmi.h>
3232
#include <asm/smp.h>
3333
#include <asm/alternative.h>
34+
#include <asm/tlbflush.h>
3435
#include <asm/timer.h>
3536
#include <asm/desc.h>
3637
#include <asm/ldt.h>
@@ -1328,7 +1329,7 @@ x86_pmu_notifier(struct notifier_block *self, unsigned long action, void *hcpu)
13281329

13291330
case CPU_STARTING:
13301331
if (x86_pmu.attr_rdpmc)
1331-
set_in_cr4(X86_CR4_PCE);
1332+
cr4_set_bits(X86_CR4_PCE);
13321333
if (x86_pmu.cpu_starting)
13331334
x86_pmu.cpu_starting(cpu);
13341335
break;
@@ -1834,9 +1835,9 @@ static void change_rdpmc(void *info)
18341835
bool enable = !!(unsigned long)info;
18351836

18361837
if (enable)
1837-
set_in_cr4(X86_CR4_PCE);
1838+
cr4_set_bits(X86_CR4_PCE);
18381839
else
1839-
clear_in_cr4(X86_CR4_PCE);
1840+
cr4_clear_bits(X86_CR4_PCE);
18401841
}
18411842

18421843
static ssize_t set_attr_rdpmc(struct device *cdev,

‎arch/x86/kernel/i387.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
#include <asm/sigcontext.h>
1414
#include <asm/processor.h>
1515
#include <asm/math_emu.h>
16+
#include <asm/tlbflush.h>
1617
#include <asm/uaccess.h>
1718
#include <asm/ptrace.h>
1819
#include <asm/i387.h>
@@ -180,7 +181,7 @@ void fpu_init(void)
180181
if (cpu_has_xmm)
181182
cr4_mask |= X86_CR4_OSXMMEXCPT;
182183
if (cr4_mask)
183-
set_in_cr4(cr4_mask);
184+
cr4_set_bits(cr4_mask);
184185

185186
cr0 = read_cr0();
186187
cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */

‎arch/x86/kernel/process.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include <asm/fpu-internal.h>
2929
#include <asm/debugreg.h>
3030
#include <asm/nmi.h>
31+
#include <asm/tlbflush.h>
3132

3233
/*
3334
* per-CPU TSS segments. Threads are completely 'soft' on Linux,
@@ -141,7 +142,7 @@ void flush_thread(void)
141142

142143
static void hard_disable_TSC(void)
143144
{
144-
write_cr4(read_cr4() | X86_CR4_TSD);
145+
cr4_set_bits(X86_CR4_TSD);
145146
}
146147

147148
void disable_TSC(void)
@@ -158,7 +159,7 @@ void disable_TSC(void)
158159

159160
static void hard_enable_TSC(void)
160161
{
161-
write_cr4(read_cr4() & ~X86_CR4_TSD);
162+
cr4_clear_bits(X86_CR4_TSD);
162163
}
163164

164165
static void enable_TSC(void)

‎arch/x86/kernel/xsave.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include <asm/i387.h>
1313
#include <asm/fpu-internal.h>
1414
#include <asm/sigframe.h>
15+
#include <asm/tlbflush.h>
1516
#include <asm/xcr.h>
1617

1718
/*
@@ -453,7 +454,7 @@ static void prepare_fx_sw_frame(void)
453454
*/
454455
static inline void xstate_enable(void)
455456
{
456-
set_in_cr4(X86_CR4_OSXSAVE);
457+
cr4_set_bits(X86_CR4_OSXSAVE);
457458
xsetbv(XCR_XFEATURE_ENABLED_MASK, pcntxt_mask);
458459
}
459460

‎arch/x86/kvm/vmx.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2812,7 +2812,7 @@ static int hardware_enable(void)
28122812
/* enable and lock */
28132813
wrmsrl(MSR_IA32_FEATURE_CONTROL, old | test_bits);
28142814
}
2815-
write_cr4(read_cr4() | X86_CR4_VMXE); /* FIXME: not cpu hotplug safe */
2815+
cr4_set_bits(X86_CR4_VMXE);
28162816

28172817
if (vmm_exclusive) {
28182818
kvm_cpu_vmxon(phys_addr);
@@ -2849,7 +2849,7 @@ static void hardware_disable(void)
28492849
vmclear_local_loaded_vmcss();
28502850
kvm_cpu_vmxoff();
28512851
}
2852-
write_cr4(read_cr4() & ~X86_CR4_VMXE);
2852+
cr4_clear_bits(X86_CR4_VMXE);
28532853
}
28542854

28552855
static __init int adjust_vmx_controls(u32 ctl_min, u32 ctl_opt,

‎arch/x86/mm/init.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -173,11 +173,11 @@ static void __init probe_page_size_mask(void)
173173

174174
/* Enable PSE if available */
175175
if (cpu_has_pse)
176-
set_in_cr4(X86_CR4_PSE);
176+
cr4_set_bits_and_update_boot(X86_CR4_PSE);
177177

178178
/* Enable PGE if available */
179179
if (cpu_has_pge) {
180-
set_in_cr4(X86_CR4_PGE);
180+
cr4_set_bits_and_update_boot(X86_CR4_PGE);
181181
__supported_pte_mask |= _PAGE_GLOBAL;
182182
}
183183
}

‎arch/x86/xen/enlighten.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1494,10 +1494,10 @@ static void xen_pvh_set_cr_flags(int cpu)
14941494
* set them here. For all, OSFXSR OSXMMEXCPT are set in fpu_init.
14951495
*/
14961496
if (cpu_has_pse)
1497-
set_in_cr4(X86_CR4_PSE);
1497+
cr4_set_bits_and_update_boot(X86_CR4_PSE);
14981498

14991499
if (cpu_has_pge)
1500-
set_in_cr4(X86_CR4_PGE);
1500+
cr4_set_bits_and_update_boot(X86_CR4_PGE);
15011501
}
15021502

15031503
/*

‎drivers/lguest/x86/core.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
#include <asm/lguest.h>
4848
#include <asm/uaccess.h>
4949
#include <asm/i387.h>
50+
#include <asm/tlbflush.h>
5051
#include "../lg.h"
5152

5253
static int cpu_had_pge;
@@ -452,9 +453,9 @@ void lguest_arch_handle_trap(struct lg_cpu *cpu)
452453
static void adjust_pge(void *on)
453454
{
454455
if (on)
455-
write_cr4(read_cr4() | X86_CR4_PGE);
456+
cr4_set_bits(X86_CR4_PGE);
456457
else
457-
write_cr4(read_cr4() & ~X86_CR4_PGE);
458+
cr4_clear_bits(X86_CR4_PGE);
458459
}
459460

460461
/*H:020

0 commit comments

Comments
 (0)