Skip to content

Commit eac3411

Browse files
torvaldstorvalds
committed
Merge branch 'x86/pti' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 PTI updates from Thomas Gleixner: "The Speck brigade sadly provides yet another large set of patches destroying the perfomance which we carefully built and preserved - PTI support for 32bit PAE. The missing counter part to the 64bit PTI code implemented by Joerg. - A set of fixes for the Global Bit mechanics for non PCID CPUs which were setting the Global Bit too widely and therefore possibly exposing interesting memory needlessly. - Protection against userspace-userspace SpectreRSB - Support for the upcoming Enhanced IBRS mode, which is preferred over IBRS. Unfortunately we dont know the performance impact of this, but it's expected to be less horrible than the IBRS hammering. - Cleanups and simplifications" * 'x86/pti' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (60 commits) x86/mm/pti: Move user W+X check into pti_finalize() x86/relocs: Add __end_rodata_aligned to S_REL x86/mm/pti: Clone kernel-image on PTE level for 32 bit x86/mm/pti: Don't clear permissions in pti_clone_pmd() x86/mm/pti: Fix 32 bit PCID check x86/mm/init: Remove freed kernel image areas from alias mapping x86/mm/init: Add helper for freeing kernel image pages x86/mm/init: Pass unconverted symbol addresses to free_init_pages() mm: Allow non-direct-map arguments to free_reserved_area() x86/mm/pti: Clear Global bit more aggressively x86/speculation: Support Enhanced IBRS on future CPUs x86/speculation: Protect against userspace-userspace spectreRSB x86/kexec: Allocate 8k PGDs for PTI Revert "perf/core: Make sure the ring-buffer is mapped in all page-tables" x86/mm: Remove in_nmi() warning from vmalloc_fault() x86/entry/32: Check for VM86 mode in slow-path check perf/core: Make sure the ring-buffer is mapped in all page-tables x86/pti: Check the return value of pti_user_pagetable_walk_pmd() x86/pti: Check the return value of pti_user_pagetable_walk_p4d() x86/entry/32: Add debug code to check entry/exit CR3 ...
2 parents d191c82 + d878efc commit eac3411

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1271
-413
lines changed

‎arch/x86/entry/entry_32.S

Lines changed: 522 additions & 110 deletions
Large diffs are not rendered by default.

‎arch/x86/include/asm/cpufeatures.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@
219219
#define X86_FEATURE_IBPB ( 7*32+26) /* Indirect Branch Prediction Barrier */
220220
#define X86_FEATURE_STIBP ( 7*32+27) /* Single Thread Indirect Branch Predictors */
221221
#define X86_FEATURE_ZEN ( 7*32+28) /* "" CPU is AMD family 0x17 (Zen) */
222+
#define X86_FEATURE_IBRS_ENHANCED ( 7*32+29) /* Enhanced IBRS */
222223

223224
/* Virtualization flags: Linux defined, word 8 */
224225
#define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */

‎arch/x86/include/asm/mmu_context.h

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,7 @@ struct ldt_struct {
7171

7272
static inline void *ldt_slot_va(int slot)
7373
{
74-
#ifdef CONFIG_X86_64
7574
return (void *)(LDT_BASE_ADDR + LDT_SLOT_STRIDE * slot);
76-
#else
77-
BUG();
78-
return (void *)fix_to_virt(FIX_HOLE);
79-
#endif
8075
}
8176

8277
/*

‎arch/x86/include/asm/nospec-branch.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ enum spectre_v2_mitigation {
214214
SPECTRE_V2_RETPOLINE_MINIMAL_AMD,
215215
SPECTRE_V2_RETPOLINE_GENERIC,
216216
SPECTRE_V2_RETPOLINE_AMD,
217-
SPECTRE_V2_IBRS,
217+
SPECTRE_V2_IBRS_ENHANCED,
218218
};
219219

220220
/* The Speculative Store Bypass disable variants */

‎arch/x86/include/asm/pgtable-2level.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ static inline void native_set_pte(pte_t *ptep , pte_t pte)
1919

2020
static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
2121
{
22+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
23+
pmd.pud.p4d.pgd = pti_set_user_pgtbl(&pmdp->pud.p4d.pgd, pmd.pud.p4d.pgd);
24+
#endif
2225
*pmdp = pmd;
2326
}
2427

@@ -58,6 +61,9 @@ static inline pte_t native_ptep_get_and_clear(pte_t *xp)
5861
#ifdef CONFIG_SMP
5962
static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
6063
{
64+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
65+
pti_set_user_pgtbl(&xp->pud.p4d.pgd, __pgd(0));
66+
#endif
6167
return __pmd(xchg((pmdval_t *)xp, 0));
6268
}
6369
#else
@@ -67,6 +73,9 @@ static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp)
6773
#ifdef CONFIG_SMP
6874
static inline pud_t native_pudp_get_and_clear(pud_t *xp)
6975
{
76+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
77+
pti_set_user_pgtbl(&xp->p4d.pgd, __pgd(0));
78+
#endif
7079
return __pud(xchg((pudval_t *)xp, 0));
7180
}
7281
#else

‎arch/x86/include/asm/pgtable-2level_types.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,7 @@ typedef union {
3535

3636
#define PTRS_PER_PTE 1024
3737

38+
/* This covers all VMSPLIT_* and VMSPLIT_*_OPT variants */
39+
#define PGD_KERNEL_START (CONFIG_PAGE_OFFSET >> PGDIR_SHIFT)
40+
3841
#endif /* _ASM_X86_PGTABLE_2LEVEL_DEFS_H */

‎arch/x86/include/asm/pgtable-3level.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd)
9898

9999
static inline void native_set_pud(pud_t *pudp, pud_t pud)
100100
{
101+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
102+
pud.p4d.pgd = pti_set_user_pgtbl(&pudp->p4d.pgd, pud.p4d.pgd);
103+
#endif
101104
set_64bit((unsigned long long *)(pudp), native_pud_val(pud));
102105
}
103106

@@ -229,6 +232,10 @@ static inline pud_t native_pudp_get_and_clear(pud_t *pudp)
229232
{
230233
union split_pud res, *orig = (union split_pud *)pudp;
231234

235+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
236+
pti_set_user_pgtbl(&pudp->p4d.pgd, __pgd(0));
237+
#endif
238+
232239
/* xchg acts as a barrier before setting of the high bits */
233240
res.pud_low = xchg(&orig->pud_low, 0);
234241
res.pud_high = orig->pud_high;

‎arch/x86/include/asm/pgtable-3level_types.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@ typedef union {
2121
#endif /* !__ASSEMBLY__ */
2222

2323
#ifdef CONFIG_PARAVIRT
24-
#define SHARED_KERNEL_PMD (pv_info.shared_kernel_pmd)
24+
#define SHARED_KERNEL_PMD ((!static_cpu_has(X86_FEATURE_PTI) && \
25+
(pv_info.shared_kernel_pmd)))
2526
#else
26-
#define SHARED_KERNEL_PMD 1
27+
#define SHARED_KERNEL_PMD (!static_cpu_has(X86_FEATURE_PTI))
2728
#endif
2829

2930
/*
@@ -45,5 +46,6 @@ typedef union {
4546
#define PTRS_PER_PTE 512
4647

4748
#define MAX_POSSIBLE_PHYSMEM_BITS 36
49+
#define PGD_KERNEL_START (CONFIG_PAGE_OFFSET >> PGDIR_SHIFT)
4850

4951
#endif /* _ASM_X86_PGTABLE_3LEVEL_DEFS_H */

‎arch/x86/include/asm/pgtable.h

Lines changed: 92 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,14 @@ int __init __early_make_pgtable(unsigned long address, pmdval_t pmd);
3030
void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd);
3131
void ptdump_walk_pgd_level_debugfs(struct seq_file *m, pgd_t *pgd, bool user);
3232
void ptdump_walk_pgd_level_checkwx(void);
33+
void ptdump_walk_user_pgd_level_checkwx(void);
3334

3435
#ifdef CONFIG_DEBUG_WX
35-
#define debug_checkwx() ptdump_walk_pgd_level_checkwx()
36+
#define debug_checkwx() ptdump_walk_pgd_level_checkwx()
37+
#define debug_checkwx_user() ptdump_walk_user_pgd_level_checkwx()
3638
#else
37-
#define debug_checkwx() do { } while (0)
39+
#define debug_checkwx() do { } while (0)
40+
#define debug_checkwx_user() do { } while (0)
3841
#endif
3942

4043
/*
@@ -640,8 +643,31 @@ static inline int is_new_memtype_allowed(u64 paddr, unsigned long size,
640643

641644
pmd_t *populate_extra_pmd(unsigned long vaddr);
642645
pte_t *populate_extra_pte(unsigned long vaddr);
646+
647+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
648+
pgd_t __pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd);
649+
650+
/*
651+
* Take a PGD location (pgdp) and a pgd value that needs to be set there.
652+
* Populates the user and returns the resulting PGD that must be set in
653+
* the kernel copy of the page tables.
654+
*/
655+
static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
656+
{
657+
if (!static_cpu_has(X86_FEATURE_PTI))
658+
return pgd;
659+
return __pti_set_user_pgtbl(pgdp, pgd);
660+
}
661+
#else /* CONFIG_PAGE_TABLE_ISOLATION */
662+
static inline pgd_t pti_set_user_pgtbl(pgd_t *pgdp, pgd_t pgd)
663+
{
664+
return pgd;
665+
}
666+
#endif /* CONFIG_PAGE_TABLE_ISOLATION */
667+
643668
#endif /* __ASSEMBLY__ */
644669

670+
645671
#ifdef CONFIG_X86_32
646672
# include <asm/pgtable_32.h>
647673
#else
@@ -1154,6 +1180,70 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma,
11541180
}
11551181
}
11561182
#endif
1183+
/*
1184+
* Page table pages are page-aligned. The lower half of the top
1185+
* level is used for userspace and the top half for the kernel.
1186+
*
1187+
* Returns true for parts of the PGD that map userspace and
1188+
* false for the parts that map the kernel.
1189+
*/
1190+
static inline bool pgdp_maps_userspace(void *__ptr)
1191+
{
1192+
unsigned long ptr = (unsigned long)__ptr;
1193+
1194+
return (((ptr & ~PAGE_MASK) / sizeof(pgd_t)) < PGD_KERNEL_START);
1195+
}
1196+
1197+
static inline int pgd_large(pgd_t pgd) { return 0; }
1198+
1199+
#ifdef CONFIG_PAGE_TABLE_ISOLATION
1200+
/*
1201+
* All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
1202+
* (8k-aligned and 8k in size). The kernel one is at the beginning 4k and
1203+
* the user one is in the last 4k. To switch between them, you
1204+
* just need to flip the 12th bit in their addresses.
1205+
*/
1206+
#define PTI_PGTABLE_SWITCH_BIT PAGE_SHIFT
1207+
1208+
/*
1209+
* This generates better code than the inline assembly in
1210+
* __set_bit().
1211+
*/
1212+
static inline void *ptr_set_bit(void *ptr, int bit)
1213+
{
1214+
unsigned long __ptr = (unsigned long)ptr;
1215+
1216+
__ptr |= BIT(bit);
1217+
return (void *)__ptr;
1218+
}
1219+
static inline void *ptr_clear_bit(void *ptr, int bit)
1220+
{
1221+
unsigned long __ptr = (unsigned long)ptr;
1222+
1223+
__ptr &= ~BIT(bit);
1224+
return (void *)__ptr;
1225+
}
1226+
1227+
static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
1228+
{
1229+
return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
1230+
}
1231+
1232+
static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
1233+
{
1234+
return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
1235+
}
1236+
1237+
static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
1238+
{
1239+
return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
1240+
}
1241+
1242+
static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
1243+
{
1244+
return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
1245+
}
1246+
#endif /* CONFIG_PAGE_TABLE_ISOLATION */
11571247

11581248
/*
11591249
* clone_pgd_range(pgd_t *dst, pgd_t *src, int count);

‎arch/x86/include/asm/pgtable_32.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,6 @@ static inline void check_pgt_cache(void) { }
3434
void paging_init(void);
3535
void sync_initial_page_table(void);
3636

37-
static inline int pgd_large(pgd_t pgd) { return 0; }
38-
3937
/*
4038
* Define this if things work differently on an i386 and an i486:
4139
* it will (on an i486) warn about kernel memory accesses that are

‎arch/x86/include/asm/pgtable_32_types.h

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,18 @@ extern bool __vmalloc_start_set; /* set once high_memory is set */
5050
((FIXADDR_TOT_START - PAGE_SIZE * (CPU_ENTRY_AREA_PAGES + 1)) \
5151
& PMD_MASK)
5252

53-
#define PKMAP_BASE \
53+
#define LDT_BASE_ADDR \
5454
((CPU_ENTRY_AREA_BASE - PAGE_SIZE) & PMD_MASK)
5555

56+
#define LDT_END_ADDR (LDT_BASE_ADDR + PMD_SIZE)
57+
58+
#define PKMAP_BASE \
59+
((LDT_BASE_ADDR - PAGE_SIZE) & PMD_MASK)
60+
5661
#ifdef CONFIG_HIGHMEM
5762
# define VMALLOC_END (PKMAP_BASE - 2 * PAGE_SIZE)
5863
#else
59-
# define VMALLOC_END (CPU_ENTRY_AREA_BASE - 2 * PAGE_SIZE)
64+
# define VMALLOC_END (LDT_BASE_ADDR - 2 * PAGE_SIZE)
6065
#endif
6166

6267
#define MODULES_VADDR VMALLOC_START

‎arch/x86/include/asm/pgtable_64.h

Lines changed: 2 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -132,90 +132,6 @@ static inline pud_t native_pudp_get_and_clear(pud_t *xp)
132132
#endif
133133
}
134134

135-
#ifdef CONFIG_PAGE_TABLE_ISOLATION
136-
/*
137-
* All top-level PAGE_TABLE_ISOLATION page tables are order-1 pages
138-
* (8k-aligned and 8k in size). The kernel one is at the beginning 4k and
139-
* the user one is in the last 4k. To switch between them, you
140-
* just need to flip the 12th bit in their addresses.
141-
*/
142-
#define PTI_PGTABLE_SWITCH_BIT PAGE_SHIFT
143-
144-
/*
145-
* This generates better code than the inline assembly in
146-
* __set_bit().
147-
*/
148-
static inline void *ptr_set_bit(void *ptr, int bit)
149-
{
150-
unsigned long __ptr = (unsigned long)ptr;
151-
152-
__ptr |= BIT(bit);
153-
return (void *)__ptr;
154-
}
155-
static inline void *ptr_clear_bit(void *ptr, int bit)
156-
{
157-
unsigned long __ptr = (unsigned long)ptr;
158-
159-
__ptr &= ~BIT(bit);
160-
return (void *)__ptr;
161-
}
162-
163-
static inline pgd_t *kernel_to_user_pgdp(pgd_t *pgdp)
164-
{
165-
return ptr_set_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
166-
}
167-
168-
static inline pgd_t *user_to_kernel_pgdp(pgd_t *pgdp)
169-
{
170-
return ptr_clear_bit(pgdp, PTI_PGTABLE_SWITCH_BIT);
171-
}
172-
173-
static inline p4d_t *kernel_to_user_p4dp(p4d_t *p4dp)
174-
{
175-
return ptr_set_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
176-
}
177-
178-
static inline p4d_t *user_to_kernel_p4dp(p4d_t *p4dp)
179-
{
180-
return ptr_clear_bit(p4dp, PTI_PGTABLE_SWITCH_BIT);
181-
}
182-
#endif /* CONFIG_PAGE_TABLE_ISOLATION */
183-
184-
/*
185-
* Page table pages are page-aligned. The lower half of the top
186-
* level is used for userspace and the top half for the kernel.
187-
*
188-
* Returns true for parts of the PGD that map userspace and
189-
* false for the parts that map the kernel.
190-
*/
191-
static inline bool pgdp_maps_userspace(void *__ptr)
192-
{
193-
unsigned long ptr = (unsigned long)__ptr;
194-
195-
return (ptr & ~PAGE_MASK) < (PAGE_SIZE / 2);
196-
}
197-
198-
#ifdef CONFIG_PAGE_TABLE_ISOLATION
199-
pgd_t __pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd);
200-
201-
/*
202-
* Take a PGD location (pgdp) and a pgd value that needs to be set there.
203-
* Populates the user and returns the resulting PGD that must be set in
204-
* the kernel copy of the page tables.
205-
*/
206-
static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd)
207-
{
208-
if (!static_cpu_has(X86_FEATURE_PTI))
209-
return pgd;
210-
return __pti_set_user_pgd(pgdp, pgd);
211-
}
212-
#else
213-
static inline pgd_t pti_set_user_pgd(pgd_t *pgdp, pgd_t pgd)
214-
{
215-
return pgd;
216-
}
217-
#endif
218-
219135
static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
220136
{
221137
pgd_t pgd;
@@ -226,7 +142,7 @@ static inline void native_set_p4d(p4d_t *p4dp, p4d_t p4d)
226142
}
227143

228144
pgd = native_make_pgd(native_p4d_val(p4d));
229-
pgd = pti_set_user_pgd((pgd_t *)p4dp, pgd);
145+
pgd = pti_set_user_pgtbl((pgd_t *)p4dp, pgd);
230146
*p4dp = native_make_p4d(native_pgd_val(pgd));
231147
}
232148

@@ -237,7 +153,7 @@ static inline void native_p4d_clear(p4d_t *p4d)
237153

238154
static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
239155
{
240-
*pgdp = pti_set_user_pgd(pgdp, pgd);
156+
*pgdp = pti_set_user_pgtbl(pgdp, pgd);
241157
}
242158

243159
static inline void native_pgd_clear(pgd_t *pgd)
@@ -255,7 +171,6 @@ extern void sync_global_pgds(unsigned long start, unsigned long end);
255171
/*
256172
* Level 4 access.
257173
*/
258-
static inline int pgd_large(pgd_t pgd) { return 0; }
259174
#define mk_kernel_pgd(address) __pgd((address) | _KERNPG_TABLE)
260175

261176
/* PUD - Level3 access */

‎arch/x86/include/asm/pgtable_64_types.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ extern unsigned int ptrs_per_p4d;
115115
#define LDT_PGD_ENTRY_L5 -112UL
116116
#define LDT_PGD_ENTRY (pgtable_l5_enabled() ? LDT_PGD_ENTRY_L5 : LDT_PGD_ENTRY_L4)
117117
#define LDT_BASE_ADDR (LDT_PGD_ENTRY << PGDIR_SHIFT)
118+
#define LDT_END_ADDR (LDT_BASE_ADDR + PGDIR_SIZE)
118119

119120
#define __VMALLOC_BASE_L4 0xffffc90000000000UL
120121
#define __VMALLOC_BASE_L5 0xffa0000000000000UL
@@ -153,4 +154,6 @@ extern unsigned int ptrs_per_p4d;
153154

154155
#define EARLY_DYNAMIC_PAGE_TABLES 64
155156

157+
#define PGD_KERNEL_START ((PAGE_SIZE / 2) / sizeof(pgd_t))
158+
156159
#endif /* _ASM_X86_PGTABLE_64_DEFS_H */

0 commit comments

Comments
 (0)