Skip to content

Commit 0a9fe8c

Browse files
djbwIngo Molnar
djbw
authored and
Ingo Molnar
committed
x86/mm: Validate kernel_physical_mapping_init() PTE population
The usage of __flush_tlb_all() in the kernel_physical_mapping_init() path is not necessary. In general flushing the TLB is not required when updating an entry from the !present state. However, to give confidence in the future removal of TLB flushing in this path, use the new set_pte_safe() family of helpers to assert that the !present assumption is true in this path. [ mingo: Minor readability edits. ] Suggested-by: Peter Zijlstra <peterz@infradead.org> Suggested-by: Dave Hansen <dave.hansen@intel.com> Signed-off-by: Dan Williams <dan.j.williams@intel.com> Acked-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Andy Lutomirski <luto@kernel.org> Cc: Borislav Petkov <bp@alien8.de> Cc: Dave Hansen <dave.hansen@linux.intel.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Rik van Riel <riel@surriel.com> Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Cc: Thomas Gleixner <tglx@linutronix.de> Link: http://lkml.kernel.org/r/154395944177.32119.8524957429632012270.stgit@dwillia2-desk3.amr.corp.intel.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent 4369dea commit 0a9fe8c

File tree

6 files changed

+43
-12
lines changed

6 files changed

+43
-12
lines changed

‎arch/x86/include/asm/pgalloc.h

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,13 @@ static inline void pmd_populate_kernel(struct mm_struct *mm,
8080
set_pmd(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
8181
}
8282

83+
static inline void pmd_populate_kernel_safe(struct mm_struct *mm,
84+
pmd_t *pmd, pte_t *pte)
85+
{
86+
paravirt_alloc_pte(mm, __pa(pte) >> PAGE_SHIFT);
87+
set_pmd_safe(pmd, __pmd(__pa(pte) | _PAGE_TABLE));
88+
}
89+
8390
static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
8491
struct page *pte)
8592
{
@@ -132,6 +139,12 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
132139
paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
133140
set_pud(pud, __pud(_PAGE_TABLE | __pa(pmd)));
134141
}
142+
143+
static inline void pud_populate_safe(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
144+
{
145+
paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
146+
set_pud_safe(pud, __pud(_PAGE_TABLE | __pa(pmd)));
147+
}
135148
#endif /* CONFIG_X86_PAE */
136149

137150
#if CONFIG_PGTABLE_LEVELS > 3
@@ -141,6 +154,12 @@ static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
141154
set_p4d(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
142155
}
143156

157+
static inline void p4d_populate_safe(struct mm_struct *mm, p4d_t *p4d, pud_t *pud)
158+
{
159+
paravirt_alloc_pud(mm, __pa(pud) >> PAGE_SHIFT);
160+
set_p4d_safe(p4d, __p4d(_PAGE_TABLE | __pa(pud)));
161+
}
162+
144163
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
145164
{
146165
gfp_t gfp = GFP_KERNEL_ACCOUNT;
@@ -173,6 +192,14 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
173192
set_pgd(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
174193
}
175194

195+
static inline void pgd_populate_safe(struct mm_struct *mm, pgd_t *pgd, p4d_t *p4d)
196+
{
197+
if (!pgtable_l5_enabled())
198+
return;
199+
paravirt_alloc_p4d(mm, __pa(p4d) >> PAGE_SHIFT);
200+
set_pgd_safe(pgd, __pgd(_PAGE_TABLE | __pa(p4d)));
201+
}
202+
176203
static inline p4d_t *p4d_alloc_one(struct mm_struct *mm, unsigned long addr)
177204
{
178205
gfp_t gfp = GFP_KERNEL_ACCOUNT;

‎arch/x86/mm/init_64.c

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ phys_pte_init(pte_t *pte_page, unsigned long paddr, unsigned long paddr_end,
432432
E820_TYPE_RAM) &&
433433
!e820__mapped_any(paddr & PAGE_MASK, paddr_next,
434434
E820_TYPE_RESERVED_KERN))
435-
set_pte(pte, __pte(0));
435+
set_pte_safe(pte, __pte(0));
436436
continue;
437437
}
438438

@@ -452,7 +452,7 @@ phys_pte_init(pte_t *pte_page, unsigned long paddr, unsigned long paddr_end,
452452
pr_info(" pte=%p addr=%lx pte=%016lx\n", pte, paddr,
453453
pfn_pte(paddr >> PAGE_SHIFT, PAGE_KERNEL).pte);
454454
pages++;
455-
set_pte(pte, pfn_pte(paddr >> PAGE_SHIFT, prot));
455+
set_pte_safe(pte, pfn_pte(paddr >> PAGE_SHIFT, prot));
456456
paddr_last = (paddr & PAGE_MASK) + PAGE_SIZE;
457457
}
458458

@@ -487,7 +487,7 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end,
487487
E820_TYPE_RAM) &&
488488
!e820__mapped_any(paddr & PMD_MASK, paddr_next,
489489
E820_TYPE_RESERVED_KERN))
490-
set_pmd(pmd, __pmd(0));
490+
set_pmd_safe(pmd, __pmd(0));
491491
continue;
492492
}
493493

@@ -524,7 +524,7 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end,
524524
if (page_size_mask & (1<<PG_LEVEL_2M)) {
525525
pages++;
526526
spin_lock(&init_mm.page_table_lock);
527-
set_pte((pte_t *)pmd,
527+
set_pte_safe((pte_t *)pmd,
528528
pfn_pte((paddr & PMD_MASK) >> PAGE_SHIFT,
529529
__pgprot(pgprot_val(prot) | _PAGE_PSE)));
530530
spin_unlock(&init_mm.page_table_lock);
@@ -536,7 +536,7 @@ phys_pmd_init(pmd_t *pmd_page, unsigned long paddr, unsigned long paddr_end,
536536
paddr_last = phys_pte_init(pte, paddr, paddr_end, new_prot);
537537

538538
spin_lock(&init_mm.page_table_lock);
539-
pmd_populate_kernel(&init_mm, pmd, pte);
539+
pmd_populate_kernel_safe(&init_mm, pmd, pte);
540540
spin_unlock(&init_mm.page_table_lock);
541541
}
542542
update_page_count(PG_LEVEL_2M, pages);
@@ -573,7 +573,7 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end,
573573
E820_TYPE_RAM) &&
574574
!e820__mapped_any(paddr & PUD_MASK, paddr_next,
575575
E820_TYPE_RESERVED_KERN))
576-
set_pud(pud, __pud(0));
576+
set_pud_safe(pud, __pud(0));
577577
continue;
578578
}
579579

@@ -611,7 +611,7 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end,
611611
if (page_size_mask & (1<<PG_LEVEL_1G)) {
612612
pages++;
613613
spin_lock(&init_mm.page_table_lock);
614-
set_pte((pte_t *)pud,
614+
set_pte_safe((pte_t *)pud,
615615
pfn_pte((paddr & PUD_MASK) >> PAGE_SHIFT,
616616
PAGE_KERNEL_LARGE));
617617
spin_unlock(&init_mm.page_table_lock);
@@ -624,7 +624,7 @@ phys_pud_init(pud_t *pud_page, unsigned long paddr, unsigned long paddr_end,
624624
page_size_mask, prot);
625625

626626
spin_lock(&init_mm.page_table_lock);
627-
pud_populate(&init_mm, pud, pmd);
627+
pud_populate_safe(&init_mm, pud, pmd);
628628
spin_unlock(&init_mm.page_table_lock);
629629
}
630630
__flush_tlb_all();
@@ -659,7 +659,7 @@ phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end,
659659
E820_TYPE_RAM) &&
660660
!e820__mapped_any(paddr & P4D_MASK, paddr_next,
661661
E820_TYPE_RESERVED_KERN))
662-
set_p4d(p4d, __p4d(0));
662+
set_p4d_safe(p4d, __p4d(0));
663663
continue;
664664
}
665665

@@ -677,7 +677,7 @@ phys_p4d_init(p4d_t *p4d_page, unsigned long paddr, unsigned long paddr_end,
677677
page_size_mask);
678678

679679
spin_lock(&init_mm.page_table_lock);
680-
p4d_populate(&init_mm, p4d, pud);
680+
p4d_populate_safe(&init_mm, p4d, pud);
681681
spin_unlock(&init_mm.page_table_lock);
682682
}
683683
__flush_tlb_all();
@@ -723,9 +723,9 @@ kernel_physical_mapping_init(unsigned long paddr_start,
723723

724724
spin_lock(&init_mm.page_table_lock);
725725
if (pgtable_l5_enabled())
726-
pgd_populate(&init_mm, pgd, p4d);
726+
pgd_populate_safe(&init_mm, pgd, p4d);
727727
else
728-
p4d_populate(&init_mm, p4d_offset(pgd, vaddr), (pud_t *) p4d);
728+
p4d_populate_safe(&init_mm, p4d_offset(pgd, vaddr), (pud_t *) p4d);
729729
spin_unlock(&init_mm.page_table_lock);
730730
pgd_changed = true;
731731
}

‎include/asm-generic/5level-fixup.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#define p4d_clear(p4d) pgd_clear(p4d)
2727
#define p4d_val(p4d) pgd_val(p4d)
2828
#define p4d_populate(mm, p4d, pud) pgd_populate(mm, p4d, pud)
29+
#define p4d_populate_safe(mm, p4d, pud) pgd_populate(mm, p4d, pud)
2930
#define p4d_page(p4d) pgd_page(p4d)
3031
#define p4d_page_vaddr(p4d) pgd_page_vaddr(p4d)
3132

‎include/asm-generic/pgtable-nop4d-hack.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ static inline void pgd_clear(pgd_t *pgd) { }
3131
#define pud_ERROR(pud) (pgd_ERROR((pud).pgd))
3232

3333
#define pgd_populate(mm, pgd, pud) do { } while (0)
34+
#define pgd_populate_safe(mm, pgd, pud) do { } while (0)
3435
/*
3536
* (puds are folded into pgds so this doesn't get actually called,
3637
* but the define is needed for a generic inline function.)

‎include/asm-generic/pgtable-nop4d.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ static inline void pgd_clear(pgd_t *pgd) { }
2626
#define p4d_ERROR(p4d) (pgd_ERROR((p4d).pgd))
2727

2828
#define pgd_populate(mm, pgd, p4d) do { } while (0)
29+
#define pgd_populate_safe(mm, pgd, p4d) do { } while (0)
2930
/*
3031
* (p4ds are folded into pgds so this doesn't get actually called,
3132
* but the define is needed for a generic inline function.)

‎include/asm-generic/pgtable-nopud.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ static inline void p4d_clear(p4d_t *p4d) { }
3535
#define pud_ERROR(pud) (p4d_ERROR((pud).p4d))
3636

3737
#define p4d_populate(mm, p4d, pud) do { } while (0)
38+
#define p4d_populate_safe(mm, p4d, pud) do { } while (0)
3839
/*
3940
* (puds are folded into p4ds so this doesn't get actually called,
4041
* but the define is needed for a generic inline function.)

0 commit comments

Comments
 (0)