Skip to content

Commit 203b4fc

Browse files
torvaldstorvalds
committed
Merge branch 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 mm updates from Thomas Gleixner: - Make lazy TLB mode even lazier to avoid pointless switch_mm() operations, which reduces CPU load by 1-2% for memcache workloads - Small cleanups and improvements all over the place * 'x86-mm-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/mm: Remove redundant check for kmem_cache_create() arm/asm/tlb.h: Fix build error implicit func declaration x86/mm/tlb: Make clear_asid_other() static x86/mm/tlb: Skip atomic operations for 'init_mm' in switch_mm_irqs_off() x86/mm/tlb: Always use lazy TLB mode x86/mm/tlb: Only send page table free TLB flush to lazy TLB CPUs x86/mm/tlb: Make lazy TLB mode lazier x86/mm/tlb: Restructure switch_mm_irqs_off() x86/mm/tlb: Leave lazy TLB mode at page table free time mm: Allocate the mm_cpumask (mm->cpu_bitmap[]) dynamically based on nr_cpu_ids x86/mm: Add TLB purge to free pmd/pte page interfaces ioremap: Update pgtable free interfaces with addr x86/mm: Disable ioremap free page handling on x86-PAE
2 parents 7edcf0d + 765d28f commit 203b4fc

File tree

13 files changed

+408
-225
lines changed

13 files changed

+408
-225
lines changed

‎arch/arm/include/asm/tlb.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -292,5 +292,13 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb,
292292
{
293293
}
294294

295+
static inline void tlb_flush_remove_tables(struct mm_struct *mm)
296+
{
297+
}
298+
299+
static inline void tlb_flush_remove_tables_local(void *arg)
300+
{
301+
}
302+
295303
#endif /* CONFIG_MMU */
296304
#endif

‎arch/arm64/mm/mmu.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -977,12 +977,12 @@ int pmd_clear_huge(pmd_t *pmdp)
977977
return 1;
978978
}
979979

980-
int pud_free_pmd_page(pud_t *pud)
980+
int pud_free_pmd_page(pud_t *pud, unsigned long addr)
981981
{
982982
return pud_none(*pud);
983983
}
984984

985-
int pmd_free_pte_page(pmd_t *pmd)
985+
int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
986986
{
987987
return pmd_none(*pmd);
988988
}

‎arch/x86/include/asm/tlbflush.h

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -148,22 +148,6 @@ static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
148148
#define __flush_tlb_one_user(addr) __native_flush_tlb_one_user(addr)
149149
#endif
150150

151-
static inline bool tlb_defer_switch_to_init_mm(void)
152-
{
153-
/*
154-
* If we have PCID, then switching to init_mm is reasonably
155-
* fast. If we don't have PCID, then switching to init_mm is
156-
* quite slow, so we try to defer it in the hopes that we can
157-
* avoid it entirely. The latter approach runs the risk of
158-
* receiving otherwise unnecessary IPIs.
159-
*
160-
* This choice is just a heuristic. The tlb code can handle this
161-
* function returning true or false regardless of whether we have
162-
* PCID.
163-
*/
164-
return !static_cpu_has(X86_FEATURE_PCID);
165-
}
166-
167151
struct tlb_context {
168152
u64 ctx_id;
169153
u64 tlb_gen;
@@ -554,4 +538,9 @@ extern void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch);
554538
native_flush_tlb_others(mask, info)
555539
#endif
556540

541+
extern void tlb_flush_remove_tables(struct mm_struct *mm);
542+
extern void tlb_flush_remove_tables_local(void *arg);
543+
544+
#define HAVE_TLB_FLUSH_REMOVE_TABLES
545+
557546
#endif /* _ASM_X86_TLBFLUSH_H */

‎arch/x86/mm/pgtable.c

Lines changed: 53 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -329,9 +329,6 @@ static int __init pgd_cache_init(void)
329329
*/
330330
pgd_cache = kmem_cache_create("pgd_cache", PGD_SIZE, PGD_ALIGN,
331331
SLAB_PANIC, NULL);
332-
if (!pgd_cache)
333-
return -ENOMEM;
334-
335332
return 0;
336333
}
337334
core_initcall(pgd_cache_init);
@@ -719,28 +716,50 @@ int pmd_clear_huge(pmd_t *pmd)
719716
return 0;
720717
}
721718

719+
#ifdef CONFIG_X86_64
722720
/**
723721
* pud_free_pmd_page - Clear pud entry and free pmd page.
724722
* @pud: Pointer to a PUD.
723+
* @addr: Virtual address associated with pud.
725724
*
726-
* Context: The pud range has been unmaped and TLB purged.
725+
* Context: The pud range has been unmapped and TLB purged.
727726
* Return: 1 if clearing the entry succeeded. 0 otherwise.
727+
*
728+
* NOTE: Callers must allow a single page allocation.
728729
*/
729-
int pud_free_pmd_page(pud_t *pud)
730+
int pud_free_pmd_page(pud_t *pud, unsigned long addr)
730731
{
731-
pmd_t *pmd;
732+
pmd_t *pmd, *pmd_sv;
733+
pte_t *pte;
732734
int i;
733735

734736
if (pud_none(*pud))
735737
return 1;
736738

737739
pmd = (pmd_t *)pud_page_vaddr(*pud);
740+
pmd_sv = (pmd_t *)__get_free_page(GFP_KERNEL);
741+
if (!pmd_sv)
742+
return 0;
738743

739-
for (i = 0; i < PTRS_PER_PMD; i++)
740-
if (!pmd_free_pte_page(&pmd[i]))
741-
return 0;
744+
for (i = 0; i < PTRS_PER_PMD; i++) {
745+
pmd_sv[i] = pmd[i];
746+
if (!pmd_none(pmd[i]))
747+
pmd_clear(&pmd[i]);
748+
}
742749

743750
pud_clear(pud);
751+
752+
/* INVLPG to clear all paging-structure caches */
753+
flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1);
754+
755+
for (i = 0; i < PTRS_PER_PMD; i++) {
756+
if (!pmd_none(pmd_sv[i])) {
757+
pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]);
758+
free_page((unsigned long)pte);
759+
}
760+
}
761+
762+
free_page((unsigned long)pmd_sv);
744763
free_page((unsigned long)pmd);
745764

746765
return 1;
@@ -749,11 +768,12 @@ int pud_free_pmd_page(pud_t *pud)
749768
/**
750769
* pmd_free_pte_page - Clear pmd entry and free pte page.
751770
* @pmd: Pointer to a PMD.
771+
* @addr: Virtual address associated with pmd.
752772
*
753-
* Context: The pmd range has been unmaped and TLB purged.
773+
* Context: The pmd range has been unmapped and TLB purged.
754774
* Return: 1 if clearing the entry succeeded. 0 otherwise.
755775
*/
756-
int pmd_free_pte_page(pmd_t *pmd)
776+
int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
757777
{
758778
pte_t *pte;
759779

@@ -762,8 +782,30 @@ int pmd_free_pte_page(pmd_t *pmd)
762782

763783
pte = (pte_t *)pmd_page_vaddr(*pmd);
764784
pmd_clear(pmd);
785+
786+
/* INVLPG to clear all paging-structure caches */
787+
flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1);
788+
765789
free_page((unsigned long)pte);
766790

767791
return 1;
768792
}
793+
794+
#else /* !CONFIG_X86_64 */
795+
796+
int pud_free_pmd_page(pud_t *pud, unsigned long addr)
797+
{
798+
return pud_none(*pud);
799+
}
800+
801+
/*
802+
* Disable free page handling on x86-PAE. This assures that ioremap()
803+
* does not update sync'd pmd entries. See vmalloc_sync_one().
804+
*/
805+
int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
806+
{
807+
return pmd_none(*pmd);
808+
}
809+
810+
#endif /* CONFIG_X86_64 */
769811
#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */

0 commit comments

Comments
 (0)