Skip to content

Commit c481290

Browse files
kiryltorvalds
kiryl
authored and
torvalds
committed
mm: introduce wrappers to access mm->nr_ptes
Let's add wrappers for ->nr_ptes with the same interface as for nr_pmd and nr_pud. The patch also makes nr_ptes accounting dependent onto CONFIG_MMU. Page table accounting doesn't make sense if you don't have page tables. It's preparation for consolidation of page-table counters in mm_struct. Link: http://lkml.kernel.org/r/20171006100651.44742-1-kirill.shutemov@linux.intel.com Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Acked-by: Michal Hocko <mhocko@suse.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent b4e98d9 commit c481290

File tree

12 files changed

+54
-21
lines changed

12 files changed

+54
-21
lines changed

‎arch/arm/mm/pgd.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base)
141141
pte = pmd_pgtable(*pmd);
142142
pmd_clear(pmd);
143143
pte_free(mm, pte);
144-
atomic_long_dec(&mm->nr_ptes);
144+
mm_dec_nr_ptes(mm);
145145
no_pmd:
146146
pud_clear(pud);
147147
pmd_free(mm, pmd);

‎arch/sparc/mm/hugetlbpage.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,7 @@ static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
397397

398398
pmd_clear(pmd);
399399
pte_free_tlb(tlb, token, addr);
400-
atomic_long_dec(&tlb->mm->nr_ptes);
400+
mm_dec_nr_ptes(tlb->mm);
401401
}
402402

403403
static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud,

‎arch/unicore32/mm/pgd.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ void free_pgd_slow(struct mm_struct *mm, pgd_t *pgd)
9797
pte = pmd_pgtable(*pmd);
9898
pmd_clear(pmd);
9999
pte_free(mm, pte);
100-
atomic_long_dec(&mm->nr_ptes);
100+
mm_dec_nr_ptes(mm);
101101
pmd_free(mm, pmd);
102102
mm_dec_nr_pmds(mm);
103103
free:

‎fs/proc/task_mmu.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ void task_mem(struct seq_file *m, struct mm_struct *mm)
5050
text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> 10;
5151
lib = (mm->exec_vm << (PAGE_SHIFT-10)) - text;
5252
swap = get_mm_counter(mm, MM_SWAPENTS);
53-
ptes = PTRS_PER_PTE * sizeof(pte_t) * atomic_long_read(&mm->nr_ptes);
53+
ptes = PTRS_PER_PTE * sizeof(pte_t) * mm_nr_ptes(mm);
5454
pmds = PTRS_PER_PMD * sizeof(pmd_t) * mm_nr_pmds(mm);
5555
puds = PTRS_PER_PUD * sizeof(pud_t) * mm_nr_puds(mm);
5656
seq_printf(m,

‎include/linux/mm.h

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1680,6 +1680,38 @@ static inline void mm_dec_nr_pmds(struct mm_struct *mm)
16801680
}
16811681
#endif
16821682

1683+
#ifdef CONFIG_MMU
1684+
static inline void mm_nr_ptes_init(struct mm_struct *mm)
1685+
{
1686+
atomic_long_set(&mm->nr_ptes, 0);
1687+
}
1688+
1689+
static inline unsigned long mm_nr_ptes(const struct mm_struct *mm)
1690+
{
1691+
return atomic_long_read(&mm->nr_ptes);
1692+
}
1693+
1694+
static inline void mm_inc_nr_ptes(struct mm_struct *mm)
1695+
{
1696+
atomic_long_inc(&mm->nr_ptes);
1697+
}
1698+
1699+
static inline void mm_dec_nr_ptes(struct mm_struct *mm)
1700+
{
1701+
atomic_long_dec(&mm->nr_ptes);
1702+
}
1703+
#else
1704+
static inline void mm_nr_ptes_init(struct mm_struct *mm) {}
1705+
1706+
static inline unsigned long mm_nr_ptes(const struct mm_struct *mm)
1707+
{
1708+
return 0;
1709+
}
1710+
1711+
static inline void mm_inc_nr_ptes(struct mm_struct *mm) {}
1712+
static inline void mm_dec_nr_ptes(struct mm_struct *mm) {}
1713+
#endif
1714+
16831715
int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address);
16841716
int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
16851717

‎include/linux/mm_types.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,9 @@ struct mm_struct {
401401
*/
402402
atomic_t mm_count;
403403

404+
#ifdef CONFIG_MMU
404405
atomic_long_t nr_ptes; /* PTE page table pages */
406+
#endif
405407
#if CONFIG_PGTABLE_LEVELS > 2
406408
atomic_long_t nr_pmds; /* PMD page table pages */
407409
#endif

‎kernel/fork.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -817,7 +817,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
817817
init_rwsem(&mm->mmap_sem);
818818
INIT_LIST_HEAD(&mm->mmlist);
819819
mm->core_state = NULL;
820-
atomic_long_set(&mm->nr_ptes, 0);
820+
mm_nr_ptes_init(mm);
821821
mm_nr_pmds_init(mm);
822822
mm_nr_puds_init(mm);
823823
mm->map_count = 0;
@@ -873,9 +873,9 @@ static void check_mm(struct mm_struct *mm)
873873
"mm:%p idx:%d val:%ld\n", mm, i, x);
874874
}
875875

876-
if (atomic_long_read(&mm->nr_ptes))
876+
if (mm_nr_ptes(mm))
877877
pr_alert("BUG: non-zero nr_ptes on freeing mm: %ld\n",
878-
atomic_long_read(&mm->nr_ptes));
878+
mm_nr_ptes(mm));
879879
if (mm_nr_pmds(mm))
880880
pr_alert("BUG: non-zero nr_pmds on freeing mm: %ld\n",
881881
mm_nr_pmds(mm));

‎mm/debug.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ void dump_mm(const struct mm_struct *mm)
136136
mm->mmap_base, mm->mmap_legacy_base, mm->highest_vm_end,
137137
mm->pgd, atomic_read(&mm->mm_users),
138138
atomic_read(&mm->mm_count),
139-
atomic_long_read((atomic_long_t *)&mm->nr_ptes),
139+
mm_nr_ptes(mm),
140140
mm_nr_pmds(mm),
141141
mm_nr_puds(mm),
142142
mm->map_count,

‎mm/huge_memory.c

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -606,7 +606,7 @@ static int __do_huge_pmd_anonymous_page(struct vm_fault *vmf, struct page *page,
606606
pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable);
607607
set_pmd_at(vma->vm_mm, haddr, vmf->pmd, entry);
608608
add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
609-
atomic_long_inc(&vma->vm_mm->nr_ptes);
609+
mm_inc_nr_ptes(vma->vm_mm);
610610
spin_unlock(vmf->ptl);
611611
count_vm_event(THP_FAULT_ALLOC);
612612
}
@@ -662,7 +662,7 @@ static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
662662
if (pgtable)
663663
pgtable_trans_huge_deposit(mm, pmd, pgtable);
664664
set_pmd_at(mm, haddr, pmd, entry);
665-
atomic_long_inc(&mm->nr_ptes);
665+
mm_inc_nr_ptes(mm);
666666
return true;
667667
}
668668

@@ -747,7 +747,7 @@ static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
747747

748748
if (pgtable) {
749749
pgtable_trans_huge_deposit(mm, pmd, pgtable);
750-
atomic_long_inc(&mm->nr_ptes);
750+
mm_inc_nr_ptes(mm);
751751
}
752752

753753
set_pmd_at(mm, addr, pmd, entry);
@@ -978,7 +978,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
978978
get_page(src_page);
979979
page_dup_rmap(src_page, true);
980980
add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
981-
atomic_long_inc(&dst_mm->nr_ptes);
981+
mm_inc_nr_ptes(dst_mm);
982982
pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
983983

984984
pmdp_set_wrprotect(src_mm, addr, src_pmd);
@@ -1695,7 +1695,7 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd)
16951695

16961696
pgtable = pgtable_trans_huge_withdraw(mm, pmd);
16971697
pte_free(mm, pgtable);
1698-
atomic_long_dec(&mm->nr_ptes);
1698+
mm_dec_nr_ptes(mm);
16991699
}
17001700

17011701
int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,

‎mm/khugepaged.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1270,7 +1270,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
12701270
_pmd = pmdp_collapse_flush(vma, addr, pmd);
12711271
spin_unlock(ptl);
12721272
up_write(&vma->vm_mm->mmap_sem);
1273-
atomic_long_dec(&vma->vm_mm->nr_ptes);
1273+
mm_dec_nr_ptes(vma->vm_mm);
12741274
pte_free(vma->vm_mm, pmd_pgtable(_pmd));
12751275
}
12761276
}

‎mm/memory.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd,
438438
pgtable_t token = pmd_pgtable(*pmd);
439439
pmd_clear(pmd);
440440
pte_free_tlb(tlb, token, addr);
441-
atomic_long_dec(&tlb->mm->nr_ptes);
441+
mm_dec_nr_ptes(tlb->mm);
442442
}
443443

444444
static inline void free_pmd_range(struct mmu_gather *tlb, pud_t *pud,
@@ -666,7 +666,7 @@ int __pte_alloc(struct mm_struct *mm, pmd_t *pmd, unsigned long address)
666666

667667
ptl = pmd_lock(mm, pmd);
668668
if (likely(pmd_none(*pmd))) { /* Has another populated it ? */
669-
atomic_long_inc(&mm->nr_ptes);
669+
mm_inc_nr_ptes(mm);
670670
pmd_populate(mm, pmd, new);
671671
new = NULL;
672672
}
@@ -3238,7 +3238,7 @@ static int pte_alloc_one_map(struct vm_fault *vmf)
32383238
goto map_pte;
32393239
}
32403240

3241-
atomic_long_inc(&vma->vm_mm->nr_ptes);
3241+
mm_inc_nr_ptes(vma->vm_mm);
32423242
pmd_populate(vma->vm_mm, vmf->pmd, vmf->prealloc_pte);
32433243
spin_unlock(vmf->ptl);
32443244
vmf->prealloc_pte = NULL;
@@ -3297,7 +3297,7 @@ static void deposit_prealloc_pte(struct vm_fault *vmf)
32973297
* We are going to consume the prealloc table,
32983298
* count that as nr_ptes.
32993299
*/
3300-
atomic_long_inc(&vma->vm_mm->nr_ptes);
3300+
mm_inc_nr_ptes(vma->vm_mm);
33013301
vmf->prealloc_pte = NULL;
33023302
}
33033303

‎mm/oom_kill.c

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,7 @@ unsigned long oom_badness(struct task_struct *p, struct mem_cgroup *memcg,
221221
* task's rss, pagetable and swap space use.
222222
*/
223223
points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) +
224-
atomic_long_read(&p->mm->nr_ptes) + mm_nr_pmds(p->mm) +
225-
mm_nr_puds(p->mm);
224+
mm_nr_ptes(p->mm) + mm_nr_pmds(p->mm) + mm_nr_puds(p->mm);
226225
task_unlock(p);
227226

228227
/*
@@ -417,7 +416,7 @@ static void dump_tasks(struct mem_cgroup *memcg, const nodemask_t *nodemask)
417416
pr_info("[%5d] %5d %5d %8lu %8lu %7ld %7ld %7ld %8lu %5hd %s\n",
418417
task->pid, from_kuid(&init_user_ns, task_uid(task)),
419418
task->tgid, task->mm->total_vm, get_mm_rss(task->mm),
420-
atomic_long_read(&task->mm->nr_ptes),
419+
mm_nr_ptes(task->mm),
421420
mm_nr_pmds(task->mm),
422421
mm_nr_puds(task->mm),
423422
get_mm_counter(task->mm, MM_SWAPENTS),

0 commit comments

Comments
 (0)