Skip to content

Commit b3b9c29

Browse files
koct9itorvalds
koct9i
authored and
torvalds
committed
mm, x86, pat: rework linear pfn-mmap tracking
Replace the generic vma-flag VM_PFN_AT_MMAP with x86-only VM_PAT. We can toss mapping address from remap_pfn_range() into track_pfn_vma_new(), and collect all PAT-related logic together in arch/x86/. This patch also restores orignal frustration-free is_cow_mapping() check in remap_pfn_range(), as it was before commit v2.6.28-rc8-88-g3c8bb73 ("x86: PAT: store vm_pgoff for all linear_over_vma_region mappings - v3") is_linear_pfn_mapping() checks can be removed from mm/huge_memory.c, because it already handled by VM_PFNMAP in VM_NO_THP bit-mask. [suresh.b.siddha@intel.com: Reset the VM_PAT flag as part of untrack_pfn_vma()] Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org> Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com> Cc: Venkatesh Pallipadi <venki@google.com> Cc: H. Peter Anvin <hpa@zytor.com> Cc: Nick Piggin <npiggin@kernel.dk> Cc: Ingo Molnar <mingo@redhat.com> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Carsten Otte <cotte@de.ibm.com> Cc: Chris Metcalf <cmetcalf@tilera.com> Cc: Cyrill Gorcunov <gorcunov@openvz.org> Cc: Eric Paris <eparis@redhat.com> Cc: Hugh Dickins <hughd@google.com> Cc: James Morris <james.l.morris@oracle.com> Cc: Jason Baron <jbaron@redhat.com> Cc: Kentaro Takeda <takedakn@nttdata.co.jp> Cc: Matt Helsley <matthltc@us.ibm.com> Cc: Oleg Nesterov <oleg@redhat.com> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Robert Richter <robert.richter@amd.com> Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp> Cc: Venkatesh Pallipadi <venki@google.com> Acked-by: Linus Torvalds <torvalds@linux-foundation.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent 5180da4 commit b3b9c29

File tree

5 files changed

+30
-58
lines changed

5 files changed

+30
-58
lines changed

‎arch/x86/mm/pat.c

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -677,7 +677,7 @@ int track_pfn_copy(struct vm_area_struct *vma)
677677
unsigned long vma_size = vma->vm_end - vma->vm_start;
678678
pgprot_t pgprot;
679679

680-
if (is_linear_pfn_mapping(vma)) {
680+
if (vma->vm_flags & VM_PAT) {
681681
/*
682682
* reserve the whole chunk covered by vma. We need the
683683
* starting address and protection from pte.
@@ -699,14 +699,20 @@ int track_pfn_copy(struct vm_area_struct *vma)
699699
* single reserve_pfn_range call.
700700
*/
701701
int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
702-
unsigned long pfn, unsigned long size)
702+
unsigned long pfn, unsigned long addr, unsigned long size)
703703
{
704704
resource_size_t paddr = (resource_size_t)pfn << PAGE_SHIFT;
705705
unsigned long flags;
706706

707707
/* reserve the whole chunk starting from paddr */
708-
if (is_linear_pfn_mapping(vma))
709-
return reserve_pfn_range(paddr, size, prot, 0);
708+
if (addr == vma->vm_start && size == (vma->vm_end - vma->vm_start)) {
709+
int ret;
710+
711+
ret = reserve_pfn_range(paddr, size, prot, 0);
712+
if (!ret)
713+
vma->vm_flags |= VM_PAT;
714+
return ret;
715+
}
710716

711717
if (!pat_enabled)
712718
return 0;
@@ -758,7 +764,7 @@ void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
758764
resource_size_t paddr;
759765
unsigned long prot;
760766

761-
if (!is_linear_pfn_mapping(vma))
767+
if (!(vma->vm_flags & VM_PAT))
762768
return;
763769

764770
/* free the chunk starting from pfn or the whole chunk */
@@ -772,6 +778,7 @@ void untrack_pfn(struct vm_area_struct *vma, unsigned long pfn,
772778
size = vma->vm_end - vma->vm_start;
773779
}
774780
free_pfn_range(paddr, size);
781+
vma->vm_flags &= ~VM_PAT;
775782
}
776783

777784
pgprot_t pgprot_writecombine(pgprot_t prot)

‎include/asm-generic/pgtable.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -391,7 +391,8 @@ static inline void ptep_modify_prot_commit(struct mm_struct *mm,
391391
* by remap_pfn_range() for physical range indicated by pfn and size.
392392
*/
393393
static inline int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
394-
unsigned long pfn, unsigned long size)
394+
unsigned long pfn, unsigned long addr,
395+
unsigned long size)
395396
{
396397
return 0;
397398
}
@@ -426,7 +427,8 @@ static inline void untrack_pfn(struct vm_area_struct *vma,
426427
}
427428
#else
428429
extern int track_pfn_remap(struct vm_area_struct *vma, pgprot_t *prot,
429-
unsigned long pfn, unsigned long size);
430+
unsigned long pfn, unsigned long addr,
431+
unsigned long size);
430432
extern int track_pfn_insert(struct vm_area_struct *vma, pgprot_t *prot,
431433
unsigned long pfn);
432434
extern int track_pfn_copy(struct vm_area_struct *vma);

‎include/linux/mm.h

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ extern unsigned int kobjsize(const void *objp);
117117
#define VM_CAN_NONLINEAR 0x08000000 /* Has ->fault & does nonlinear pages */
118118
#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
119119
#define VM_SAO 0x20000000 /* Strong Access Ordering (powerpc) */
120-
#define VM_PFN_AT_MMAP 0x40000000 /* PFNMAP vma that is fully mapped at mmap time */
120+
#define VM_PAT 0x40000000 /* PAT reserves whole VMA at once (x86) */
121121
#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
122122

123123
/* Bits set in the VMA until the stack is in its final location */
@@ -158,24 +158,6 @@ extern pgprot_t protection_map[16];
158158
#define FAULT_FLAG_RETRY_NOWAIT 0x10 /* Don't drop mmap_sem and wait when retrying */
159159
#define FAULT_FLAG_KILLABLE 0x20 /* The fault task is in SIGKILL killable region */
160160

161-
/*
162-
* This interface is used by x86 PAT code to identify a pfn mapping that is
163-
* linear over entire vma. This is to optimize PAT code that deals with
164-
* marking the physical region with a particular prot. This is not for generic
165-
* mm use. Note also that this check will not work if the pfn mapping is
166-
* linear for a vma starting at physical address 0. In which case PAT code
167-
* falls back to slow path of reserving physical range page by page.
168-
*/
169-
static inline int is_linear_pfn_mapping(struct vm_area_struct *vma)
170-
{
171-
return !!(vma->vm_flags & VM_PFN_AT_MMAP);
172-
}
173-
174-
static inline int is_pfn_mapping(struct vm_area_struct *vma)
175-
{
176-
return !!(vma->vm_flags & VM_PFNMAP);
177-
}
178-
179161
/*
180162
* vm_fault is filled by the the pagefault handler and passed to the vma's
181163
* ->fault function. The vma's ->fault is responsible for returning a bitmask

‎mm/huge_memory.c

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1655,11 +1655,7 @@ int khugepaged_enter_vma_merge(struct vm_area_struct *vma)
16551655
if (vma->vm_ops)
16561656
/* khugepaged not yet working on file or special mappings */
16571657
return 0;
1658-
/*
1659-
* If is_pfn_mapping() is true is_learn_pfn_mapping() must be
1660-
* true too, verify it here.
1661-
*/
1662-
VM_BUG_ON(is_linear_pfn_mapping(vma) || vma->vm_flags & VM_NO_THP);
1658+
VM_BUG_ON(vma->vm_flags & VM_NO_THP);
16631659
hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
16641660
hend = vma->vm_end & HPAGE_PMD_MASK;
16651661
if (hstart < hend)
@@ -1912,11 +1908,7 @@ static void collapse_huge_page(struct mm_struct *mm,
19121908
goto out;
19131909
if (is_vma_temporary_stack(vma))
19141910
goto out;
1915-
/*
1916-
* If is_pfn_mapping() is true is_learn_pfn_mapping() must be
1917-
* true too, verify it here.
1918-
*/
1919-
VM_BUG_ON(is_linear_pfn_mapping(vma) || vma->vm_flags & VM_NO_THP);
1911+
VM_BUG_ON(vma->vm_flags & VM_NO_THP);
19201912

19211913
pgd = pgd_offset(mm, address);
19221914
if (!pgd_present(*pgd))
@@ -2154,12 +2146,7 @@ static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
21542146
goto skip;
21552147
if (is_vma_temporary_stack(vma))
21562148
goto skip;
2157-
/*
2158-
* If is_pfn_mapping() is true is_learn_pfn_mapping()
2159-
* must be true too, verify it here.
2160-
*/
2161-
VM_BUG_ON(is_linear_pfn_mapping(vma) ||
2162-
vma->vm_flags & VM_NO_THP);
2149+
VM_BUG_ON(vma->vm_flags & VM_NO_THP);
21632150

21642151
hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
21652152
hend = vma->vm_end & HPAGE_PMD_MASK;

‎mm/memory.c

Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1055,7 +1055,7 @@ int copy_page_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
10551055
if (is_vm_hugetlb_page(vma))
10561056
return copy_hugetlb_page_range(dst_mm, src_mm, vma);
10571057

1058-
if (unlikely(is_pfn_mapping(vma))) {
1058+
if (unlikely(vma->vm_flags & VM_PFNMAP)) {
10591059
/*
10601060
* We do not free on error cases below as remove_vma
10611061
* gets called on error from higher level routine
@@ -1327,7 +1327,7 @@ static void unmap_single_vma(struct mmu_gather *tlb,
13271327
if (vma->vm_file)
13281328
uprobe_munmap(vma, start, end);
13291329

1330-
if (unlikely(is_pfn_mapping(vma)))
1330+
if (unlikely(vma->vm_flags & VM_PFNMAP))
13311331
untrack_pfn(vma, 0, 0);
13321332

13331333
if (start != end) {
@@ -2299,26 +2299,20 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr,
22992299
* There's a horrible special case to handle copy-on-write
23002300
* behaviour that some programs depend on. We mark the "original"
23012301
* un-COW'ed pages by matching them up with "vma->vm_pgoff".
2302+
* See vm_normal_page() for details.
23022303
*/
2303-
if (addr == vma->vm_start && end == vma->vm_end) {
2304+
if (is_cow_mapping(vma->vm_flags)) {
2305+
if (addr != vma->vm_start || end != vma->vm_end)
2306+
return -EINVAL;
23042307
vma->vm_pgoff = pfn;
2305-
vma->vm_flags |= VM_PFN_AT_MMAP;
2306-
} else if (is_cow_mapping(vma->vm_flags))
2308+
}
2309+
2310+
err = track_pfn_remap(vma, &prot, pfn, addr, PAGE_ALIGN(size));
2311+
if (err)
23072312
return -EINVAL;
23082313

23092314
vma->vm_flags |= VM_IO | VM_RESERVED | VM_PFNMAP;
23102315

2311-
err = track_pfn_remap(vma, &prot, pfn, PAGE_ALIGN(size));
2312-
if (err) {
2313-
/*
2314-
* To indicate that track_pfn related cleanup is not
2315-
* needed from higher level routine calling unmap_vmas
2316-
*/
2317-
vma->vm_flags &= ~(VM_IO | VM_RESERVED | VM_PFNMAP);
2318-
vma->vm_flags &= ~VM_PFN_AT_MMAP;
2319-
return -EINVAL;
2320-
}
2321-
23222316
BUG_ON(addr >= end);
23232317
pfn -= addr >> PAGE_SHIFT;
23242318
pgd = pgd_offset(mm, addr);

0 commit comments

Comments
 (0)