Skip to content

Commit aac2fea

Browse files
kiryltorvalds
kiryl
authored and
torvalds
committed
rmap: do not call mmu_notifier_invalidate_page() under ptl
MMU notifiers can sleep, but in page_mkclean_one() we call mmu_notifier_invalidate_page() under page table lock. Let's instead use mmu_notifier_invalidate_range() outside page_vma_mapped_walk() loop. [jglisse@redhat.com: try_to_unmap_one() do not call mmu_notifier under ptl] Link: http://lkml.kernel.org/r/20170809204333.27485-1-jglisse@redhat.com Link: http://lkml.kernel.org/r/20170804134928.l4klfcnqatni7vsc@black.fi.intel.com Fixes: c7ab0d2 ("mm: convert try_to_unmap_one() to use page_vma_mapped_walk()") Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com> Signed-off-by: Jérôme Glisse <jglisse@redhat.com> Reported-by: axie <axie@amd.com> Cc: Alex Deucher <alexander.deucher@amd.com> Cc: "Writer, Tim" <Tim.Writer@amd.com> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent d041353 commit aac2fea

File tree

1 file changed

+30
-22
lines changed

1 file changed

+30
-22
lines changed

‎mm/rmap.c

Lines changed: 30 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -888,22 +888,22 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
888888
.flags = PVMW_SYNC,
889889
};
890890
int *cleaned = arg;
891+
bool invalidation_needed = false;
891892

892893
while (page_vma_mapped_walk(&pvmw)) {
893894
int ret = 0;
894-
address = pvmw.address;
895895
if (pvmw.pte) {
896896
pte_t entry;
897897
pte_t *pte = pvmw.pte;
898898

899899
if (!pte_dirty(*pte) && !pte_write(*pte))
900900
continue;
901901

902-
flush_cache_page(vma, address, pte_pfn(*pte));
903-
entry = ptep_clear_flush(vma, address, pte);
902+
flush_cache_page(vma, pvmw.address, pte_pfn(*pte));
903+
entry = ptep_clear_flush(vma, pvmw.address, pte);
904904
entry = pte_wrprotect(entry);
905905
entry = pte_mkclean(entry);
906-
set_pte_at(vma->vm_mm, address, pte, entry);
906+
set_pte_at(vma->vm_mm, pvmw.address, pte, entry);
907907
ret = 1;
908908
} else {
909909
#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
@@ -913,11 +913,11 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
913913
if (!pmd_dirty(*pmd) && !pmd_write(*pmd))
914914
continue;
915915

916-
flush_cache_page(vma, address, page_to_pfn(page));
917-
entry = pmdp_huge_clear_flush(vma, address, pmd);
916+
flush_cache_page(vma, pvmw.address, page_to_pfn(page));
917+
entry = pmdp_huge_clear_flush(vma, pvmw.address, pmd);
918918
entry = pmd_wrprotect(entry);
919919
entry = pmd_mkclean(entry);
920-
set_pmd_at(vma->vm_mm, address, pmd, entry);
920+
set_pmd_at(vma->vm_mm, pvmw.address, pmd, entry);
921921
ret = 1;
922922
#else
923923
/* unexpected pmd-mapped page? */
@@ -926,11 +926,16 @@ static bool page_mkclean_one(struct page *page, struct vm_area_struct *vma,
926926
}
927927

928928
if (ret) {
929-
mmu_notifier_invalidate_page(vma->vm_mm, address);
930929
(*cleaned)++;
930+
invalidation_needed = true;
931931
}
932932
}
933933

934+
if (invalidation_needed) {
935+
mmu_notifier_invalidate_range(vma->vm_mm, address,
936+
address + (1UL << compound_order(page)));
937+
}
938+
934939
return true;
935940
}
936941

@@ -1323,7 +1328,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
13231328
};
13241329
pte_t pteval;
13251330
struct page *subpage;
1326-
bool ret = true;
1331+
bool ret = true, invalidation_needed = false;
13271332
enum ttu_flags flags = (enum ttu_flags)arg;
13281333

13291334
/* munlock has nothing to gain from examining un-locked vmas */
@@ -1363,11 +1368,9 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
13631368
VM_BUG_ON_PAGE(!pvmw.pte, page);
13641369

13651370
subpage = page - page_to_pfn(page) + pte_pfn(*pvmw.pte);
1366-
address = pvmw.address;
1367-
13681371

13691372
if (!(flags & TTU_IGNORE_ACCESS)) {
1370-
if (ptep_clear_flush_young_notify(vma, address,
1373+
if (ptep_clear_flush_young_notify(vma, pvmw.address,
13711374
pvmw.pte)) {
13721375
ret = false;
13731376
page_vma_mapped_walk_done(&pvmw);
@@ -1376,7 +1379,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
13761379
}
13771380

13781381
/* Nuke the page table entry. */
1379-
flush_cache_page(vma, address, pte_pfn(*pvmw.pte));
1382+
flush_cache_page(vma, pvmw.address, pte_pfn(*pvmw.pte));
13801383
if (should_defer_flush(mm, flags)) {
13811384
/*
13821385
* We clear the PTE but do not flush so potentially
@@ -1386,11 +1389,12 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
13861389
* transition on a cached TLB entry is written through
13871390
* and traps if the PTE is unmapped.
13881391
*/
1389-
pteval = ptep_get_and_clear(mm, address, pvmw.pte);
1392+
pteval = ptep_get_and_clear(mm, pvmw.address,
1393+
pvmw.pte);
13901394

13911395
set_tlb_ubc_flush_pending(mm, pte_dirty(pteval));
13921396
} else {
1393-
pteval = ptep_clear_flush(vma, address, pvmw.pte);
1397+
pteval = ptep_clear_flush(vma, pvmw.address, pvmw.pte);
13941398
}
13951399

13961400
/* Move the dirty bit to the page. Now the pte is gone. */
@@ -1405,12 +1409,12 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
14051409
if (PageHuge(page)) {
14061410
int nr = 1 << compound_order(page);
14071411
hugetlb_count_sub(nr, mm);
1408-
set_huge_swap_pte_at(mm, address,
1412+
set_huge_swap_pte_at(mm, pvmw.address,
14091413
pvmw.pte, pteval,
14101414
vma_mmu_pagesize(vma));
14111415
} else {
14121416
dec_mm_counter(mm, mm_counter(page));
1413-
set_pte_at(mm, address, pvmw.pte, pteval);
1417+
set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
14141418
}
14151419

14161420
} else if (pte_unused(pteval)) {
@@ -1434,7 +1438,7 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
14341438
swp_pte = swp_entry_to_pte(entry);
14351439
if (pte_soft_dirty(pteval))
14361440
swp_pte = pte_swp_mksoft_dirty(swp_pte);
1437-
set_pte_at(mm, address, pvmw.pte, swp_pte);
1441+
set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
14381442
} else if (PageAnon(page)) {
14391443
swp_entry_t entry = { .val = page_private(subpage) };
14401444
pte_t swp_pte;
@@ -1460,15 +1464,15 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
14601464
* If the page was redirtied, it cannot be
14611465
* discarded. Remap the page to page table.
14621466
*/
1463-
set_pte_at(mm, address, pvmw.pte, pteval);
1467+
set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
14641468
SetPageSwapBacked(page);
14651469
ret = false;
14661470
page_vma_mapped_walk_done(&pvmw);
14671471
break;
14681472
}
14691473

14701474
if (swap_duplicate(entry) < 0) {
1471-
set_pte_at(mm, address, pvmw.pte, pteval);
1475+
set_pte_at(mm, pvmw.address, pvmw.pte, pteval);
14721476
ret = false;
14731477
page_vma_mapped_walk_done(&pvmw);
14741478
break;
@@ -1484,14 +1488,18 @@ static bool try_to_unmap_one(struct page *page, struct vm_area_struct *vma,
14841488
swp_pte = swp_entry_to_pte(entry);
14851489
if (pte_soft_dirty(pteval))
14861490
swp_pte = pte_swp_mksoft_dirty(swp_pte);
1487-
set_pte_at(mm, address, pvmw.pte, swp_pte);
1491+
set_pte_at(mm, pvmw.address, pvmw.pte, swp_pte);
14881492
} else
14891493
dec_mm_counter(mm, mm_counter_file(page));
14901494
discard:
14911495
page_remove_rmap(subpage, PageHuge(page));
14921496
put_page(page);
1493-
mmu_notifier_invalidate_page(mm, address);
1497+
invalidation_needed = true;
14941498
}
1499+
1500+
if (invalidation_needed)
1501+
mmu_notifier_invalidate_range(mm, address,
1502+
address + (1UL << compound_order(page)));
14951503
return ret;
14961504
}
14971505

0 commit comments

Comments
 (0)