Skip to content

Commit 70ffdb9

Browse files
David HildenbrandIngo Molnar
David Hildenbrand
authored and
Ingo Molnar
committed
mm/fault, arch: Use pagefault_disable() to check for disabled pagefaults in the handler
Introduce faulthandler_disabled() and use it to check for irq context and disabled pagefaults (via pagefault_disable()) in the pagefault handlers. Please note that we keep the in_atomic() checks in place - to detect whether in irq context (in which case preemption is always properly disabled). In contrast, preempt_disable() should never be used to disable pagefaults. With !CONFIG_PREEMPT_COUNT, preempt_disable() doesn't modify the preempt counter, and therefore the result of in_atomic() differs. We validate that condition by using might_fault() checks when calling might_sleep(). Therefore, add a comment to faulthandler_disabled(), describing why this is needed. faulthandler_disabled() and pagefault_disable() are defined in linux/uaccess.h, so let's properly add that include to all relevant files. This patch is based on a patch from Thomas Gleixner. Reviewed-and-tested-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: David Hildenbrand <dahi@linux.vnet.ibm.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: David.Laight@ACULAB.COM Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: airlied@linux.ie Cc: akpm@linux-foundation.org Cc: benh@kernel.crashing.org Cc: bigeasy@linutronix.de Cc: borntraeger@de.ibm.com Cc: daniel.vetter@intel.com Cc: heiko.carstens@de.ibm.com Cc: herbert@gondor.apana.org.au Cc: hocko@suse.cz Cc: hughd@google.com Cc: mst@redhat.com Cc: paulus@samba.org Cc: ralf@linux-mips.org Cc: schwidefsky@de.ibm.com Cc: yang.shi@windriver.com Link: http://lkml.kernel.org/r/1431359540-32227-7-git-send-email-dahi@linux.vnet.ibm.com Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent ce01948 commit 70ffdb9

File tree

30 files changed

+72
-57
lines changed

30 files changed

+72
-57
lines changed

‎arch/alpha/mm/fault.c

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,7 @@
2323
#include <linux/smp.h>
2424
#include <linux/interrupt.h>
2525
#include <linux/module.h>
26-
27-
#include <asm/uaccess.h>
26+
#include <linux/uaccess.h>
2827

2928
extern void die_if_kernel(char *,struct pt_regs *,long, unsigned long *);
3029

@@ -107,7 +106,7 @@ do_page_fault(unsigned long address, unsigned long mmcsr,
107106

108107
/* If we're in an interrupt context, or have no user context,
109108
we must not take the fault. */
110-
if (!mm || in_atomic())
109+
if (!mm || faulthandler_disabled())
111110
goto no_context;
112111

113112
#ifdef CONFIG_ALPHA_LARGE_VMALLOC

‎arch/arc/mm/fault.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ void do_page_fault(unsigned long address, struct pt_regs *regs)
8686
* If we're in an interrupt or have no user
8787
* context, we must not take the fault..
8888
*/
89-
if (in_atomic() || !mm)
89+
if (faulthandler_disabled() || !mm)
9090
goto no_context;
9191

9292
if (user_mode(regs))

‎arch/arm/mm/fault.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
276276
* If we're in an interrupt or have no user
277277
* context, we must not take the fault..
278278
*/
279-
if (in_atomic() || !mm)
279+
if (faulthandler_disabled() || !mm)
280280
goto no_context;
281281

282282
if (user_mode(regs))

‎arch/arm64/mm/fault.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
211211
* If we're in an interrupt or have no user context, we must not take
212212
* the fault.
213213
*/
214-
if (in_atomic() || !mm)
214+
if (faulthandler_disabled() || !mm)
215215
goto no_context;
216216

217217
if (user_mode(regs))

‎arch/avr32/mm/fault.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@
1414
#include <linux/pagemap.h>
1515
#include <linux/kdebug.h>
1616
#include <linux/kprobes.h>
17+
#include <linux/uaccess.h>
1718

1819
#include <asm/mmu_context.h>
1920
#include <asm/sysreg.h>
2021
#include <asm/tlb.h>
21-
#include <asm/uaccess.h>
2222

2323
#ifdef CONFIG_KPROBES
2424
static inline int notify_page_fault(struct pt_regs *regs, int trap)
@@ -81,7 +81,7 @@ asmlinkage void do_page_fault(unsigned long ecr, struct pt_regs *regs)
8181
* If we're in an interrupt or have no user context, we must
8282
* not take the fault...
8383
*/
84-
if (in_atomic() || !mm || regs->sr & SYSREG_BIT(GM))
84+
if (faulthandler_disabled() || !mm || regs->sr & SYSREG_BIT(GM))
8585
goto no_context;
8686

8787
local_irq_enable();

‎arch/cris/mm/fault.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
#include <linux/interrupt.h>
99
#include <linux/module.h>
1010
#include <linux/wait.h>
11-
#include <asm/uaccess.h>
11+
#include <linux/uaccess.h>
1212
#include <arch/system.h>
1313

1414
extern int find_fixup_code(struct pt_regs *);
@@ -109,11 +109,11 @@ do_page_fault(unsigned long address, struct pt_regs *regs,
109109
info.si_code = SEGV_MAPERR;
110110

111111
/*
112-
* If we're in an interrupt or "atomic" operation or have no
112+
* If we're in an interrupt, have pagefaults disabled or have no
113113
* user context, we must not take the fault.
114114
*/
115115

116-
if (in_atomic() || !mm)
116+
if (faulthandler_disabled() || !mm)
117117
goto no_context;
118118

119119
if (user_mode(regs))

‎arch/frv/mm/fault.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
#include <linux/kernel.h>
2020
#include <linux/ptrace.h>
2121
#include <linux/hardirq.h>
22+
#include <linux/uaccess.h>
2223

2324
#include <asm/pgtable.h>
24-
#include <asm/uaccess.h>
2525
#include <asm/gdb-stub.h>
2626

2727
/*****************************************************************************/
@@ -78,7 +78,7 @@ asmlinkage void do_page_fault(int datammu, unsigned long esr0, unsigned long ear
7878
* If we're in an interrupt or have no user
7979
* context, we must not take the fault..
8080
*/
81-
if (in_atomic() || !mm)
81+
if (faulthandler_disabled() || !mm)
8282
goto no_context;
8383

8484
if (user_mode(__frame))

‎arch/ia64/mm/fault.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,10 @@
1111
#include <linux/kprobes.h>
1212
#include <linux/kdebug.h>
1313
#include <linux/prefetch.h>
14+
#include <linux/uaccess.h>
1415

1516
#include <asm/pgtable.h>
1617
#include <asm/processor.h>
17-
#include <asm/uaccess.h>
1818

1919
extern int die(char *, struct pt_regs *, long);
2020

@@ -96,7 +96,7 @@ ia64_do_page_fault (unsigned long address, unsigned long isr, struct pt_regs *re
9696
/*
9797
* If we're in an interrupt or have no user context, we must not take the fault..
9898
*/
99-
if (in_atomic() || !mm)
99+
if (faulthandler_disabled() || !mm)
100100
goto no_context;
101101

102102
#ifdef CONFIG_VIRTUAL_MEM_MAP

‎arch/m32r/mm/fault.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@
2424
#include <linux/vt_kern.h> /* For unblank_screen() */
2525
#include <linux/highmem.h>
2626
#include <linux/module.h>
27+
#include <linux/uaccess.h>
2728

2829
#include <asm/m32r.h>
29-
#include <asm/uaccess.h>
3030
#include <asm/hardirq.h>
3131
#include <asm/mmu_context.h>
3232
#include <asm/tlbflush.h>
@@ -111,10 +111,10 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code,
111111
mm = tsk->mm;
112112

113113
/*
114-
* If we're in an interrupt or have no user context or are running in an
115-
* atomic region then we must not take the fault..
114+
* If we're in an interrupt or have no user context or have pagefaults
115+
* disabled then we must not take the fault.
116116
*/
117-
if (in_atomic() || !mm)
117+
if (faulthandler_disabled() || !mm)
118118
goto bad_area_nosemaphore;
119119

120120
if (error_code & ACE_USERMODE)

‎arch/m68k/mm/fault.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@
1010
#include <linux/ptrace.h>
1111
#include <linux/interrupt.h>
1212
#include <linux/module.h>
13+
#include <linux/uaccess.h>
1314

1415
#include <asm/setup.h>
1516
#include <asm/traps.h>
16-
#include <asm/uaccess.h>
1717
#include <asm/pgalloc.h>
1818

1919
extern void die_if_kernel(char *, struct pt_regs *, long);
@@ -81,7 +81,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
8181
* If we're in an interrupt or have no user
8282
* context, we must not take the fault..
8383
*/
84-
if (in_atomic() || !mm)
84+
if (faulthandler_disabled() || !mm)
8585
goto no_context;
8686

8787
if (user_mode(regs))

‎arch/metag/mm/fault.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ int do_page_fault(struct pt_regs *regs, unsigned long address,
105105

106106
mm = tsk->mm;
107107

108-
if (in_atomic() || !mm)
108+
if (faulthandler_disabled() || !mm)
109109
goto no_context;
110110

111111
if (user_mode(regs))

‎arch/microblaze/mm/fault.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -107,14 +107,14 @@ void do_page_fault(struct pt_regs *regs, unsigned long address,
107107
if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11)
108108
is_write = 0;
109109

110-
if (unlikely(in_atomic() || !mm)) {
110+
if (unlikely(faulthandler_disabled() || !mm)) {
111111
if (kernel_mode(regs))
112112
goto bad_area_nosemaphore;
113113

114-
/* in_atomic() in user mode is really bad,
114+
/* faulthandler_disabled() in user mode is really bad,
115115
as is current->mm == NULL. */
116-
pr_emerg("Page fault in user mode with in_atomic(), mm = %p\n",
117-
mm);
116+
pr_emerg("Page fault in user mode with faulthandler_disabled(), mm = %p\n",
117+
mm);
118118
pr_emerg("r15 = %lx MSR = %lx\n",
119119
regs->r15, regs->msr);
120120
die("Weird page fault", regs, SIGSEGV);

‎arch/mips/mm/fault.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,10 @@
2121
#include <linux/module.h>
2222
#include <linux/kprobes.h>
2323
#include <linux/perf_event.h>
24+
#include <linux/uaccess.h>
2425

2526
#include <asm/branch.h>
2627
#include <asm/mmu_context.h>
27-
#include <asm/uaccess.h>
2828
#include <asm/ptrace.h>
2929
#include <asm/highmem.h> /* For VMALLOC_END */
3030
#include <linux/kdebug.h>
@@ -94,7 +94,7 @@ static void __kprobes __do_page_fault(struct pt_regs *regs, unsigned long write,
9494
* If we're in an interrupt or have no user
9595
* context, we must not take the fault..
9696
*/
97-
if (in_atomic() || !mm)
97+
if (faulthandler_disabled() || !mm)
9898
goto bad_area_nosemaphore;
9999

100100
if (user_mode(regs))

‎arch/mn10300/mm/fault.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,8 @@
2323
#include <linux/interrupt.h>
2424
#include <linux/init.h>
2525
#include <linux/vt_kern.h> /* For unblank_screen() */
26+
#include <linux/uaccess.h>
2627

27-
#include <asm/uaccess.h>
2828
#include <asm/pgalloc.h>
2929
#include <asm/hardirq.h>
3030
#include <asm/cpu-regs.h>
@@ -168,7 +168,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long fault_code,
168168
* If we're in an interrupt or have no user
169169
* context, we must not take the fault..
170170
*/
171-
if (in_atomic() || !mm)
171+
if (faulthandler_disabled() || !mm)
172172
goto no_context;
173173

174174
if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR)

‎arch/nios2/mm/fault.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long cause,
7777
* If we're in an interrupt or have no user
7878
* context, we must not take the fault..
7979
*/
80-
if (in_atomic() || !mm)
80+
if (faulthandler_disabled() || !mm)
8181
goto bad_area_nosemaphore;
8282

8383
if (user_mode(regs))

‎arch/parisc/kernel/traps.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@
2626
#include <linux/console.h>
2727
#include <linux/bug.h>
2828
#include <linux/ratelimit.h>
29+
#include <linux/uaccess.h>
2930

3031
#include <asm/assembly.h>
31-
#include <asm/uaccess.h>
3232
#include <asm/io.h>
3333
#include <asm/irq.h>
3434
#include <asm/traps.h>
@@ -800,7 +800,7 @@ void notrace handle_interruption(int code, struct pt_regs *regs)
800800
* unless pagefault_disable() was called before.
801801
*/
802802

803-
if (fault_space == 0 && !in_atomic())
803+
if (fault_space == 0 && !faulthandler_disabled())
804804
{
805805
pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC);
806806
parisc_terminate("Kernel Fault", regs, code, fault_address);

‎arch/parisc/mm/fault.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
#include <linux/sched.h>
1616
#include <linux/interrupt.h>
1717
#include <linux/module.h>
18+
#include <linux/uaccess.h>
1819

19-
#include <asm/uaccess.h>
2020
#include <asm/traps.h>
2121

2222
/* Various important other fields */
@@ -207,7 +207,7 @@ void do_page_fault(struct pt_regs *regs, unsigned long code,
207207
int fault;
208208
unsigned int flags;
209209

210-
if (in_atomic())
210+
if (pagefault_disabled())
211211
goto no_context;
212212

213213
tsk = current;

‎arch/powerpc/mm/fault.c

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@
3333
#include <linux/ratelimit.h>
3434
#include <linux/context_tracking.h>
3535
#include <linux/hugetlb.h>
36+
#include <linux/uaccess.h>
3637

3738
#include <asm/firmware.h>
3839
#include <asm/page.h>
3940
#include <asm/pgtable.h>
4041
#include <asm/mmu.h>
4142
#include <asm/mmu_context.h>
42-
#include <asm/uaccess.h>
4343
#include <asm/tlbflush.h>
4444
#include <asm/siginfo.h>
4545
#include <asm/debug.h>
@@ -272,15 +272,16 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address,
272272
if (!arch_irq_disabled_regs(regs))
273273
local_irq_enable();
274274

275-
if (in_atomic() || mm == NULL) {
275+
if (faulthandler_disabled() || mm == NULL) {
276276
if (!user_mode(regs)) {
277277
rc = SIGSEGV;
278278
goto bail;
279279
}
280-
/* in_atomic() in user mode is really bad,
280+
/* faulthandler_disabled() in user mode is really bad,
281281
as is current->mm == NULL. */
282282
printk(KERN_EMERG "Page fault in user mode with "
283-
"in_atomic() = %d mm = %p\n", in_atomic(), mm);
283+
"faulthandler_disabled() = %d mm = %p\n",
284+
faulthandler_disabled(), mm);
284285
printk(KERN_EMERG "NIP = %lx MSR = %lx\n",
285286
regs->nip, regs->msr);
286287
die("Weird page fault", regs, SIGSEGV);

‎arch/s390/mm/fault.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -399,7 +399,7 @@ static inline int do_exception(struct pt_regs *regs, int access)
399399
* user context.
400400
*/
401401
fault = VM_FAULT_BADCONTEXT;
402-
if (unlikely(!user_space_fault(regs) || in_atomic() || !mm))
402+
if (unlikely(!user_space_fault(regs) || faulthandler_disabled() || !mm))
403403
goto out;
404404

405405
address = trans_exc_code & __FAIL_ADDR_MASK;

‎arch/score/mm/fault.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
#include <linux/string.h>
3535
#include <linux/types.h>
3636
#include <linux/ptrace.h>
37+
#include <linux/uaccess.h>
3738

3839
/*
3940
* This routine handles page faults. It determines the address,
@@ -73,7 +74,7 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
7374
* If we're in an interrupt or have no user
7475
* context, we must not take the fault..
7576
*/
76-
if (in_atomic() || !mm)
77+
if (pagefault_disabled() || !mm)
7778
goto bad_area_nosemaphore;
7879

7980
if (user_mode(regs))

‎arch/sh/mm/fault.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
#include <linux/kprobes.h>
1818
#include <linux/perf_event.h>
1919
#include <linux/kdebug.h>
20+
#include <linux/uaccess.h>
2021
#include <asm/io_trapped.h>
2122
#include <asm/mmu_context.h>
2223
#include <asm/tlbflush.h>
@@ -438,9 +439,9 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
438439

439440
/*
440441
* If we're in an interrupt, have no user context or are running
441-
* in an atomic region then we must not take the fault:
442+
* with pagefaults disabled then we must not take the fault:
442443
*/
443-
if (unlikely(in_atomic() || !mm)) {
444+
if (unlikely(faulthandler_disabled() || !mm)) {
444445
bad_area_nosemaphore(regs, error_code, address);
445446
return;
446447
}

‎arch/sparc/mm/fault_32.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include <linux/perf_event.h>
2222
#include <linux/interrupt.h>
2323
#include <linux/kdebug.h>
24+
#include <linux/uaccess.h>
2425

2526
#include <asm/page.h>
2627
#include <asm/pgtable.h>
@@ -29,7 +30,6 @@
2930
#include <asm/setup.h>
3031
#include <asm/smp.h>
3132
#include <asm/traps.h>
32-
#include <asm/uaccess.h>
3333

3434
#include "mm_32.h"
3535

@@ -196,7 +196,7 @@ asmlinkage void do_sparc_fault(struct pt_regs *regs, int text_fault, int write,
196196
* If we're in an interrupt or have no user
197197
* context, we must not take the fault..
198198
*/
199-
if (in_atomic() || !mm)
199+
if (pagefault_disabled() || !mm)
200200
goto no_context;
201201

202202
perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);

0 commit comments

Comments
 (0)