Skip to content

Commit 5bdcd51

Browse files
anadavIngo Molnar
anadav
authored and
Ingo Molnar
committed
x86/jump-labels: Macrofy inline assembly code to work around GCC inlining bugs
As described in: 77b0bf5: ("kbuild/Makefile: Prepare for using macros in inline assembly code to work around asm() related GCC inlining bugs") GCC's inlining heuristics are broken with common asm() patterns used in kernel code, resulting in the effective disabling of inlining. The workaround is to set an assembly macro and call it from the inline assembly block - which is also a minor cleanup for the jump-label code. As a result the code size is slightly increased, but inlining decisions are better: text data bss dec hex filename 18163528 10226300 2957312 31347140 1de51c4 ./vmlinux before 18163608 10227348 2957312 31348268 1de562c ./vmlinux after (+1128) And functions such as intel_pstate_adjust_policy_max(), kvm_cpu_accept_dm_intr(), kvm_register_readl() are inlined. Tested-by: Kees Cook <keescook@chromium.org> Signed-off-by: Nadav Amit <namit@vmware.com> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: Andy Lutomirski <luto@amacapital.net> Cc: Borislav Petkov <bp@alien8.de> Cc: Brian Gerst <brgerst@gmail.com> Cc: Denys Vlasenko <dvlasenk@redhat.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Kate Stewart <kstewart@linuxfoundation.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Philippe Ombredanne <pombredanne@nexb.com> Cc: Thomas Gleixner <tglx@linutronix.de> Link: http://lkml.kernel.org/r/20181005202718.229565-4-namit@vmware.com Link: https://lore.kernel.org/lkml/20181003213100.189959-11-namit@vmware.com/T/#u Signed-off-by: Ingo Molnar <mingo@kernel.org>
1 parent d5a581d commit 5bdcd51

File tree

3 files changed

+20
-55
lines changed

3 files changed

+20
-55
lines changed

‎arch/x86/entry/calling.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ For 32-bit we have the following conventions - kernel is built with
338338
.macro CALL_enter_from_user_mode
339339
#ifdef CONFIG_CONTEXT_TRACKING
340340
#ifdef HAVE_JUMP_LABEL
341-
STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_enabled, def=0
341+
STATIC_BRANCH_JMP l_yes=.Lafter_call_\@, key=context_tracking_enabled, branch=1
342342
#endif
343343
call enter_from_user_mode
344344
.Lafter_call_\@:

‎arch/x86/include/asm/jump_label.h

Lines changed: 18 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,6 @@
22
#ifndef _ASM_X86_JUMP_LABEL_H
33
#define _ASM_X86_JUMP_LABEL_H
44

5-
#ifndef HAVE_JUMP_LABEL
6-
/*
7-
* For better or for worse, if jump labels (the gcc extension) are missing,
8-
* then the entire static branch patching infrastructure is compiled out.
9-
* If that happens, the code in here will malfunction. Raise a compiler
10-
* error instead.
11-
*
12-
* In theory, jump labels and the static branch patching infrastructure
13-
* could be decoupled to fix this.
14-
*/
15-
#error asm/jump_label.h included on a non-jump-label kernel
16-
#endif
17-
185
#define JUMP_LABEL_NOP_SIZE 5
196

207
#ifdef CONFIG_X86_64
@@ -33,30 +20,18 @@
3320

3421
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
3522
{
36-
asm_volatile_goto("1:"
37-
".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
38-
".pushsection __jump_table, \"aw\" \n\t"
39-
_ASM_ALIGN "\n\t"
40-
".long 1b - ., %l[l_yes] - . \n\t"
41-
_ASM_PTR "%c0 + %c1 - .\n\t"
42-
".popsection \n\t"
43-
: : "i" (key), "i" (branch) : : l_yes);
44-
23+
asm_volatile_goto("STATIC_BRANCH_NOP l_yes=\"%l[l_yes]\" key=\"%c0\" "
24+
"branch=\"%c1\""
25+
: : "i" (key), "i" (branch) : : l_yes);
4526
return false;
4627
l_yes:
4728
return true;
4829
}
4930

5031
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
5132
{
52-
asm_volatile_goto("1:"
53-
".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
54-
"2:\n\t"
55-
".pushsection __jump_table, \"aw\" \n\t"
56-
_ASM_ALIGN "\n\t"
57-
".long 1b - ., %l[l_yes] - . \n\t"
58-
_ASM_PTR "%c0 + %c1 - .\n\t"
59-
".popsection \n\t"
33+
asm_volatile_goto("STATIC_BRANCH_JMP l_yes=\"%l[l_yes]\" key=\"%c0\" "
34+
"branch=\"%c1\""
6035
: : "i" (key), "i" (branch) : : l_yes);
6136

6237
return false;
@@ -66,37 +41,26 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
6641

6742
#else /* __ASSEMBLY__ */
6843

69-
.macro STATIC_JUMP_IF_TRUE target, key, def
70-
.Lstatic_jump_\@:
71-
.if \def
72-
/* Equivalent to "jmp.d32 \target" */
73-
.byte 0xe9
74-
.long \target - .Lstatic_jump_after_\@
75-
.Lstatic_jump_after_\@:
76-
.else
77-
.byte STATIC_KEY_INIT_NOP
78-
.endif
44+
.macro STATIC_BRANCH_NOP l_yes:req key:req branch:req
45+
.Lstatic_branch_nop_\@:
46+
.byte STATIC_KEY_INIT_NOP
47+
.Lstatic_branch_no_after_\@:
7948
.pushsection __jump_table, "aw"
8049
_ASM_ALIGN
81-
.long .Lstatic_jump_\@ - ., \target - .
82-
_ASM_PTR \key - .
50+
.long .Lstatic_branch_nop_\@ - ., \l_yes - .
51+
_ASM_PTR \key + \branch - .
8352
.popsection
8453
.endm
8554

86-
.macro STATIC_JUMP_IF_FALSE target, key, def
87-
.Lstatic_jump_\@:
88-
.if \def
89-
.byte STATIC_KEY_INIT_NOP
90-
.else
91-
/* Equivalent to "jmp.d32 \target" */
92-
.byte 0xe9
93-
.long \target - .Lstatic_jump_after_\@
94-
.Lstatic_jump_after_\@:
95-
.endif
55+
.macro STATIC_BRANCH_JMP l_yes:req key:req branch:req
56+
.Lstatic_branch_jmp_\@:
57+
.byte 0xe9
58+
.long \l_yes - .Lstatic_branch_jmp_after_\@
59+
.Lstatic_branch_jmp_after_\@:
9660
.pushsection __jump_table, "aw"
9761
_ASM_ALIGN
98-
.long .Lstatic_jump_\@ - ., \target - .
99-
_ASM_PTR \key + 1 - .
62+
.long .Lstatic_branch_jmp_\@ - ., \l_yes - .
63+
_ASM_PTR \key + \branch - .
10064
.popsection
10165
.endm
10266

‎arch/x86/kernel/macros.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,3 +13,4 @@
1313
#include <asm/paravirt.h>
1414
#include <asm/asm.h>
1515
#include <asm/cpufeature.h>
16+
#include <asm/jump_label.h>

0 commit comments

Comments
 (0)