Skip to content

Commit 02678a5

Browse files
author
Ingo Molnar
committed
Merge branch 'core/core' into x86/build, to prevent conflicts
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2 parents bce6824 + 77ac1c0 commit 02678a5

File tree

21 files changed

+226
-174
lines changed

21 files changed

+226
-174
lines changed

‎arch/Kconfig

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,9 @@ config HAVE_PERF_USER_STACK_DUMP
359359
config HAVE_ARCH_JUMP_LABEL
360360
bool
361361

362+
config HAVE_ARCH_JUMP_LABEL_RELATIVE
363+
bool
364+
362365
config HAVE_RCU_TABLE_FREE
363366
bool
364367

‎arch/arm64/Kconfig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ config ARM64
104104
select HAVE_ARCH_BITREVERSE
105105
select HAVE_ARCH_HUGE_VMAP
106106
select HAVE_ARCH_JUMP_LABEL
107+
select HAVE_ARCH_JUMP_LABEL_RELATIVE
107108
select HAVE_ARCH_KASAN if !(ARM64_16K_PAGES && ARM64_VA_BITS_48)
108109
select HAVE_ARCH_KGDB
109110
select HAVE_ARCH_MMAP_RND_BITS

‎arch/arm64/include/asm/jump_label.h

Lines changed: 18 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -26,41 +26,39 @@
2626

2727
#define JUMP_LABEL_NOP_SIZE AARCH64_INSN_SIZE
2828

29-
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
29+
static __always_inline bool arch_static_branch(struct static_key *key,
30+
bool branch)
3031
{
31-
asm_volatile_goto("1: nop\n\t"
32-
".pushsection __jump_table, \"aw\"\n\t"
33-
".align 3\n\t"
34-
".quad 1b, %l[l_yes], %c0\n\t"
35-
".popsection\n\t"
32+
asm_volatile_goto(
33+
"1: nop \n\t"
34+
" .pushsection __jump_table, \"aw\" \n\t"
35+
" .align 3 \n\t"
36+
" .long 1b - ., %l[l_yes] - . \n\t"
37+
" .quad %c0 - . \n\t"
38+
" .popsection \n\t"
3639
: : "i"(&((char *)key)[branch]) : : l_yes);
3740

3841
return false;
3942
l_yes:
4043
return true;
4144
}
4245

43-
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
46+
static __always_inline bool arch_static_branch_jump(struct static_key *key,
47+
bool branch)
4448
{
45-
asm_volatile_goto("1: b %l[l_yes]\n\t"
46-
".pushsection __jump_table, \"aw\"\n\t"
47-
".align 3\n\t"
48-
".quad 1b, %l[l_yes], %c0\n\t"
49-
".popsection\n\t"
49+
asm_volatile_goto(
50+
"1: b %l[l_yes] \n\t"
51+
" .pushsection __jump_table, \"aw\" \n\t"
52+
" .align 3 \n\t"
53+
" .long 1b - ., %l[l_yes] - . \n\t"
54+
" .quad %c0 - . \n\t"
55+
" .popsection \n\t"
5056
: : "i"(&((char *)key)[branch]) : : l_yes);
5157

5258
return false;
5359
l_yes:
5460
return true;
5561
}
5662

57-
typedef u64 jump_label_t;
58-
59-
struct jump_entry {
60-
jump_label_t code;
61-
jump_label_t target;
62-
jump_label_t key;
63-
};
64-
6563
#endif /* __ASSEMBLY__ */
6664
#endif /* __ASM_JUMP_LABEL_H */

‎arch/arm64/kernel/jump_label.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,12 +25,12 @@
2525
void arch_jump_label_transform(struct jump_entry *entry,
2626
enum jump_label_type type)
2727
{
28-
void *addr = (void *)entry->code;
28+
void *addr = (void *)jump_entry_code(entry);
2929
u32 insn;
3030

3131
if (type == JUMP_LABEL_JMP) {
32-
insn = aarch64_insn_gen_branch_imm(entry->code,
33-
entry->target,
32+
insn = aarch64_insn_gen_branch_imm(jump_entry_code(entry),
33+
jump_entry_target(entry),
3434
AARCH64_INSN_BRANCH_NOLINK);
3535
} else {
3636
insn = aarch64_insn_gen_nop();

‎arch/s390/Kconfig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,7 @@ config S390
120120
select HAVE_ALIGNED_STRUCT_PAGE if SLUB
121121
select HAVE_ARCH_AUDITSYSCALL
122122
select HAVE_ARCH_JUMP_LABEL
123+
select HAVE_ARCH_JUMP_LABEL_RELATIVE
123124
select CPU_NO_EFFICIENT_FFS if !HAVE_MARCH_Z9_109_FEATURES
124125
select HAVE_ARCH_SECCOMP_FILTER
125126
select HAVE_ARCH_SOFT_DIRTY

‎arch/s390/include/asm/jump_label.h

Lines changed: 16 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,41 +14,33 @@
1414
* We use a brcl 0,2 instruction for jump labels at compile time so it
1515
* can be easily distinguished from a hotpatch generated instruction.
1616
*/
17-
static __always_inline bool arch_static_branch(struct static_key *key, bool branch)
17+
static inline bool arch_static_branch(struct static_key *key, bool branch)
1818
{
19-
asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n"
20-
".pushsection __jump_table, \"aw\"\n"
21-
".balign 8\n"
22-
".quad 0b, %l[label], %0\n"
23-
".popsection\n"
24-
: : "X" (&((char *)key)[branch]) : : label);
25-
19+
asm_volatile_goto("0: brcl 0,"__stringify(JUMP_LABEL_NOP_OFFSET)"\n"
20+
".pushsection __jump_table,\"aw\"\n"
21+
".balign 8\n"
22+
".long 0b-.,%l[label]-.\n"
23+
".quad %0-.\n"
24+
".popsection\n"
25+
: : "X" (&((char *)key)[branch]) : : label);
2626
return false;
2727
label:
2828
return true;
2929
}
3030

31-
static __always_inline bool arch_static_branch_jump(struct static_key *key, bool branch)
31+
static inline bool arch_static_branch_jump(struct static_key *key, bool branch)
3232
{
33-
asm_volatile_goto("0: brcl 15, %l[label]\n"
34-
".pushsection __jump_table, \"aw\"\n"
35-
".balign 8\n"
36-
".quad 0b, %l[label], %0\n"
37-
".popsection\n"
38-
: : "X" (&((char *)key)[branch]) : : label);
39-
33+
asm_volatile_goto("0: brcl 15,%l[label]\n"
34+
".pushsection __jump_table,\"aw\"\n"
35+
".balign 8\n"
36+
".long 0b-.,%l[label]-.\n"
37+
".quad %0-.\n"
38+
".popsection\n"
39+
: : "X" (&((char *)key)[branch]) : : label);
4040
return false;
4141
label:
4242
return true;
4343
}
4444

45-
typedef unsigned long jump_label_t;
46-
47-
struct jump_entry {
48-
jump_label_t code;
49-
jump_label_t target;
50-
jump_label_t key;
51-
};
52-
5345
#endif /* __ASSEMBLY__ */
5446
#endif

‎arch/s390/kernel/jump_label.c

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ static void jump_label_make_branch(struct jump_entry *entry, struct insn *insn)
3333
{
3434
/* brcl 15,offset */
3535
insn->opcode = 0xc0f4;
36-
insn->offset = (entry->target - entry->code) >> 1;
36+
insn->offset = (jump_entry_target(entry) - jump_entry_code(entry)) >> 1;
3737
}
3838

3939
static void jump_label_bug(struct jump_entry *entry, struct insn *expected,
4040
struct insn *new)
4141
{
42-
unsigned char *ipc = (unsigned char *)entry->code;
42+
unsigned char *ipc = (unsigned char *)jump_entry_code(entry);
4343
unsigned char *ipe = (unsigned char *)expected;
4444
unsigned char *ipn = (unsigned char *)new;
4545

@@ -59,6 +59,7 @@ static void __jump_label_transform(struct jump_entry *entry,
5959
enum jump_label_type type,
6060
int init)
6161
{
62+
void *code = (void *)jump_entry_code(entry);
6263
struct insn old, new;
6364

6465
if (type == JUMP_LABEL_JMP) {
@@ -69,13 +70,13 @@ static void __jump_label_transform(struct jump_entry *entry,
6970
jump_label_make_nop(entry, &new);
7071
}
7172
if (init) {
72-
if (memcmp((void *)entry->code, &orignop, sizeof(orignop)))
73+
if (memcmp(code, &orignop, sizeof(orignop)))
7374
jump_label_bug(entry, &orignop, &new);
7475
} else {
75-
if (memcmp((void *)entry->code, &old, sizeof(old)))
76+
if (memcmp(code, &old, sizeof(old)))
7677
jump_label_bug(entry, &old, &new);
7778
}
78-
s390_kernel_write((void *)entry->code, &new, sizeof(new));
79+
s390_kernel_write(code, &new, sizeof(new));
7980
}
8081

8182
static int __sm_arch_jump_label_transform(void *data)

‎arch/s390/kernel/vmlinux.lds.S

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ SECTIONS
6464
__start_ro_after_init = .;
6565
.data..ro_after_init : {
6666
*(.data..ro_after_init)
67+
JUMP_TABLE_DATA
6768
}
6869
EXCEPTION_TABLE(16)
6970
. = ALIGN(PAGE_SIZE);

‎arch/x86/Kconfig

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ config X86
119119
select HAVE_ARCH_AUDITSYSCALL
120120
select HAVE_ARCH_HUGE_VMAP if X86_64 || X86_PAE
121121
select HAVE_ARCH_JUMP_LABEL
122+
select HAVE_ARCH_JUMP_LABEL_RELATIVE
122123
select HAVE_ARCH_KASAN if X86_64
123124
select HAVE_ARCH_KGDB
124125
select HAVE_ARCH_MMAP_RND_BITS if MMU

‎arch/x86/include/asm/elf.h

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,7 @@ typedef struct user_fxsr_struct elf_fpxregset_t;
6262
#define R_X86_64_PC16 13 /* 16 bit sign extended pc relative */
6363
#define R_X86_64_8 14 /* Direct 8 bit sign extended */
6464
#define R_X86_64_PC8 15 /* 8 bit sign extended pc relative */
65-
66-
#define R_X86_64_NUM 16
65+
#define R_X86_64_PC64 24 /* Place relative 64-bit signed */
6766

6867
/*
6968
* These are used to set parameters in the core dumps.

‎arch/x86/include/asm/jump_label.h

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ static __always_inline bool arch_static_branch(struct static_key *key, bool bran
3737
".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
3838
".pushsection __jump_table, \"aw\" \n\t"
3939
_ASM_ALIGN "\n\t"
40-
_ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
40+
".long 1b - ., %l[l_yes] - . \n\t"
41+
_ASM_PTR "%c0 + %c1 - .\n\t"
4142
".popsection \n\t"
4243
: : "i" (key), "i" (branch) : : l_yes);
4344

@@ -53,7 +54,8 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
5354
"2:\n\t"
5455
".pushsection __jump_table, \"aw\" \n\t"
5556
_ASM_ALIGN "\n\t"
56-
_ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
57+
".long 1b - ., %l[l_yes] - . \n\t"
58+
_ASM_PTR "%c0 + %c1 - .\n\t"
5759
".popsection \n\t"
5860
: : "i" (key), "i" (branch) : : l_yes);
5961

@@ -62,18 +64,6 @@ static __always_inline bool arch_static_branch_jump(struct static_key *key, bool
6264
return true;
6365
}
6466

65-
#ifdef CONFIG_X86_64
66-
typedef u64 jump_label_t;
67-
#else
68-
typedef u32 jump_label_t;
69-
#endif
70-
71-
struct jump_entry {
72-
jump_label_t code;
73-
jump_label_t target;
74-
jump_label_t key;
75-
};
76-
7767
#else /* __ASSEMBLY__ */
7868

7969
.macro STATIC_JUMP_IF_TRUE target, key, def
@@ -88,7 +78,8 @@ struct jump_entry {
8878
.endif
8979
.pushsection __jump_table, "aw"
9080
_ASM_ALIGN
91-
_ASM_PTR .Lstatic_jump_\@, \target, \key
81+
.long .Lstatic_jump_\@ - ., \target - .
82+
_ASM_PTR \key - .
9283
.popsection
9384
.endm
9485

@@ -104,7 +95,8 @@ struct jump_entry {
10495
.endif
10596
.pushsection __jump_table, "aw"
10697
_ASM_ALIGN
107-
_ASM_PTR .Lstatic_jump_\@, \target, \key + 1
98+
.long .Lstatic_jump_\@ - ., \target - .
99+
_ASM_PTR \key + 1 - .
108100
.popsection
109101
.endm
110102

‎arch/x86/kernel/jump_label.c

Lines changed: 25 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -42,55 +42,40 @@ static void __ref __jump_label_transform(struct jump_entry *entry,
4242
void *(*poker)(void *, const void *, size_t),
4343
int init)
4444
{
45-
union jump_code_union code;
45+
union jump_code_union jmp;
4646
const unsigned char default_nop[] = { STATIC_KEY_INIT_NOP };
4747
const unsigned char *ideal_nop = ideal_nops[NOP_ATOMIC5];
48+
const void *expect, *code;
49+
int line;
50+
51+
jmp.jump = 0xe9;
52+
jmp.offset = jump_entry_target(entry) -
53+
(jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
4854

4955
if (early_boot_irqs_disabled)
5056
poker = text_poke_early;
5157

5258
if (type == JUMP_LABEL_JMP) {
5359
if (init) {
54-
/*
55-
* Jump label is enabled for the first time.
56-
* So we expect a default_nop...
57-
*/
58-
if (unlikely(memcmp((void *)entry->code, default_nop, 5)
59-
!= 0))
60-
bug_at((void *)entry->code, __LINE__);
60+
expect = default_nop; line = __LINE__;
6161
} else {
62-
/*
63-
* ...otherwise expect an ideal_nop. Otherwise
64-
* something went horribly wrong.
65-
*/
66-
if (unlikely(memcmp((void *)entry->code, ideal_nop, 5)
67-
!= 0))
68-
bug_at((void *)entry->code, __LINE__);
62+
expect = ideal_nop; line = __LINE__;
6963
}
7064

71-
code.jump = 0xe9;
72-
code.offset = entry->target -
73-
(entry->code + JUMP_LABEL_NOP_SIZE);
65+
code = &jmp.code;
7466
} else {
75-
/*
76-
* We are disabling this jump label. If it is not what
77-
* we think it is, then something must have gone wrong.
78-
* If this is the first initialization call, then we
79-
* are converting the default nop to the ideal nop.
80-
*/
8167
if (init) {
82-
if (unlikely(memcmp((void *)entry->code, default_nop, 5) != 0))
83-
bug_at((void *)entry->code, __LINE__);
68+
expect = default_nop; line = __LINE__;
8469
} else {
85-
code.jump = 0xe9;
86-
code.offset = entry->target -
87-
(entry->code + JUMP_LABEL_NOP_SIZE);
88-
if (unlikely(memcmp((void *)entry->code, &code, 5) != 0))
89-
bug_at((void *)entry->code, __LINE__);
70+
expect = &jmp.code; line = __LINE__;
9071
}
91-
memcpy(&code, ideal_nops[NOP_ATOMIC5], JUMP_LABEL_NOP_SIZE);
72+
73+
code = ideal_nop;
9274
}
9375

76+
if (memcmp((void *)jump_entry_code(entry), expect, JUMP_LABEL_NOP_SIZE))
77+
bug_at((void *)jump_entry_code(entry), line);
78+
9479
/*
9580
* Make text_poke_bp() a default fallback poker.
9681
*
@@ -99,11 +84,14 @@ static void __ref __jump_label_transform(struct jump_entry *entry,
9984
* always nop being the 'currently valid' instruction
10085
*
10186
*/
102-
if (poker)
103-
(*poker)((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE);
104-
else
105-
text_poke_bp((void *)entry->code, &code, JUMP_LABEL_NOP_SIZE,
106-
(void *)entry->code + JUMP_LABEL_NOP_SIZE);
87+
if (poker) {
88+
(*poker)((void *)jump_entry_code(entry), code,
89+
JUMP_LABEL_NOP_SIZE);
90+
return;
91+
}
92+
93+
text_poke_bp((void *)jump_entry_code(entry), code, JUMP_LABEL_NOP_SIZE,
94+
(void *)jump_entry_code(entry) + JUMP_LABEL_NOP_SIZE);
10795
}
10896

10997
void arch_jump_label_transform(struct jump_entry *entry,

‎arch/x86/kernel/module.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,12 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
201201
goto overflow;
202202
#endif
203203
break;
204+
case R_X86_64_PC64:
205+
if (*(u64 *)loc != 0)
206+
goto invalid_relocation;
207+
val -= (u64)loc;
208+
*(u64 *)loc = val;
209+
break;
204210
default:
205211
pr_err("%s: Unknown rela relocation: %llu\n",
206212
me->name, ELF64_R_TYPE(rel[i].r_info));

0 commit comments

Comments
 (0)