Skip to content

Commit 6ea98b4

Browse files
torvaldstorvalds
committed
Merge branch 'x86-alternatives-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 alternative instruction updates from Ingo Molnar: "Small RDTSCP opimization, enabled by the newly added ALTERNATIVE_3(), and other small improvements" * 'x86-alternatives-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/TSC: Use RDTSCP x86/alternatives: Add an ALTERNATIVE_3() macro x86/alternatives: Print containing function x86/alternatives: Add macro comments
2 parents 45802da + 093ae8f commit 6ea98b4

File tree

3 files changed

+48
-11
lines changed

3 files changed

+48
-11
lines changed

‎arch/x86/include/asm/alternative.h

Lines changed: 32 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,13 +94,12 @@ static inline int alternatives_text_reserved(void *start, void *end)
9494
#define alt_total_slen alt_end_marker"b-661b"
9595
#define alt_rlen(num) e_replacement(num)"f-"b_replacement(num)"f"
9696

97-
#define __OLDINSTR(oldinstr, num) \
97+
#define OLDINSTR(oldinstr, num) \
98+
"# ALT: oldnstr\n" \
9899
"661:\n\t" oldinstr "\n662:\n" \
100+
"# ALT: padding\n" \
99101
".skip -(((" alt_rlen(num) ")-(" alt_slen ")) > 0) * " \
100-
"((" alt_rlen(num) ")-(" alt_slen ")),0x90\n"
101-
102-
#define OLDINSTR(oldinstr, num) \
103-
__OLDINSTR(oldinstr, num) \
102+
"((" alt_rlen(num) ")-(" alt_slen ")),0x90\n" \
104103
alt_end_marker ":\n"
105104

106105
/*
@@ -116,11 +115,23 @@ static inline int alternatives_text_reserved(void *start, void *end)
116115
* additionally longer than the first replacement alternative.
117116
*/
118117
#define OLDINSTR_2(oldinstr, num1, num2) \
118+
"# ALT: oldinstr2\n" \
119119
"661:\n\t" oldinstr "\n662:\n" \
120+
"# ALT: padding2\n" \
120121
".skip -((" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")) > 0) * " \
121122
"(" alt_max_short(alt_rlen(num1), alt_rlen(num2)) " - (" alt_slen ")), 0x90\n" \
122123
alt_end_marker ":\n"
123124

125+
#define OLDINSTR_3(oldinsn, n1, n2, n3) \
126+
"# ALT: oldinstr3\n" \
127+
"661:\n\t" oldinsn "\n662:\n" \
128+
"# ALT: padding3\n" \
129+
".skip -((" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \
130+
" - (" alt_slen ")) > 0) * " \
131+
"(" alt_max_short(alt_max_short(alt_rlen(n1), alt_rlen(n2)), alt_rlen(n3)) \
132+
" - (" alt_slen ")), 0x90\n" \
133+
alt_end_marker ":\n"
134+
124135
#define ALTINSTR_ENTRY(feature, num) \
125136
" .long 661b - .\n" /* label */ \
126137
" .long " b_replacement(num)"f - .\n" /* new instruction */ \
@@ -129,8 +140,9 @@ static inline int alternatives_text_reserved(void *start, void *end)
129140
" .byte " alt_rlen(num) "\n" /* replacement len */ \
130141
" .byte " alt_pad_len "\n" /* pad len */
131142

132-
#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \
133-
b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n\t"
143+
#define ALTINSTR_REPLACEMENT(newinstr, feature, num) /* replacement */ \
144+
"# ALT: replacement " #num "\n" \
145+
b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"
134146

135147
/* alternative assembly primitive: */
136148
#define ALTERNATIVE(oldinstr, newinstr, feature) \
@@ -153,6 +165,19 @@ static inline int alternatives_text_reserved(void *start, void *end)
153165
ALTINSTR_REPLACEMENT(newinstr2, feature2, 2) \
154166
".popsection\n"
155167

168+
#define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \
169+
OLDINSTR_3(oldinsn, 1, 2, 3) \
170+
".pushsection .altinstructions,\"a\"\n" \
171+
ALTINSTR_ENTRY(feat1, 1) \
172+
ALTINSTR_ENTRY(feat2, 2) \
173+
ALTINSTR_ENTRY(feat3, 3) \
174+
".popsection\n" \
175+
".pushsection .altinstr_replacement, \"ax\"\n" \
176+
ALTINSTR_REPLACEMENT(newinsn1, feat1, 1) \
177+
ALTINSTR_REPLACEMENT(newinsn2, feat2, 2) \
178+
ALTINSTR_REPLACEMENT(newinsn3, feat3, 3) \
179+
".popsection\n"
180+
156181
/*
157182
* Alternative instructions for different CPU types or capabilities.
158183
*

‎arch/x86/include/asm/msr.h

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,8 @@ static __always_inline unsigned long long rdtsc(void)
217217
*/
218218
static __always_inline unsigned long long rdtsc_ordered(void)
219219
{
220+
DECLARE_ARGS(val, low, high);
221+
220222
/*
221223
* The RDTSC instruction is not ordered relative to memory
222224
* access. The Intel SDM and the AMD APM are both vague on this
@@ -227,9 +229,19 @@ static __always_inline unsigned long long rdtsc_ordered(void)
227229
* ordering guarantees as reading from a global memory location
228230
* that some other imaginary CPU is updating continuously with a
229231
* time stamp.
232+
*
233+
* Thus, use the preferred barrier on the respective CPU, aiming for
234+
* RDTSCP as the default.
230235
*/
231-
barrier_nospec();
232-
return rdtsc();
236+
asm volatile(ALTERNATIVE_3("rdtsc",
237+
"mfence; rdtsc", X86_FEATURE_MFENCE_RDTSC,
238+
"lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
239+
"rdtscp", X86_FEATURE_RDTSCP)
240+
: EAX_EDX_RET(val, low, high)
241+
/* RDTSCP clobbers ECX with MSR_TSC_AUX. */
242+
:: "ecx");
243+
244+
return EAX_EDX_VAL(val, low, high);
233245
}
234246

235247
static inline unsigned long long native_read_pmc(int counter)

‎arch/x86/kernel/alternative.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,10 +394,10 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
394394
continue;
395395
}
396396

397-
DPRINTK("feat: %d*32+%d, old: (%px len: %d), repl: (%px, len: %d), pad: %d",
397+
DPRINTK("feat: %d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d), pad: %d",
398398
a->cpuid >> 5,
399399
a->cpuid & 0x1f,
400-
instr, a->instrlen,
400+
instr, instr, a->instrlen,
401401
replacement, a->replacementlen, a->padlen);
402402

403403
DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);

0 commit comments

Comments
 (0)