Skip to content

Commit 5d1dd96

Browse files
bp3tk0vbp3tk0v
committed
x86/alternatives: Add alt_instr.flags
Add a struct alt_instr.flags field which will contain different flags controlling alternatives patching behavior. The initial idea was to be able to specify it as a separate macro parameter but that would mean touching all possible invocations of the alternatives macros and thus a lot of churn. What is more, as PeterZ suggested, being able to say ALT_NOT(feature) is very readable and explains exactly what is meant. So make the feature field a u32 where the patching flags are the upper u16 part of the dword quantity while the lower u16 word is the feature. The highest feature number currently is 0x26a (i.e., word 19) so there is plenty of space. If that becomes insufficient, the field can be extended to u64 which will then make struct alt_instr of the nice size of 16 bytes (14 bytes currently). There should be no functional changes resulting from this. Suggested-by: Peter Zijlstra (Intel) <[email protected]> Signed-off-by: Borislav Petkov (AMD) <[email protected]> Reviewed-by: Ingo Molnar <[email protected]> Link: https://lore.kernel.org/r/[email protected]
1 parent 1b929c0 commit 5d1dd96

File tree

3 files changed

+85
-67
lines changed

3 files changed

+85
-67
lines changed

‎arch/x86/include/asm/alternative.h

Lines changed: 76 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@
66
#include <linux/stringify.h>
77
#include <asm/asm.h>
88

9-
#define ALTINSTR_FLAG_INV (1 << 15)
10-
#define ALT_NOT(feat) ((feat) | ALTINSTR_FLAG_INV)
9+
#define ALT_FLAGS_SHIFT 16
10+
11+
#define ALT_FLAG_NOT BIT(0)
12+
#define ALT_NOT(feature) ((ALT_FLAG_NOT << ALT_FLAGS_SHIFT) | (feature))
1113

1214
#ifndef __ASSEMBLY__
1315

@@ -59,10 +61,27 @@
5961
".long 999b - .\n\t" \
6062
".popsection\n\t"
6163

64+
/*
65+
* The patching flags are part of the upper bits of the @ft_flags parameter when
66+
* specifying them. The split is currently like this:
67+
*
68+
* [31... flags ...16][15... CPUID feature bit ...0]
69+
*
70+
* but since this is all hidden in the macros argument being split, those fields can be
71+
* extended in the future to fit in a u64 or however the need arises.
72+
*/
6273
struct alt_instr {
6374
s32 instr_offset; /* original instruction */
6475
s32 repl_offset; /* offset to replacement instruction */
65-
u16 cpuid; /* cpuid bit set for replacement */
76+
77+
union {
78+
struct {
79+
u32 cpuid: 16; /* CPUID bit set for replacement */
80+
u32 flags: 16; /* patching control flags */
81+
};
82+
u32 ft_flags;
83+
};
84+
6685
u8 instrlen; /* length of original instruction */
6786
u8 replacementlen; /* length of new instruction */
6887
} __packed;
@@ -182,10 +201,10 @@ static inline int alternatives_text_reserved(void *start, void *end)
182201
" - (" alt_slen ")), 0x90\n" \
183202
alt_end_marker ":\n"
184203

185-
#define ALTINSTR_ENTRY(feature, num) \
204+
#define ALTINSTR_ENTRY(ft_flags, num) \
186205
" .long 661b - .\n" /* label */ \
187206
" .long " b_replacement(num)"f - .\n" /* new instruction */ \
188-
" .word " __stringify(feature) "\n" /* feature bit */ \
207+
" .4byte " __stringify(ft_flags) "\n" /* feature + flags */ \
189208
" .byte " alt_total_slen "\n" /* source len */ \
190209
" .byte " alt_rlen(num) "\n" /* replacement len */
191210

@@ -194,42 +213,43 @@ static inline int alternatives_text_reserved(void *start, void *end)
194213
b_replacement(num)":\n\t" newinstr "\n" e_replacement(num) ":\n"
195214

196215
/* alternative assembly primitive: */
197-
#define ALTERNATIVE(oldinstr, newinstr, feature) \
216+
#define ALTERNATIVE(oldinstr, newinstr, ft_flags) \
198217
OLDINSTR(oldinstr, 1) \
199218
".pushsection .altinstructions,\"a\"\n" \
200-
ALTINSTR_ENTRY(feature, 1) \
219+
ALTINSTR_ENTRY(ft_flags, 1) \
201220
".popsection\n" \
202221
".pushsection .altinstr_replacement, \"ax\"\n" \
203222
ALTINSTR_REPLACEMENT(newinstr, 1) \
204223
".popsection\n"
205224

206-
#define ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2)\
225+
#define ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \
207226
OLDINSTR_2(oldinstr, 1, 2) \
208227
".pushsection .altinstructions,\"a\"\n" \
209-
ALTINSTR_ENTRY(feature1, 1) \
210-
ALTINSTR_ENTRY(feature2, 2) \
228+
ALTINSTR_ENTRY(ft_flags1, 1) \
229+
ALTINSTR_ENTRY(ft_flags2, 2) \
211230
".popsection\n" \
212231
".pushsection .altinstr_replacement, \"ax\"\n" \
213232
ALTINSTR_REPLACEMENT(newinstr1, 1) \
214233
ALTINSTR_REPLACEMENT(newinstr2, 2) \
215234
".popsection\n"
216235

217236
/* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
218-
#define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \
237+
#define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \
219238
ALTERNATIVE_2(oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \
220-
newinstr_yes, feature)
221-
222-
#define ALTERNATIVE_3(oldinsn, newinsn1, feat1, newinsn2, feat2, newinsn3, feat3) \
223-
OLDINSTR_3(oldinsn, 1, 2, 3) \
224-
".pushsection .altinstructions,\"a\"\n" \
225-
ALTINSTR_ENTRY(feat1, 1) \
226-
ALTINSTR_ENTRY(feat2, 2) \
227-
ALTINSTR_ENTRY(feat3, 3) \
228-
".popsection\n" \
229-
".pushsection .altinstr_replacement, \"ax\"\n" \
230-
ALTINSTR_REPLACEMENT(newinsn1, 1) \
231-
ALTINSTR_REPLACEMENT(newinsn2, 2) \
232-
ALTINSTR_REPLACEMENT(newinsn3, 3) \
239+
newinstr_yes, ft_flags)
240+
241+
#define ALTERNATIVE_3(oldinsn, newinsn1, ft_flags1, newinsn2, ft_flags2, \
242+
newinsn3, ft_flags3) \
243+
OLDINSTR_3(oldinsn, 1, 2, 3) \
244+
".pushsection .altinstructions,\"a\"\n" \
245+
ALTINSTR_ENTRY(ft_flags1, 1) \
246+
ALTINSTR_ENTRY(ft_flags2, 2) \
247+
ALTINSTR_ENTRY(ft_flags3, 3) \
248+
".popsection\n" \
249+
".pushsection .altinstr_replacement, \"ax\"\n" \
250+
ALTINSTR_REPLACEMENT(newinsn1, 1) \
251+
ALTINSTR_REPLACEMENT(newinsn2, 2) \
252+
ALTINSTR_REPLACEMENT(newinsn3, 3) \
233253
".popsection\n"
234254

235255
/*
@@ -244,14 +264,14 @@ static inline int alternatives_text_reserved(void *start, void *end)
244264
* For non barrier like inlines please define new variants
245265
* without volatile and memory clobber.
246266
*/
247-
#define alternative(oldinstr, newinstr, feature) \
248-
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) : : : "memory")
267+
#define alternative(oldinstr, newinstr, ft_flags) \
268+
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) : : : "memory")
249269

250-
#define alternative_2(oldinstr, newinstr1, feature1, newinstr2, feature2) \
251-
asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, newinstr2, feature2) ::: "memory")
270+
#define alternative_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) \
271+
asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2) ::: "memory")
252272

253-
#define alternative_ternary(oldinstr, feature, newinstr_yes, newinstr_no) \
254-
asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) ::: "memory")
273+
#define alternative_ternary(oldinstr, ft_flags, newinstr_yes, newinstr_no) \
274+
asm_inline volatile(ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) ::: "memory")
255275

256276
/*
257277
* Alternative inline assembly with input.
@@ -261,8 +281,8 @@ static inline int alternatives_text_reserved(void *start, void *end)
261281
* Argument numbers start with 1.
262282
* Leaving an unused argument 0 to keep API compatibility.
263283
*/
264-
#define alternative_input(oldinstr, newinstr, feature, input...) \
265-
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
284+
#define alternative_input(oldinstr, newinstr, ft_flags, input...) \
285+
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \
266286
: : "i" (0), ## input)
267287

268288
/*
@@ -273,20 +293,20 @@ static inline int alternatives_text_reserved(void *start, void *end)
273293
* Otherwise, if CPU has feature1, newinstr1 is used.
274294
* Otherwise, oldinstr is used.
275295
*/
276-
#define alternative_input_2(oldinstr, newinstr1, feature1, newinstr2, \
277-
feature2, input...) \
278-
asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, feature1, \
279-
newinstr2, feature2) \
296+
#define alternative_input_2(oldinstr, newinstr1, ft_flags1, newinstr2, \
297+
ft_flags2, input...) \
298+
asm_inline volatile(ALTERNATIVE_2(oldinstr, newinstr1, ft_flags1, \
299+
newinstr2, ft_flags2) \
280300
: : "i" (0), ## input)
281301

282302
/* Like alternative_input, but with a single output argument */
283-
#define alternative_io(oldinstr, newinstr, feature, output, input...) \
284-
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, feature) \
303+
#define alternative_io(oldinstr, newinstr, ft_flags, output, input...) \
304+
asm_inline volatile (ALTERNATIVE(oldinstr, newinstr, ft_flags) \
285305
: output : "i" (0), ## input)
286306

287307
/* Like alternative_io, but for replacing a direct call with another one. */
288-
#define alternative_call(oldfunc, newfunc, feature, output, input...) \
289-
asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", feature) \
308+
#define alternative_call(oldfunc, newfunc, ft_flags, output, input...) \
309+
asm_inline volatile (ALTERNATIVE("call %P[old]", "call %P[new]", ft_flags) \
290310
: output : [old] "i" (oldfunc), [new] "i" (newfunc), ## input)
291311

292312
/*
@@ -295,10 +315,10 @@ static inline int alternatives_text_reserved(void *start, void *end)
295315
* Otherwise, if CPU has feature1, function1 is used.
296316
* Otherwise, old function is used.
297317
*/
298-
#define alternative_call_2(oldfunc, newfunc1, feature1, newfunc2, feature2, \
318+
#define alternative_call_2(oldfunc, newfunc1, ft_flags1, newfunc2, ft_flags2, \
299319
output, input...) \
300-
asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", feature1,\
301-
"call %P[new2]", feature2) \
320+
asm_inline volatile (ALTERNATIVE_2("call %P[old]", "call %P[new1]", ft_flags1,\
321+
"call %P[new2]", ft_flags2) \
302322
: output, ASM_CALL_CONSTRAINT \
303323
: [old] "i" (oldfunc), [new1] "i" (newfunc1), \
304324
[new2] "i" (newfunc2), ## input)
@@ -347,10 +367,10 @@ static inline int alternatives_text_reserved(void *start, void *end)
347367
* enough information for the alternatives patching code to patch an
348368
* instruction. See apply_alternatives().
349369
*/
350-
.macro altinstruction_entry orig alt feature orig_len alt_len
370+
.macro altinstr_entry orig alt ft_flags orig_len alt_len
351371
.long \orig - .
352372
.long \alt - .
353-
.word \feature
373+
.4byte \ft_flags
354374
.byte \orig_len
355375
.byte \alt_len
356376
.endm
@@ -361,15 +381,15 @@ static inline int alternatives_text_reserved(void *start, void *end)
361381
* @newinstr. ".skip" directive takes care of proper instruction padding
362382
* in case @newinstr is longer than @oldinstr.
363383
*/
364-
.macro ALTERNATIVE oldinstr, newinstr, feature
384+
.macro ALTERNATIVE oldinstr, newinstr, ft_flags
365385
140:
366386
\oldinstr
367387
141:
368388
.skip -(((144f-143f)-(141b-140b)) > 0) * ((144f-143f)-(141b-140b)),0x90
369389
142:
370390

371391
.pushsection .altinstructions,"a"
372-
altinstruction_entry 140b,143f,\feature,142b-140b,144f-143f
392+
altinstr_entry 140b,143f,\ft_flags,142b-140b,144f-143f
373393
.popsection
374394

375395
.pushsection .altinstr_replacement,"ax"
@@ -399,7 +419,7 @@ static inline int alternatives_text_reserved(void *start, void *end)
399419
* has @feature1, it replaces @oldinstr with @newinstr1. If CPU has
400420
* @feature2, it replaces @oldinstr with @feature2.
401421
*/
402-
.macro ALTERNATIVE_2 oldinstr, newinstr1, feature1, newinstr2, feature2
422+
.macro ALTERNATIVE_2 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2
403423
140:
404424
\oldinstr
405425
141:
@@ -408,8 +428,8 @@ static inline int alternatives_text_reserved(void *start, void *end)
408428
142:
409429

410430
.pushsection .altinstructions,"a"
411-
altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f
412-
altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f
431+
altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f
432+
altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f
413433
.popsection
414434

415435
.pushsection .altinstr_replacement,"ax"
@@ -421,7 +441,7 @@ static inline int alternatives_text_reserved(void *start, void *end)
421441
.popsection
422442
.endm
423443

424-
.macro ALTERNATIVE_3 oldinstr, newinstr1, feature1, newinstr2, feature2, newinstr3, feature3
444+
.macro ALTERNATIVE_3 oldinstr, newinstr1, ft_flags1, newinstr2, ft_flags2, newinstr3, ft_flags3
425445
140:
426446
\oldinstr
427447
141:
@@ -430,9 +450,9 @@ static inline int alternatives_text_reserved(void *start, void *end)
430450
142:
431451

432452
.pushsection .altinstructions,"a"
433-
altinstruction_entry 140b,143f,\feature1,142b-140b,144f-143f
434-
altinstruction_entry 140b,144f,\feature2,142b-140b,145f-144f
435-
altinstruction_entry 140b,145f,\feature3,142b-140b,146f-145f
453+
altinstr_entry 140b,143f,\ft_flags1,142b-140b,144f-143f
454+
altinstr_entry 140b,144f,\ft_flags2,142b-140b,145f-144f
455+
altinstr_entry 140b,145f,\ft_flags3,142b-140b,146f-145f
436456
.popsection
437457

438458
.pushsection .altinstr_replacement,"ax"
@@ -447,9 +467,9 @@ static inline int alternatives_text_reserved(void *start, void *end)
447467
.endm
448468

449469
/* If @feature is set, patch in @newinstr_yes, otherwise @newinstr_no. */
450-
#define ALTERNATIVE_TERNARY(oldinstr, feature, newinstr_yes, newinstr_no) \
470+
#define ALTERNATIVE_TERNARY(oldinstr, ft_flags, newinstr_yes, newinstr_no) \
451471
ALTERNATIVE_2 oldinstr, newinstr_no, X86_FEATURE_ALWAYS, \
452-
newinstr_yes, feature
472+
newinstr_yes, ft_flags
453473

454474
#endif /* __ASSEMBLY__ */
455475

‎arch/x86/kernel/alternative.c

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -282,27 +282,25 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start,
282282
*/
283283
for (a = start; a < end; a++) {
284284
int insn_buff_sz = 0;
285-
/* Mask away "NOT" flag bit for feature to test. */
286-
u16 feature = a->cpuid & ~ALTINSTR_FLAG_INV;
287285

288286
instr = (u8 *)&a->instr_offset + a->instr_offset;
289287
replacement = (u8 *)&a->repl_offset + a->repl_offset;
290288
BUG_ON(a->instrlen > sizeof(insn_buff));
291-
BUG_ON(feature >= (NCAPINTS + NBUGINTS) * 32);
289+
BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);
292290

293291
/*
294292
* Patch if either:
295293
* - feature is present
296-
* - feature not present but ALTINSTR_FLAG_INV is set to mean,
294+
* - feature not present but ALT_FLAG_NOT is set to mean,
297295
* patch if feature is *NOT* present.
298296
*/
299-
if (!boot_cpu_has(feature) == !(a->cpuid & ALTINSTR_FLAG_INV))
297+
if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT))
300298
goto next;
301299

302300
DPRINTK("feat: %s%d*32+%d, old: (%pS (%px) len: %d), repl: (%px, len: %d)",
303-
(a->cpuid & ALTINSTR_FLAG_INV) ? "!" : "",
304-
feature >> 5,
305-
feature & 0x1f,
301+
(a->flags & ALT_FLAG_NOT) ? "!" : "",
302+
a->cpuid >> 5,
303+
a->cpuid & 0x1f,
306304
instr, instr, a->instrlen,
307305
replacement, a->replacementlen);
308306

‎tools/objtool/arch/x86/include/arch/special.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,11 +11,11 @@
1111
#define JUMP_NEW_OFFSET 4
1212
#define JUMP_KEY_OFFSET 8
1313

14-
#define ALT_ENTRY_SIZE 12
14+
#define ALT_ENTRY_SIZE 14
1515
#define ALT_ORIG_OFFSET 0
1616
#define ALT_NEW_OFFSET 4
1717
#define ALT_FEATURE_OFFSET 8
18-
#define ALT_ORIG_LEN_OFFSET 10
19-
#define ALT_NEW_LEN_OFFSET 11
18+
#define ALT_ORIG_LEN_OFFSET 12
19+
#define ALT_NEW_LEN_OFFSET 13
2020

2121
#endif /* _X86_ARCH_SPECIAL_H */

0 commit comments

Comments
 (0)