Skip to content

Commit 43aa0a1

Browse files
torvaldstorvalds
committed
Merge tag 'objtool-core-2021-10-31' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull objtool updates from Thomas Gleixner: - Improve retpoline code patching by separating it from alternatives which reduces memory footprint and allows to do better optimizations in the actual runtime patching. - Add proper retpoline support for x86/BPF - Address noinstr warnings in x86/kvm, lockdep and paravirtualization code - Add support to handle pv_opsindirect calls in the noinstr analysis - Classify symbols upfront and cache the result to avoid redundant str*cmp() invocations. - Add a CFI hash to reduce memory consumption which also reduces runtime on a allyesconfig by ~50% - Adjust XEN code to make objtool handling more robust and as a side effect to prevent text fragmentation due to placement of the hypercall page. * tag 'objtool-core-2021-10-31' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (41 commits) bpf,x86: Respect X86_FEATURE_RETPOLINE* bpf,x86: Simplify computing label offsets x86,bugs: Unconditionally allow spectre_v2=retpoline,amd x86/alternative: Add debug prints to apply_retpolines() x86/alternative: Try inline spectre_v2=retpoline,amd x86/alternative: Handle Jcc __x86_indirect_thunk_\reg x86/alternative: Implement .retpoline_sites support x86/retpoline: Create a retpoline thunk array x86/retpoline: Move the retpoline thunk declarations to nospec-branch.h x86/asm: Fixup odd GEN-for-each-reg.h usage x86/asm: Fix register order x86/retpoline: Remove unused replacement symbols objtool,x86: Replace alternatives with .retpoline_sites objtool: Shrink struct instruction objtool: Explicitly avoid self modifying code in .altinstr_replacement objtool: Classify symbols objtool: Support pv_opsindirect calls for noinstr x86/xen: Rework the xen_{cpu,irq,mmu}_opsarrays x86/xen: Mark xen_force_evtchn_callback() noinstr x86/xen: Make irq_disable() noinstr ...
2 parents 595b28f + 87c87ec commit 43aa0a1

40 files changed

+1157
-811
lines changed

‎arch/um/kernel/um_arch.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -421,6 +421,10 @@ void __init check_bugs(void)
421421
os_check_bugs();
422422
}
423423

424+
void apply_retpolines(s32 *start, s32 *end)
425+
{
426+
}
427+
424428
void apply_alternatives(struct alt_instr *start, struct alt_instr *end)
425429
{
426430
}

‎arch/x86/include/asm/GEN-for-each-reg.h

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,16 @@
1+
/* SPDX-License-Identifier: GPL-2.0 */
2+
/*
3+
* These are in machine order; things rely on that.
4+
*/
15
#ifdef CONFIG_64BIT
26
GEN(rax)
3-
GEN(rbx)
47
GEN(rcx)
58
GEN(rdx)
9+
GEN(rbx)
10+
GEN(rsp)
11+
GEN(rbp)
612
GEN(rsi)
713
GEN(rdi)
8-
GEN(rbp)
914
GEN(r8)
1015
GEN(r9)
1116
GEN(r10)
@@ -16,10 +21,11 @@ GEN(r14)
1621
GEN(r15)
1722
#else
1823
GEN(eax)
19-
GEN(ebx)
2024
GEN(ecx)
2125
GEN(edx)
26+
GEN(ebx)
27+
GEN(esp)
28+
GEN(ebp)
2229
GEN(esi)
2330
GEN(edi)
24-
GEN(ebp)
2531
#endif

‎arch/x86/include/asm/alternative.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ extern int alternatives_patched;
7575

7676
extern void alternative_instructions(void);
7777
extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end);
78+
extern void apply_retpolines(s32 *start, s32 *end);
7879

7980
struct module;
8081

‎arch/x86/include/asm/asm-prototypes.h

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,3 @@
1717
extern void cmpxchg8b_emu(void);
1818
#endif
1919

20-
#ifdef CONFIG_RETPOLINE
21-
22-
#undef GEN
23-
#define GEN(reg) \
24-
extern asmlinkage void __x86_indirect_thunk_ ## reg (void);
25-
#include <asm/GEN-for-each-reg.h>
26-
27-
#undef GEN
28-
#define GEN(reg) \
29-
extern asmlinkage void __x86_indirect_alt_call_ ## reg (void);
30-
#include <asm/GEN-for-each-reg.h>
31-
32-
#undef GEN
33-
#define GEN(reg) \
34-
extern asmlinkage void __x86_indirect_alt_jmp_ ## reg (void);
35-
#include <asm/GEN-for-each-reg.h>
36-
37-
#endif /* CONFIG_RETPOLINE */

‎arch/x86/include/asm/nospec-branch.h

Lines changed: 13 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,15 @@
55

66
#include <linux/static_key.h>
77
#include <linux/objtool.h>
8+
#include <linux/linkage.h>
89

910
#include <asm/alternative.h>
1011
#include <asm/cpufeatures.h>
1112
#include <asm/msr-index.h>
1213
#include <asm/unwind_hints.h>
1314

15+
#define RETPOLINE_THUNK_SIZE 32
16+
1417
/*
1518
* Fill the CPU return stack buffer.
1619
*
@@ -118,6 +121,16 @@
118121
".popsection\n\t"
119122

120123
#ifdef CONFIG_RETPOLINE
124+
125+
typedef u8 retpoline_thunk_t[RETPOLINE_THUNK_SIZE];
126+
127+
#define GEN(reg) \
128+
extern retpoline_thunk_t __x86_indirect_thunk_ ## reg;
129+
#include <asm/GEN-for-each-reg.h>
130+
#undef GEN
131+
132+
extern retpoline_thunk_t __x86_indirect_thunk_array[];
133+
121134
#ifdef CONFIG_X86_64
122135

123136
/*
@@ -303,63 +316,4 @@ static inline void mds_idle_clear_cpu_buffers(void)
303316

304317
#endif /* __ASSEMBLY__ */
305318

306-
/*
307-
* Below is used in the eBPF JIT compiler and emits the byte sequence
308-
* for the following assembly:
309-
*
310-
* With retpolines configured:
311-
*
312-
* callq do_rop
313-
* spec_trap:
314-
* pause
315-
* lfence
316-
* jmp spec_trap
317-
* do_rop:
318-
* mov %rcx,(%rsp) for x86_64
319-
* mov %edx,(%esp) for x86_32
320-
* retq
321-
*
322-
* Without retpolines configured:
323-
*
324-
* jmp *%rcx for x86_64
325-
* jmp *%edx for x86_32
326-
*/
327-
#ifdef CONFIG_RETPOLINE
328-
# ifdef CONFIG_X86_64
329-
# define RETPOLINE_RCX_BPF_JIT_SIZE 17
330-
# define RETPOLINE_RCX_BPF_JIT() \
331-
do { \
332-
EMIT1_off32(0xE8, 7); /* callq do_rop */ \
333-
/* spec_trap: */ \
334-
EMIT2(0xF3, 0x90); /* pause */ \
335-
EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
336-
EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
337-
/* do_rop: */ \
338-
EMIT4(0x48, 0x89, 0x0C, 0x24); /* mov %rcx,(%rsp) */ \
339-
EMIT1(0xC3); /* retq */ \
340-
} while (0)
341-
# else /* !CONFIG_X86_64 */
342-
# define RETPOLINE_EDX_BPF_JIT() \
343-
do { \
344-
EMIT1_off32(0xE8, 7); /* call do_rop */ \
345-
/* spec_trap: */ \
346-
EMIT2(0xF3, 0x90); /* pause */ \
347-
EMIT3(0x0F, 0xAE, 0xE8); /* lfence */ \
348-
EMIT2(0xEB, 0xF9); /* jmp spec_trap */ \
349-
/* do_rop: */ \
350-
EMIT3(0x89, 0x14, 0x24); /* mov %edx,(%esp) */ \
351-
EMIT1(0xC3); /* ret */ \
352-
} while (0)
353-
# endif
354-
#else /* !CONFIG_RETPOLINE */
355-
# ifdef CONFIG_X86_64
356-
# define RETPOLINE_RCX_BPF_JIT_SIZE 2
357-
# define RETPOLINE_RCX_BPF_JIT() \
358-
EMIT2(0xFF, 0xE1); /* jmp *%rcx */
359-
# else /* !CONFIG_X86_64 */
360-
# define RETPOLINE_EDX_BPF_JIT() \
361-
EMIT2(0xFF, 0xE2) /* jmp *%edx */
362-
# endif
363-
#endif
364-
365319
#endif /* _ASM_X86_NOSPEC_BRANCH_H_ */

‎arch/x86/include/asm/paravirt.h

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,11 @@ void __init paravirt_set_cap(void);
5252
/* The paravirtualized I/O functions */
5353
static inline void slow_down_io(void)
5454
{
55-
pv_ops.cpu.io_delay();
55+
PVOP_VCALL0(cpu.io_delay);
5656
#ifdef REALLY_SLOW_IO
57-
pv_ops.cpu.io_delay();
58-
pv_ops.cpu.io_delay();
59-
pv_ops.cpu.io_delay();
57+
PVOP_VCALL0(cpu.io_delay);
58+
PVOP_VCALL0(cpu.io_delay);
59+
PVOP_VCALL0(cpu.io_delay);
6060
#endif
6161
}
6262

@@ -113,12 +113,12 @@ static inline void __cpuid(unsigned int *eax, unsigned int *ebx,
113113
/*
114114
* These special macros can be used to get or set a debugging register
115115
*/
116-
static inline unsigned long paravirt_get_debugreg(int reg)
116+
static __always_inline unsigned long paravirt_get_debugreg(int reg)
117117
{
118118
return PVOP_CALL1(unsigned long, cpu.get_debugreg, reg);
119119
}
120120
#define get_debugreg(var, reg) var = paravirt_get_debugreg(reg)
121-
static inline void set_debugreg(unsigned long val, int reg)
121+
static __always_inline void set_debugreg(unsigned long val, int reg)
122122
{
123123
PVOP_VCALL2(cpu.set_debugreg, reg, val);
124124
}
@@ -133,14 +133,14 @@ static inline void write_cr0(unsigned long x)
133133
PVOP_VCALL1(cpu.write_cr0, x);
134134
}
135135

136-
static inline unsigned long read_cr2(void)
136+
static __always_inline unsigned long read_cr2(void)
137137
{
138138
return PVOP_ALT_CALLEE0(unsigned long, mmu.read_cr2,
139139
"mov %%cr2, %%rax;",
140140
ALT_NOT(X86_FEATURE_XENPV));
141141
}
142142

143-
static inline void write_cr2(unsigned long x)
143+
static __always_inline void write_cr2(unsigned long x)
144144
{
145145
PVOP_VCALL1(mmu.write_cr2, x);
146146
}
@@ -653,10 +653,10 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu);
653653
* functions.
654654
*/
655655
#define PV_THUNK_NAME(func) "__raw_callee_save_" #func
656-
#define PV_CALLEE_SAVE_REGS_THUNK(func) \
656+
#define __PV_CALLEE_SAVE_REGS_THUNK(func, section) \
657657
extern typeof(func) __raw_callee_save_##func; \
658658
\
659-
asm(".pushsection .text;" \
659+
asm(".pushsection " section ", \"ax\";" \
660660
".globl " PV_THUNK_NAME(func) ";" \
661661
".type " PV_THUNK_NAME(func) ", @function;" \
662662
PV_THUNK_NAME(func) ":" \
@@ -669,6 +669,9 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu);
669669
".size " PV_THUNK_NAME(func) ", .-" PV_THUNK_NAME(func) ";" \
670670
".popsection")
671671

672+
#define PV_CALLEE_SAVE_REGS_THUNK(func) \
673+
__PV_CALLEE_SAVE_REGS_THUNK(func, ".text")
674+
672675
/* Get a reference to a callee-save function */
673676
#define PV_CALLEE_SAVE(func) \
674677
((struct paravirt_callee_save) { __raw_callee_save_##func })
@@ -678,23 +681,23 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu);
678681
((struct paravirt_callee_save) { func })
679682

680683
#ifdef CONFIG_PARAVIRT_XXL
681-
static inline notrace unsigned long arch_local_save_flags(void)
684+
static __always_inline unsigned long arch_local_save_flags(void)
682685
{
683686
return PVOP_ALT_CALLEE0(unsigned long, irq.save_fl, "pushf; pop %%rax;",
684687
ALT_NOT(X86_FEATURE_XENPV));
685688
}
686689

687-
static inline notrace void arch_local_irq_disable(void)
690+
static __always_inline void arch_local_irq_disable(void)
688691
{
689692
PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT(X86_FEATURE_XENPV));
690693
}
691694

692-
static inline notrace void arch_local_irq_enable(void)
695+
static __always_inline void arch_local_irq_enable(void)
693696
{
694697
PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT(X86_FEATURE_XENPV));
695698
}
696699

697-
static inline notrace unsigned long arch_local_irq_save(void)
700+
static __always_inline unsigned long arch_local_irq_save(void)
698701
{
699702
unsigned long f;
700703

‎arch/x86/include/asm/ptrace.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ static inline bool any_64bit_mode(struct pt_regs *regs)
181181
#define current_user_stack_pointer() current_pt_regs()->sp
182182
#define compat_user_stack_pointer() current_pt_regs()->sp
183183

184-
static inline bool ip_within_syscall_gap(struct pt_regs *regs)
184+
static __always_inline bool ip_within_syscall_gap(struct pt_regs *regs)
185185
{
186186
bool ret = (regs->ip >= (unsigned long)entry_SYSCALL_64 &&
187187
regs->ip < (unsigned long)entry_SYSCALL_64_safe_stack);

‎arch/x86/include/asm/xen/hypercall.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -308,13 +308,13 @@ HYPERVISOR_platform_op(struct xen_platform_op *op)
308308
return _hypercall1(int, platform_op, op);
309309
}
310310

311-
static inline int
311+
static __always_inline int
312312
HYPERVISOR_set_debugreg(int reg, unsigned long value)
313313
{
314314
return _hypercall2(int, set_debugreg, reg, value);
315315
}
316316

317-
static inline unsigned long
317+
static __always_inline unsigned long
318318
HYPERVISOR_get_debugreg(int reg)
319319
{
320320
return _hypercall1(unsigned long, get_debugreg, reg);
@@ -358,7 +358,7 @@ HYPERVISOR_event_channel_op(int cmd, void *arg)
358358
return _hypercall2(int, event_channel_op, cmd, arg);
359359
}
360360

361-
static inline int
361+
static __always_inline int
362362
HYPERVISOR_xen_version(int cmd, void *arg)
363363
{
364364
return _hypercall2(int, xen_version, cmd, arg);

0 commit comments

Comments
 (0)