Skip to content

Commit fafe5e7

Browse files
jgross1suryasaimadhu
jgross1
authored and
suryasaimadhu
committed
x86/paravirt: Switch functions with custom code to ALTERNATIVE
Instead of using paravirt patching for custom code sequences use ALTERNATIVE for the functions with custom code replacements. Instead of patching an ud2 instruction for unpopulated vector entries into the caller site, use a simple function just calling BUG() as a replacement. Simplify the register defines for assembler paravirt calling, as there isn't much usage left. Signed-off-by: Juergen Gross <[email protected]> Signed-off-by: Borislav Petkov <[email protected]> Acked-by: Peter Zijlstra (Intel) <[email protected]> Link: https://lkml.kernel.org/r/[email protected]
1 parent 00aa319 commit fafe5e7

File tree

6 files changed

+58
-157
lines changed

6 files changed

+58
-157
lines changed

‎arch/x86/entry/entry_64.S

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -305,7 +305,7 @@ SYM_CODE_END(ret_from_fork)
305305
.macro DEBUG_ENTRY_ASSERT_IRQS_OFF
306306
#ifdef CONFIG_DEBUG_ENTRY
307307
pushq %rax
308-
SAVE_FLAGS(CLBR_RAX)
308+
SAVE_FLAGS
309309
testl $X86_EFLAGS_IF, %eax
310310
jz .Lokay_\@
311311
ud2

‎arch/x86/include/asm/irqflags.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ static __always_inline unsigned long arch_local_irq_save(void)
111111

112112
#ifdef CONFIG_X86_64
113113
#ifdef CONFIG_DEBUG_ENTRY
114-
#define SAVE_FLAGS(x) pushfq; popq %rax
114+
#define SAVE_FLAGS pushfq; popq %rax
115115
#endif
116116

117117
#define INTERRUPT_RETURN jmp native_iret

‎arch/x86/include/asm/paravirt.h

Lines changed: 50 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,9 @@ static inline void write_cr0(unsigned long x)
135135

136136
static inline unsigned long read_cr2(void)
137137
{
138-
return PVOP_CALLEE0(unsigned long, mmu.read_cr2);
138+
return PVOP_ALT_CALLEE0(unsigned long, mmu.read_cr2,
139+
"mov %%cr2, %%rax;",
140+
ALT_NOT(X86_FEATURE_XENPV));
139141
}
140142

141143
static inline void write_cr2(unsigned long x)
@@ -145,12 +147,14 @@ static inline void write_cr2(unsigned long x)
145147

146148
static inline unsigned long __read_cr3(void)
147149
{
148-
return PVOP_CALL0(unsigned long, mmu.read_cr3);
150+
return PVOP_ALT_CALL0(unsigned long, mmu.read_cr3,
151+
"mov %%cr3, %%rax;", ALT_NOT(X86_FEATURE_XENPV));
149152
}
150153

151154
static inline void write_cr3(unsigned long x)
152155
{
153-
PVOP_VCALL1(mmu.write_cr3, x);
156+
PVOP_ALT_VCALL1(mmu.write_cr3, x,
157+
"mov %%rdi, %%cr3", ALT_NOT(X86_FEATURE_XENPV));
154158
}
155159

156160
static inline void __write_cr4(unsigned long x)
@@ -170,7 +174,7 @@ static inline void halt(void)
170174

171175
static inline void wbinvd(void)
172176
{
173-
PVOP_VCALL0(cpu.wbinvd);
177+
PVOP_ALT_VCALL0(cpu.wbinvd, "wbinvd", ALT_NOT(X86_FEATURE_XENPV));
174178
}
175179

176180
static inline u64 paravirt_read_msr(unsigned msr)
@@ -384,22 +388,28 @@ static inline void paravirt_release_p4d(unsigned long pfn)
384388

385389
static inline pte_t __pte(pteval_t val)
386390
{
387-
return (pte_t) { PVOP_CALLEE1(pteval_t, mmu.make_pte, val) };
391+
return (pte_t) { PVOP_ALT_CALLEE1(pteval_t, mmu.make_pte, val,
392+
"mov %%rdi, %%rax",
393+
ALT_NOT(X86_FEATURE_XENPV)) };
388394
}
389395

390396
static inline pteval_t pte_val(pte_t pte)
391397
{
392-
return PVOP_CALLEE1(pteval_t, mmu.pte_val, pte.pte);
398+
return PVOP_ALT_CALLEE1(pteval_t, mmu.pte_val, pte.pte,
399+
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
393400
}
394401

395402
static inline pgd_t __pgd(pgdval_t val)
396403
{
397-
return (pgd_t) { PVOP_CALLEE1(pgdval_t, mmu.make_pgd, val) };
404+
return (pgd_t) { PVOP_ALT_CALLEE1(pgdval_t, mmu.make_pgd, val,
405+
"mov %%rdi, %%rax",
406+
ALT_NOT(X86_FEATURE_XENPV)) };
398407
}
399408

400409
static inline pgdval_t pgd_val(pgd_t pgd)
401410
{
402-
return PVOP_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd);
411+
return PVOP_ALT_CALLEE1(pgdval_t, mmu.pgd_val, pgd.pgd,
412+
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
403413
}
404414

405415
#define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION
@@ -432,12 +442,15 @@ static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
432442

433443
static inline pmd_t __pmd(pmdval_t val)
434444
{
435-
return (pmd_t) { PVOP_CALLEE1(pmdval_t, mmu.make_pmd, val) };
445+
return (pmd_t) { PVOP_ALT_CALLEE1(pmdval_t, mmu.make_pmd, val,
446+
"mov %%rdi, %%rax",
447+
ALT_NOT(X86_FEATURE_XENPV)) };
436448
}
437449

438450
static inline pmdval_t pmd_val(pmd_t pmd)
439451
{
440-
return PVOP_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd);
452+
return PVOP_ALT_CALLEE1(pmdval_t, mmu.pmd_val, pmd.pmd,
453+
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
441454
}
442455

443456
static inline void set_pud(pud_t *pudp, pud_t pud)
@@ -449,14 +462,16 @@ static inline pud_t __pud(pudval_t val)
449462
{
450463
pudval_t ret;
451464

452-
ret = PVOP_CALLEE1(pudval_t, mmu.make_pud, val);
465+
ret = PVOP_ALT_CALLEE1(pudval_t, mmu.make_pud, val,
466+
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
453467

454468
return (pud_t) { ret };
455469
}
456470

457471
static inline pudval_t pud_val(pud_t pud)
458472
{
459-
return PVOP_CALLEE1(pudval_t, mmu.pud_val, pud.pud);
473+
return PVOP_ALT_CALLEE1(pudval_t, mmu.pud_val, pud.pud,
474+
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
460475
}
461476

462477
static inline void pud_clear(pud_t *pudp)
@@ -475,14 +490,17 @@ static inline void set_p4d(p4d_t *p4dp, p4d_t p4d)
475490

476491
static inline p4d_t __p4d(p4dval_t val)
477492
{
478-
p4dval_t ret = PVOP_CALLEE1(p4dval_t, mmu.make_p4d, val);
493+
p4dval_t ret = PVOP_ALT_CALLEE1(p4dval_t, mmu.make_p4d, val,
494+
"mov %%rdi, %%rax",
495+
ALT_NOT(X86_FEATURE_XENPV));
479496

480497
return (p4d_t) { ret };
481498
}
482499

483500
static inline p4dval_t p4d_val(p4d_t p4d)
484501
{
485-
return PVOP_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d);
502+
return PVOP_ALT_CALLEE1(p4dval_t, mmu.p4d_val, p4d.p4d,
503+
"mov %%rdi, %%rax", ALT_NOT(X86_FEATURE_XENPV));
486504
}
487505

488506
static inline void __set_pgd(pgd_t *pgdp, pgd_t pgd)
@@ -569,7 +587,9 @@ static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock,
569587

570588
static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock)
571589
{
572-
PVOP_VCALLEE1(lock.queued_spin_unlock, lock);
590+
PVOP_ALT_VCALLEE1(lock.queued_spin_unlock, lock,
591+
"movb $0, (%%" _ASM_ARG1 ");",
592+
ALT_NOT(X86_FEATURE_PVUNLOCK));
573593
}
574594

575595
static __always_inline void pv_wait(u8 *ptr, u8 val)
@@ -584,7 +604,9 @@ static __always_inline void pv_kick(int cpu)
584604

585605
static __always_inline bool pv_vcpu_is_preempted(long cpu)
586606
{
587-
return PVOP_CALLEE1(bool, lock.vcpu_is_preempted, cpu);
607+
return PVOP_ALT_CALLEE1(bool, lock.vcpu_is_preempted, cpu,
608+
"xor %%" _ASM_AX ", %%" _ASM_AX ";",
609+
ALT_NOT(X86_FEATURE_VCPUPREEMPT));
588610
}
589611

590612
void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock);
@@ -658,17 +680,18 @@ bool __raw_callee_save___native_vcpu_is_preempted(long cpu);
658680
#ifdef CONFIG_PARAVIRT_XXL
659681
static inline notrace unsigned long arch_local_save_flags(void)
660682
{
661-
return PVOP_CALLEE0(unsigned long, irq.save_fl);
683+
return PVOP_ALT_CALLEE0(unsigned long, irq.save_fl, "pushf; pop %%rax;",
684+
ALT_NOT(X86_FEATURE_XENPV));
662685
}
663686

664687
static inline notrace void arch_local_irq_disable(void)
665688
{
666-
PVOP_VCALLEE0(irq.irq_disable);
689+
PVOP_ALT_VCALLEE0(irq.irq_disable, "cli;", ALT_NOT(X86_FEATURE_XENPV));
667690
}
668691

669692
static inline notrace void arch_local_irq_enable(void)
670693
{
671-
PVOP_VCALLEE0(irq.irq_enable);
694+
PVOP_ALT_VCALLEE0(irq.irq_enable, "sti;", ALT_NOT(X86_FEATURE_XENPV));
672695
}
673696

674697
static inline notrace unsigned long arch_local_irq_save(void)
@@ -713,35 +736,9 @@ extern void default_banner(void);
713736
.popsection
714737

715738

716-
#define COND_PUSH(set, mask, reg) \
717-
.if ((~(set)) & mask); push %reg; .endif
718-
#define COND_POP(set, mask, reg) \
719-
.if ((~(set)) & mask); pop %reg; .endif
720-
721739
#ifdef CONFIG_X86_64
722740
#ifdef CONFIG_PARAVIRT_XXL
723741

724-
#define PV_SAVE_REGS(set) \
725-
COND_PUSH(set, CLBR_RAX, rax); \
726-
COND_PUSH(set, CLBR_RCX, rcx); \
727-
COND_PUSH(set, CLBR_RDX, rdx); \
728-
COND_PUSH(set, CLBR_RSI, rsi); \
729-
COND_PUSH(set, CLBR_RDI, rdi); \
730-
COND_PUSH(set, CLBR_R8, r8); \
731-
COND_PUSH(set, CLBR_R9, r9); \
732-
COND_PUSH(set, CLBR_R10, r10); \
733-
COND_PUSH(set, CLBR_R11, r11)
734-
#define PV_RESTORE_REGS(set) \
735-
COND_POP(set, CLBR_R11, r11); \
736-
COND_POP(set, CLBR_R10, r10); \
737-
COND_POP(set, CLBR_R9, r9); \
738-
COND_POP(set, CLBR_R8, r8); \
739-
COND_POP(set, CLBR_RDI, rdi); \
740-
COND_POP(set, CLBR_RSI, rsi); \
741-
COND_POP(set, CLBR_RDX, rdx); \
742-
COND_POP(set, CLBR_RCX, rcx); \
743-
COND_POP(set, CLBR_RAX, rax)
744-
745742
#define PARA_PATCH(off) ((off) / 8)
746743
#define PARA_SITE(ptype, ops) _PVSITE(ptype, ops, .quad, 8)
747744
#define PARA_INDIRECT(addr) *addr(%rip)
@@ -752,12 +749,14 @@ extern void default_banner(void);
752749
X86_FEATURE_XENPV, "jmp xen_iret;", "jmp native_iret;")
753750

754751
#ifdef CONFIG_DEBUG_ENTRY
755-
#define SAVE_FLAGS(clobbers) \
756-
PARA_SITE(PARA_PATCH(PV_IRQ_save_fl), \
757-
PV_SAVE_REGS(clobbers | CLBR_CALLEE_SAVE); \
758-
ANNOTATE_RETPOLINE_SAFE; \
759-
call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl); \
760-
PV_RESTORE_REGS(clobbers | CLBR_CALLEE_SAVE);)
752+
.macro PARA_IRQ_save_fl
753+
PARA_SITE(PARA_PATCH(PV_IRQ_save_fl),
754+
ANNOTATE_RETPOLINE_SAFE;
755+
call PARA_INDIRECT(pv_ops+PV_IRQ_save_fl);)
756+
.endm
757+
758+
#define SAVE_FLAGS ALTERNATIVE "PARA_IRQ_save_fl;", "pushf; pop %rax;", \
759+
ALT_NOT(X86_FEATURE_XENPV)
761760
#endif
762761
#endif /* CONFIG_PARAVIRT_XXL */
763762
#endif /* CONFIG_X86_64 */

‎arch/x86/include/asm/paravirt_types.h

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
#define _ASM_X86_PARAVIRT_TYPES_H
44

55
/* Bitmask of what can be clobbered: usually at least eax. */
6-
#define CLBR_NONE 0
76
#define CLBR_EAX (1 << 0)
87
#define CLBR_ECX (1 << 1)
98
#define CLBR_EDX (1 << 2)
@@ -15,7 +14,6 @@
1514

1615
#define CLBR_ARG_REGS (CLBR_EAX | CLBR_EDX | CLBR_ECX)
1716
#define CLBR_RET_REG (CLBR_EAX | CLBR_EDX)
18-
#define CLBR_SCRATCH (0)
1917
#else
2018
#define CLBR_RAX CLBR_EAX
2119
#define CLBR_RCX CLBR_ECX
@@ -32,12 +30,9 @@
3230
#define CLBR_ARG_REGS (CLBR_RDI | CLBR_RSI | CLBR_RDX | \
3331
CLBR_RCX | CLBR_R8 | CLBR_R9)
3432
#define CLBR_RET_REG (CLBR_RAX)
35-
#define CLBR_SCRATCH (CLBR_R10 | CLBR_R11)
3633

3734
#endif /* X86_64 */
3835

39-
#define CLBR_CALLEE_SAVE ((CLBR_ARG_REGS | CLBR_SCRATCH) & ~CLBR_RET_REG)
40-
4136
#ifndef __ASSEMBLY__
4237

4338
#include <asm/desc_defs.h>
@@ -322,7 +317,6 @@ extern void (*paravirt_iret)(void);
322317
/* Simple instruction patching code. */
323318
#define NATIVE_LABEL(a,x,b) "\n\t.globl " a #x "_" #b "\n" a #x "_" #b ":\n\t"
324319

325-
unsigned paravirt_patch_ident_64(void *insn_buff, unsigned len);
326320
unsigned paravirt_patch_default(u8 type, void *insn_buff, unsigned long addr, unsigned len);
327321
unsigned paravirt_patch_insns(void *insn_buff, unsigned len, const char *start, const char *end);
328322

‎arch/x86/kernel/paravirt.c

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,10 @@ void __init default_banner(void)
5353
}
5454

5555
/* Undefined instruction for dealing with missing ops pointers. */
56-
static const unsigned char ud2a[] = { 0x0f, 0x0b };
56+
static void paravirt_BUG(void)
57+
{
58+
BUG();
59+
}
5760

5861
struct branch {
5962
unsigned char opcode;
@@ -107,17 +110,10 @@ unsigned paravirt_patch_default(u8 type, void *insn_buff,
107110
unsigned ret;
108111

109112
if (opfunc == NULL)
110-
/* If there's no function, patch it with a ud2a (BUG) */
111-
ret = paravirt_patch_insns(insn_buff, len, ud2a, ud2a+sizeof(ud2a));
113+
/* If there's no function, patch it with paravirt_BUG() */
114+
ret = paravirt_patch_call(insn_buff, paravirt_BUG, addr, len);
112115
else if (opfunc == _paravirt_nop)
113116
ret = 0;
114-
115-
#ifdef CONFIG_PARAVIRT_XXL
116-
/* identity functions just return their single argument */
117-
else if (opfunc == _paravirt_ident_64)
118-
ret = paravirt_patch_ident_64(insn_buff, len);
119-
120-
#endif
121117
else
122118
/* Otherwise call the function. */
123119
ret = paravirt_patch_call(insn_buff, opfunc, addr, len);

0 commit comments

Comments
 (0)