diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index edebf1020e04..9d2f06b795e4 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -121,6 +121,10 @@ extern char irq_entries_start[]; extern char spurious_entries_start[]; +#ifdef CONFIG_HYPERV_VTL_MODE +extern char hv_vtl_redirected_entries_start[]; +#endif + #define VECTOR_UNUSED NULL #define VECTOR_SHUTDOWN ((void *)-1L) #define VECTOR_RETRIGGERED ((void *)-2L) diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h index ad5c68f0509d..5f12dbcd198b 100644 --- a/arch/x86/include/asm/idtentry.h +++ b/arch/x86/include/asm/idtentry.h @@ -547,36 +547,50 @@ static inline void fred_install_sysvec(unsigned int vector, const idtentry_t fun * point is to mask off the bits above bit 7 because the push is sign * extending. */ +.macro __idtentry_stubs name, vector_nr_start, vector_nr, isr .align IDT_ALIGN -SYM_CODE_START(irq_entries_start) - vector=FIRST_EXTERNAL_VECTOR - .rept NR_EXTERNAL_VECTORS +SYM_CODE_START(\name) + vector=\vector_nr_start + .rept (\vector_nr) UNWIND_HINT_IRET_REGS 0 : ENDBR .byte 0x6a, vector - jmp asm_common_interrupt + jmp asm_\isr /* Ensure that the above is IDT_ALIGN bytes max */ .fill 0b + IDT_ALIGN - ., 1, 0xcc vector = vector+1 + .endr -SYM_CODE_END(irq_entries_start) + +SYM_CODE_END(\name) +.endm + +/** + * DEFINE_IDTENTRY_STUBS - DECLARE_IDTENTRY_DF - Define a block of IDT entries + * @name: Name of the entry point + * @vector_nr_start: First vector number + * @vector_nr: Number of vectors to emit + * @isr: Name of the C handler called from ASM entry point + * + * Emit a block of @vector_nr IDT entry stubs packed into IDT_ALIGN bytes. + */ +#define DEFINE_IDTENTRY_STUBS(name, vector_nr_start, vector_nr, isr) \ + __idtentry_stubs name, (vector_nr_start), (vector_nr), isr + +DEFINE_IDTENTRY_STUBS(irq_entries_start, FIRST_EXTERNAL_VECTOR, + NR_EXTERNAL_VECTORS, common_interrupt) #ifdef CONFIG_X86_LOCAL_APIC - .align IDT_ALIGN -SYM_CODE_START(spurious_entries_start) - vector=FIRST_SYSTEM_VECTOR - .rept NR_SYSTEM_VECTORS - UNWIND_HINT_IRET_REGS -0 : - ENDBR - .byte 0x6a, vector - jmp asm_spurious_interrupt - /* Ensure that the above is IDT_ALIGN bytes max */ - .fill 0b + IDT_ALIGN - ., 1, 0xcc - vector = vector+1 - .endr -SYM_CODE_END(spurious_entries_start) +DEFINE_IDTENTRY_STUBS(spurious_entries_start, FIRST_SYSTEM_VECTOR, + NR_SYSTEM_VECTORS, spurious_interrupt) +#endif + +#ifdef CONFIG_HYPERV_VTL_MODE +DEFINE_IDTENTRY_STUBS(hv_vtl_redirected_entries_start, + FIRST_HV_VTL_REDIRECTED_VECTOR, + NR_HV_VTL_REDIRECTED_VECTORS, + hv_vtl_redirected_interrupt) #endif #endif /* __ASSEMBLY__ */ @@ -761,6 +775,9 @@ DECLARE_IDTENTRY_SYSVEC(POSTED_MSI_NOTIFICATION_VECTOR, sysvec_posted_msi_notifi DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback); DECLARE_IDTENTRY_SYSVEC(HYPERV_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment); DECLARE_IDTENTRY_SYSVEC(HYPERV_STIMER0_VECTOR, sysvec_hyperv_stimer0); +#ifdef CONFIG_HYPERV_VTL_MODE +DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, hv_vtl_redirected_interrupt); +#endif #endif #if IS_ENABLED(CONFIG_ACRN_GUEST) diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h index 47051871b436..0ebfd4c54351 100644 --- a/arch/x86/include/asm/irq_vectors.h +++ b/arch/x86/include/asm/irq_vectors.h @@ -18,7 +18,9 @@ * Vectors 0 ... 31 : system traps and exceptions - hardcoded events * Vectors 32 ... 127 : device interrupts * Vector 128 : legacy int80 syscall interface - * Vectors 129 ... FIRST_SYSTEM_VECTOR-1 : device interrupts + * Vectors 129 ... FIRST_HV_VTL_REDIRECTED_VECTOR-1: more device interrupts + * Vectors FIRST_HV_VTL_REDIRECTED_VECTOR ... FIRST_SYSTEM_VECTOR-1 + * : HyperV redirected device interrupts * Vectors FIRST_SYSTEM_VECTOR ... 255 : special interrupts * * 64-bit x86 has per CPU IDT tables, 32-bit has one shared IDT table. @@ -91,6 +93,7 @@ #define MANAGED_IRQ_SHUTDOWN_VECTOR 0xef #if IS_ENABLED(CONFIG_HYPERV) +#define FIRST_HV_VTL_REDIRECTED_VECTOR 0xcc #define HYPERV_REENLIGHTENMENT_VECTOR 0xee #define HYPERV_STIMER0_VECTOR 0xed #endif @@ -111,7 +114,12 @@ #define FIRST_SYSTEM_VECTOR NR_VECTORS #endif +#ifdef CONFIG_HYPERV_VTL_MODE +#define NR_EXTERNAL_VECTORS (FIRST_HV_VTL_REDIRECTED_VECTOR - FIRST_EXTERNAL_VECTOR) +#define NR_HV_VTL_REDIRECTED_VECTORS (FIRST_SYSTEM_VECTOR - FIRST_HV_VTL_REDIRECTED_VECTOR) +#else #define NR_EXTERNAL_VECTORS (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) +#endif #define NR_SYSTEM_VECTORS (NR_VECTORS - FIRST_SYSTEM_VECTOR) /* diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index c6797cf9f37c..e7977c00b276 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -114,6 +115,9 @@ static void (*vmbus_handler)(void); static void (*hv_stimer0_handler)(void); static void (*hv_kexec_handler)(void); static void (*hv_crash_handler)(struct pt_regs *regs); +#ifdef CONFIG_HYPERV_VTL_MODE +static void (*hv_redirected_proxy_intr_handler)(u32 vector); +#endif DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_callback) { @@ -165,6 +169,48 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0) set_irq_regs(old_regs); } +#ifdef CONFIG_HYPERV_VTL_MODE +#define REDIRECTED_VECTOR_UNDEF 0xffff +static u32 redirected_proxy_intr[NR_HV_VTL_REDIRECTED_VECTORS] = { + [0 ... NR_HV_VTL_REDIRECTED_VECTORS - 1] = REDIRECTED_VECTOR_UNDEF }; + +static DEFINE_RAW_SPINLOCK(redirected_proxy_intr_lock); +static seqcount_raw_spinlock_t redirected_proxy_intr_seq = + SEQCNT_RAW_SPINLOCK_ZERO(redirected_proxy_intr_seq, + &redirected_proxy_intr_lock); + +DEFINE_IDTENTRY_IRQ(hv_vtl_redirected_interrupt) +{ + unsigned int seq; + u32 proxy_intr; + + if (!hv_redirected_proxy_intr_handler) { + pr_err_ratelimited("redirected intr handler not defined. hw intr %u not handled\n", vector); + goto out; + } + + if (vector < FIRST_HV_VTL_REDIRECTED_VECTOR || vector > FIRST_SYSTEM_VECTOR) { + pr_err_ratelimited("redirected hw vector %u is invalid\n", vector); + goto out; + } + + do { + seq = read_seqcount_begin(&redirected_proxy_intr_seq); + proxy_intr = redirected_proxy_intr[vector - FIRST_HV_VTL_REDIRECTED_VECTOR]; + } while (read_seqcount_retry(&redirected_proxy_intr_seq, seq)); + + if (proxy_intr == REDIRECTED_VECTOR_UNDEF) { + pr_err_ratelimited("hw vector %u not redictected to proxy interrupt\n", vector); + goto out; + } + + hv_redirected_proxy_intr_handler(proxy_intr); + +out: + apic_eoi(); +} +#endif + /* For x86/x64, override weak placeholders in hyperv_timer.c */ void hv_setup_stimer0_handler(void (*handler)(void)) { @@ -197,6 +243,23 @@ void hv_remove_crash_handler(void) hv_crash_handler = NULL; } +#ifdef CONFIG_HYPERV_VTL_MODE +void hv_setup_redirected_proxy_intr_handler(void (*handler)(u32 vector)) +{ + hv_redirected_proxy_intr_handler = handler; +} + +void hv_remove_redirected_proxy_intr_handler(void) +{ + /* + * The interrupt gate is still allocated. Interrupts will + * continue to be acked with out a handler they cannot be + * proper processed. + */ + hv_redirected_proxy_intr_handler = NULL; +} +#endif + #ifdef CONFIG_KEXEC_CORE static void hv_machine_shutdown(void) { @@ -708,6 +771,83 @@ static bool hv_sev_es_hcall_finish(struct ghcb *ghcb, struct pt_regs *regs) } #endif +#ifdef CONFIG_HYPERV_VTL_MODE +static int find_redirected_proxy_intr(u32 vector) +{ + int v; + + for (v = 0; v < ARRAY_SIZE(redirected_proxy_intr); v++) { + if (redirected_proxy_intr[v] == vector) + return v; + } + + return -ENOENT; +} + +static void redirected_proxy_intr_write_lock(unsigned long *flags) +{ + raw_spin_lock_irqsave(&redirected_proxy_intr_lock, *flags); + write_seqcount_begin(&redirected_proxy_intr_seq); +} + +static void redirected_proxy_intr_write_unlock(unsigned long *flags) +{ + write_seqcount_end(&redirected_proxy_intr_seq); + raw_spin_unlock_irqrestore(&redirected_proxy_intr_lock, *flags); +} + +int mshv_vtl_map_redirected_intr(u32 proxy_vector) +{ + int vec_idx; + unsigned long flags; + + if (proxy_vector >= NR_VECTORS) + return -EINVAL; + + redirected_proxy_intr_write_lock(&flags); + + /* Check if the vector is already redirected. */ + vec_idx = find_redirected_proxy_intr(proxy_vector); + if (vec_idx >= 0) { + redirected_proxy_intr_write_unlock(&flags); + /* Caller expects the hardware vector, not the array index. */ + return vec_idx + FIRST_HV_VTL_REDIRECTED_VECTOR; + } + + /* Now that we know it is not redirected, find a free slot. */ + vec_idx = find_redirected_proxy_intr(REDIRECTED_VECTOR_UNDEF); + if (vec_idx < 0) { + redirected_proxy_intr_write_unlock(&flags); + return -EBUSY; + } + + redirected_proxy_intr[vec_idx] = proxy_vector; + + redirected_proxy_intr_write_unlock(&flags); + + /* Caller expects the hardware vector, not the array index. */ + return vec_idx + FIRST_HV_VTL_REDIRECTED_VECTOR; +} + +int mshv_vtl_unmap_redirected_intr(u32 hw_vector) +{ + unsigned long flags; + int vec_idx = hw_vector - FIRST_HV_VTL_REDIRECTED_VECTOR; + + if (vec_idx < 0 || vec_idx >= ARRAY_SIZE(redirected_proxy_intr)) + return -EINVAL; + + if (redirected_proxy_intr[vec_idx] == REDIRECTED_VECTOR_UNDEF) + return -ENOENT; + + redirected_proxy_intr_write_lock(&flags); + redirected_proxy_intr[vec_idx] = REDIRECTED_VECTOR_UNDEF; + redirected_proxy_intr_write_unlock(&flags); + + return 0; +} +#endif /* CONFIG_HYPERV_VTL_MODE */ + const __initconst struct hypervisor_x86 x86_hyper_ms_hyperv = { .name = "Microsoft Hyper-V", .detect = ms_hyperv_platform, diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index f445bec516a0..d3fc5856ebcc 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -278,6 +278,20 @@ static void __init idt_map_in_cea(void) idt_descr.address = CPU_ENTRY_AREA_RO_IDT; } +static void __init idt_setup_hv_vtl_redirected_vectors(void) +{ +#ifdef CONFIG_HYPERV_VTL_MODE + int i = FIRST_HV_VTL_REDIRECTED_VECTOR; + + for (i = FIRST_HV_VTL_REDIRECTED_VECTOR; i < FIRST_SYSTEM_VECTOR; i++) { + void *entry = hv_vtl_redirected_entries_start + + IDT_ALIGN * (i - FIRST_HV_VTL_REDIRECTED_VECTOR); + if (!WARN_ON(test_and_set_bit(i, system_vectors))) + set_intr_gate(i, entry); + } +#endif +} + /** * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates */ @@ -288,6 +302,8 @@ void __init idt_setup_apic_and_irq_gates(void) idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true); + idt_setup_hv_vtl_redirected_vectors(); + for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) { entry = irq_entries_start + IDT_ALIGN * (i - FIRST_EXTERNAL_VECTOR); set_intr_gate(i, entry); diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c index 502d90a5b322..61bb07381202 100644 --- a/drivers/hv/hv_common.c +++ b/drivers/hv/hv_common.c @@ -871,6 +871,28 @@ u64 __weak hv_tdx_hypercall(u64 control, u64 param1, u64 param2) } EXPORT_SYMBOL_GPL(hv_tdx_hypercall); +int __weak mshv_vtl_map_redirected_intr(u32 vector) +{ + return -ENODEV; +} +EXPORT_SYMBOL_GPL(mshv_vtl_map_redirected_intr); + +int __weak mshv_vtl_unmap_redirected_intr(u32 vector) +{ + return -ENODEV; +} +EXPORT_SYMBOL_GPL(mshv_vtl_unmap_redirected_intr); + +void __weak hv_setup_redirected_proxy_intr_handler(void (*handler)(u32 vector)) +{ +} +EXPORT_SYMBOL_GPL(hv_setup_redirected_proxy_intr_handler); + +void __weak hv_remove_redirected_proxy_intr_handler(void) +{ +} +EXPORT_SYMBOL_GPL(hv_remove_redirected_proxy_intr_handler); + int hv_call_create_vp(int node, u64 partition_id, u32 vp_index, u32 flags) { struct hv_create_vp *input; diff --git a/drivers/hv/mshv_vtl_main.c b/drivers/hv/mshv_vtl_main.c index ec7a2403f841..eb49db0f5239 100644 --- a/drivers/hv/mshv_vtl_main.c +++ b/drivers/hv/mshv_vtl_main.c @@ -430,13 +430,31 @@ static int mshv_vtl_get_vsm_regs(void) return ret; } +static void do_assert_single_proxy_intr(const u32 vector, struct mshv_vtl_run *run) +{ + /* See mshv_tdx_handle_simple_icr_write() on how the bank and bit are computed. */ + const u32 bank = vector >> 5; + const u32 masked_irr = BIT(vector & 0x1f) & ~READ_ONCE(run->proxy_irr_blocked[bank]); + + /* nb atomic_t cast: See comment in mshv_tdx_handle_simple_icr_write */ + atomic_or(masked_irr, (atomic_t *)&run->proxy_irr[bank]); +} + +static void assert_single_proxy_intr(const u32 vector) +{ + struct mshv_vtl_run *run = mshv_vtl_this_run(); + + do_assert_single_proxy_intr(vector, run); + + WRITE_ONCE(run->scan_proxy_irr, 1); +} + static void mshv_vtl_scan_proxy_interrupts(struct hv_per_cpu_context *per_cpu) { struct hv_message *msg; u32 message_type; struct hv_x64_proxy_interrupt_message_payload *proxy; struct mshv_vtl_run *run; - u32 vector; msg = (struct hv_message *)per_cpu->synic_message_page + HV_SYNIC_INTERCEPTION_SINT_INDEX; for (;;) { @@ -466,13 +484,9 @@ static void mshv_vtl_scan_proxy_interrupts(struct hv_per_cpu_context *per_cpu) } } else { /* A malicious hypervisor might set a vector > 255. */ - vector = READ_ONCE(proxy->u.asserted_vector) & 0xff; - const u32 bank = vector / 32; - const u32 masked_irr = BIT(vector % 32) & - ~READ_ONCE(run->proxy_irr_blocked[bank]); + const u32 vector = READ_ONCE(proxy->u.asserted_vector) & 0xff; - /* nb atomic_t cast: See comment in mshv_tdx_handle_simple_icr_write */ - atomic_or(masked_irr, (atomic_t *)&run->proxy_irr[bank]); + do_assert_single_proxy_intr(vector, run); } WRITE_ONCE(run->scan_proxy_irr, 1); @@ -1120,8 +1134,8 @@ static int mshv_tdx_handle_simple_icr_write(struct tdx_vp_context *context) const u32 dest = context->l2_enter_guest_state.rdx; const u8 shorthand = (icr_lo >> 18) & 0b11; const u8 vector = icr_lo; - const u64 bank = vector / 32; - const u32 mask = BIT(vector % 32); + const u64 bank = vector >> 5; /* Each bank is 32 bits. Divide by 32 to find the bank. */ + const u32 mask = BIT(vector & 0x1f); /* Bit in the bank is the remainder of the division. */ const u32 self = smp_processor_id(); bool send_ipi = false; @@ -1930,6 +1944,36 @@ static inline long mshv_vtl_ioctl_kick_cpu(void __user *user_arg) return 0; } +static long mshv_vtl_ioctl_setup_redirected_intr(void __user *user_arg) +{ + struct mshv_map_device_intr intr_data; + int ret; + + if (copy_from_user(&intr_data, user_arg, sizeof(intr_data))) + return (long)-EFAULT; + + /* User space provides the hardware vector to unmap. */ + if (!intr_data.create_mapping) + return (long)mshv_vtl_unmap_redirected_intr(intr_data.vector); + + /* + * User space provides the proxy vector it wants to map to a hardware + * vector. + */ + ret = mshv_vtl_map_redirected_intr(intr_data.vector); + if (ret < 0) + return (long)ret; + + /* + * The return value is the hardware vector to which the proxy vector + * is mapped. + */ + intr_data.vector = ret; + ret = copy_to_user(user_arg, &intr_data, sizeof(intr_data)) ? -EFAULT : 0; + + return (long)ret; +} + static long mshv_vtl_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -1982,6 +2026,9 @@ mshv_vtl_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) ret = mshv_vtl_ioctl_guest_vsm_vmsa_pfn((void __user *)arg); break; #endif + case MSHV_VTL_MAP_REDIRECTED_DEVICE_INTERRUPT: + ret = mshv_vtl_ioctl_setup_redirected_intr((void __user *)arg); + break; case MSHV_VTL_KICK_CPU: ret = mshv_vtl_ioctl_kick_cpu((void __user *)arg); @@ -2685,6 +2732,8 @@ static int __init mshv_vtl_init(void) goto free_sidecar; } + hv_setup_redirected_proxy_intr_handler(assert_single_proxy_intr); + mutex_init(&mshv_vtl_poll_file_lock); device_initialize(mem_dev); @@ -2724,6 +2773,7 @@ static int __init mshv_vtl_init(void) static void __exit mshv_vtl_exit(void) { mshv_setup_vtl_func(NULL, NULL, NULL); + hv_remove_redirected_proxy_intr_handler(); mshv_tdx_free_apicid_to_cpuid_mapping(); misc_deregister(&mshv_vtl_sint_dev); misc_deregister(&mshv_vtl_hvcall); diff --git a/include/asm-generic/mshyperv.h b/include/asm-generic/mshyperv.h index fad13d0dd17d..e203bdaa9b4a 100644 --- a/include/asm-generic/mshyperv.h +++ b/include/asm-generic/mshyperv.h @@ -207,6 +207,11 @@ void hv_remove_kexec_handler(void); void hv_setup_crash_handler(void (*handler)(struct pt_regs *regs)); void hv_remove_crash_handler(void); +int mshv_vtl_map_redirected_intr(u32 proxy_vector); +int mshv_vtl_unmap_redirected_intr(u32 hw_vector); +void hv_setup_redirected_proxy_intr_handler(void (*handler)(u32 vector)); +void hv_remove_redirected_proxy_intr_handler(void); + extern int vmbus_interrupt; extern int vmbus_irq; diff --git a/include/uapi/linux/mshv.h b/include/uapi/linux/mshv.h index 7ba3a3f24989..75f29f684973 100644 --- a/include/uapi/linux/mshv.h +++ b/include/uapi/linux/mshv.h @@ -298,9 +298,13 @@ struct mshv_kick_cpus { __u64 flags; } __packed; +struct mshv_map_device_intr { + __u32 vector; + bool create_mapping; +} __packed; + #define MSHV_KICK_CPUS_FLAG_WAIT_FOR_CPUS (1 << 0) #define MSHV_KICK_CPUS_FLAG_CANCEL_CPU_RUN (1 << 1) - #define MSHV_IOCTL 0xB8 /* mshv device */ @@ -359,6 +363,8 @@ struct mshv_kick_cpus { /* For x86-64 TDX only */ #define MSHV_VTL_TDCALL _IOWR(MSHV_IOCTL, 0x32, struct mshv_tdcall) #define MSHV_VTL_READ_VMX_CR4_FIXED1 _IOR(MSHV_IOCTL, 0x33, __u64) +#define MSHV_VTL_MAP_REDIRECTED_DEVICE_INTERRUPT _IOWR(MSHV_IOCTL, 0x39, \ + struct mshv_map_device_intr) /* For x86-64 only */ #define MSHV_VTL_GUEST_VSM_VMSA_PFN _IOWR(MSHV_IOCTL, 0x34, __u64)