Savic vtl support

Microsoft/x64-cvm.config

@@ -3,5 +3,6 @@ CONFIG_VIRT_DRIVERS=y
 CONFIG_TDX_GUEST_DRIVER=y
 CONFIG_SEV_GUEST=y
 CONFIG_AMD_MEM_ENCRYPT=y
+CONFIG_AMD_SECURE_AVIC=y
 CONFIG_CRYPTO_AES=y
 CONFIG_CRYPTO_LIB_AES=y

arch/x86/Kconfig

@@ -473,6 +473,18 @@ config X86_X2APIC
 
 	  If you don't know what to do here, say N.
 
+config AMD_SECURE_AVIC
+	bool "AMD Secure AVIC"
+	depends on X86_X2APIC && AMD_MEM_ENCRYPT
+	help
+	  This enables AMD Secure AVIC support on guests that have this feature.
+
+	  AMD Secure AVIC provides hardware acceleration for performance sensitive
+	  APIC accesses and support for managing guest owned APIC state for SEV-SNP
+	  guests.
+
+	  If you don't know what to do here, say N.
+
 config X86_POSTED_MSI
 	bool "Enable MSI and MSI-x delivery by posted interrupts"
 	depends on X86_64 && IRQ_REMAP

arch/x86/boot/compressed/sev.c

@@ -394,14 +394,15 @@ void do_boot_stage2_vc(struct pt_regs *regs, unsigned long exit_code)
 				 MSR_AMD64_SNP_VMSA_REG_PROT |		\
 				 MSR_AMD64_SNP_RESERVED_BIT13 |		\
 				 MSR_AMD64_SNP_RESERVED_BIT15 |		\
+				 MSR_AMD64_SNP_SECURE_AVIC_ENABLED |	\
 				 MSR_AMD64_SNP_RESERVED_MASK)
 
 /*
  * SNP_FEATURES_PRESENT is the mask of SNP features that are implemented
  * by the guest kernel. As and when a new feature is implemented in the
  * guest kernel, a corresponding bit should be added to the mask.
  */
-#define SNP_FEATURES_PRESENT	MSR_AMD64_SNP_DEBUG_SWAP
+#define SNP_FEATURES_PRESENT	(MSR_AMD64_SNP_DEBUG_SWAP | MSR_AMD64_SNP_SECURE_AVIC_ENABLED)
 
 u64 snp_get_unsupported_features(u64 status)
 {

arch/x86/coco/core.c

@@ -100,6 +100,9 @@ static bool noinstr amd_cc_platform_has(enum cc_attr attr)
 	case CC_ATTR_HOST_SEV_SNP:
 		return cc_flags.host_sev_snp;
 
+	case CC_ATTR_SNP_SECURE_AVIC:
+		return sev_status & MSR_AMD64_SNP_SECURE_AVIC_ENABLED;
+
 	default:
 		return false;
 	}

arch/x86/coco/sev/core.c

@@ -1181,6 +1181,9 @@ static int wakeup_cpu_via_vmgexit(u32 apic_id, unsigned long start_ip, unsigned
 	vmsa->x87_ftw		= AP_INIT_X87_FTW_DEFAULT;
 	vmsa->x87_fcw		= AP_INIT_X87_FCW_DEFAULT;
 
+	if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC))
+		vmsa->vintr_ctrl	|= (V_GIF_MASK | V_NMI_ENABLE_MASK);
+
 	/* SVME must be set. */
 	vmsa->efer		= EFER_SVME;
 
@@ -1322,18 +1325,15 @@ int __init sev_es_efi_map_ghcbs(pgd_t *pgd)
 	return 0;
 }
 
-static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+static enum es_result __vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt, bool write)
 {
 	struct pt_regs *regs = ctxt->regs;
+	u64 exit_info_1 = write ? 1 : 0;
 	enum es_result ret;
-	u64 exit_info_1;
-
-	/* Is it a WRMSR? */
-	exit_info_1 = (ctxt->insn.opcode.bytes[1] == 0x30) ? 1 : 0;
 
 	if (regs->cx == MSR_SVSM_CAA) {
 		/* Writes to the SVSM CAA msr are ignored */
-		if (exit_info_1)
+		if (write)
 			return ES_OK;
 
 		regs->ax = lower_32_bits(this_cpu_read(svsm_caa_pa));
@@ -1343,21 +1343,96 @@ static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
 	}
 
 	ghcb_set_rcx(ghcb, regs->cx);
-	if (exit_info_1) {
+	if (write) {
 		ghcb_set_rax(ghcb, regs->ax);
 		ghcb_set_rdx(ghcb, regs->dx);
 	}
 
 	ret = sev_es_ghcb_hv_call(ghcb, ctxt, SVM_EXIT_MSR, exit_info_1, 0);
 
-	if ((ret == ES_OK) && (!exit_info_1)) {
+	if (ret == ES_OK && !write) {
 		regs->ax = ghcb->save.rax;
 		regs->dx = ghcb->save.rdx;
 	}
 
 	return ret;
 }
 
+static enum es_result vc_handle_msr(struct ghcb *ghcb, struct es_em_ctxt *ctxt)
+{
+	return __vc_handle_msr(ghcb, ctxt, ctxt->insn.opcode.bytes[1] == 0x30);
+}
+
+enum es_result sev_ghcb_msr_read(u64 msr, u64 *value)
+{
+	struct pt_regs regs = { .cx = msr };
+	struct es_em_ctxt ctxt = { .regs = &regs };
+	struct ghcb_state state;
+	unsigned long flags;
+	enum es_result ret;
+	struct ghcb *ghcb;
+
+	local_irq_save(flags);
+	ghcb = __sev_get_ghcb(&state);
+	vc_ghcb_invalidate(ghcb);
+
+	ret = __vc_handle_msr(ghcb, &ctxt, false);
+	if (ret == ES_OK)
+		*value = regs.ax | regs.dx << 32;
+
+	__sev_put_ghcb(&state);
+	local_irq_restore(flags);
+
+	return ret;
+}
+
+enum es_result sev_ghcb_msr_write(u64 msr, u64 value)
+{
+	struct pt_regs regs = {
+		.cx = msr,
+		.ax = lower_32_bits(value),
+		.dx = upper_32_bits(value)
+	};
+	struct es_em_ctxt ctxt = { .regs = &regs };
+	struct ghcb_state state;
+	unsigned long flags;
+	enum es_result ret;
+	struct ghcb *ghcb;
+
+	local_irq_save(flags);
+	ghcb = __sev_get_ghcb(&state);
+	vc_ghcb_invalidate(ghcb);
+
+	ret = __vc_handle_msr(ghcb, &ctxt, true);
+
+	__sev_put_ghcb(&state);
+	local_irq_restore(flags);
+
+	return ret;
+}
+
+enum es_result sev_notify_savic_gpa(u64 gpa)
+{
+	struct ghcb_state state;
+	struct es_em_ctxt ctxt;
+	unsigned long flags;
+	struct ghcb *ghcb;
+	int ret = 0;
+
+	local_irq_save(flags);
+
+	ghcb = __sev_get_ghcb(&state);
+
+	vc_ghcb_invalidate(ghcb);
+
+	ret = sev_es_ghcb_hv_call(ghcb, &ctxt, SVM_VMGEXIT_SECURE_AVIC_GPA, gpa, 0);
+
+	__sev_put_ghcb(&state);
+
+	local_irq_restore(flags);
+	return ret;
+}
+
 static void snp_register_per_cpu_ghcb(void)
 {
 	struct sev_es_runtime_data *data;

arch/x86/hyperv/hv_apic.c

@@ -288,6 +288,9 @@ static void hv_send_ipi_self(int vector)
 
 void __init hv_apic_init(void)
 {
+	if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC))
+		return;
+
 	if (ms_hyperv.hints & HV_X64_CLUSTER_IPI_RECOMMENDED) {
 		pr_info("Hyper-V: Using IPI hypercalls\n");
 		/*

arch/x86/hyperv/hv_init.c

@@ -40,6 +40,7 @@
 void *hv_hypercall_pg;
 EXPORT_SYMBOL_GPL(hv_hypercall_pg);
 
+void *hv_vp_early_input_arg;
 union hv_ghcb * __percpu *hv_ghcb_pg;
 
 /* Storage to save the hypercall page temporarily for hibernation */
@@ -84,6 +85,17 @@ static int hv_cpu_init(unsigned int cpu)
 	if (ret)
 		return ret;
 
+	/* Allow Hyper-V vector to be injected from Hypervisor. */
+	if (ms_hyperv.features & HV_ACCESS_REENLIGHTENMENT)
+		x2apic_savic_update_vector(cpu,
+					   HYPERV_REENLIGHTENMENT_VECTOR, true);
+
+	if (ms_hyperv.misc_features & HV_STIMER_DIRECT_MODE_AVAILABLE)
+		x2apic_savic_update_vector(cpu,
+					   HYPERV_STIMER0_VECTOR, true);
+
+	x2apic_savic_update_vector(cpu, HYPERVISOR_CALLBACK_VECTOR, true);
+
 	return hyperv_init_ghcb();
 }
 
@@ -357,13 +369,30 @@ void __init hyperv_init(void)
 	u64 guest_id;
 	union hv_x64_msr_hypercall_contents hypercall_msr;
 	int cpuhp;
+	int ret;
 
 	if (x86_hyper_type != X86_HYPER_MS_HYPERV)
 		return;
 
 	if (hv_common_init())
 		return;
 
+	if (cc_platform_has(CC_ATTR_SNP_SECURE_AVIC)) {
+		hv_vp_early_input_arg = kcalloc(num_possible_cpus(),
+					     PAGE_SIZE,
+					     GFP_KERNEL);
+		if (hv_vp_early_input_arg) {
+			ret = set_memory_decrypted((u64)hv_vp_early_input_arg,
+					     num_possible_cpus());
+			if (ret) {
+				kfree(hv_vp_early_input_arg);
+				goto common_free;
+			}
+		} else {
+			goto common_free;
+		}
+	}
+
 	/*
 	 * The VP assist page is useless to a TDX guest: the only use we
 	 * would have for it is lazy EOI, which can not be used with TDX.
@@ -378,7 +407,7 @@ void __init hyperv_init(void)
 		ms_hyperv.hints &= ~HV_X64_ENLIGHTENED_VMCS_RECOMMENDED;
 
 		if (!hv_isolation_type_tdx())
-			goto common_free;
+			goto free_vp_early_input_arg;
 	}
 
 	if (ms_hyperv.paravisor_present && hv_isolation_type_snp()) {
@@ -538,6 +567,10 @@ void __init hyperv_init(void)
 free_vp_assist_page:
 	kfree(hv_vp_assist_page);
 	hv_vp_assist_page = NULL;
+free_vp_early_input_arg:
+	set_memory_encrypted((u64)hv_vp_early_input_arg, num_possible_cpus());
+	kfree(hv_vp_early_input_arg);
+	hv_vp_early_input_arg = NULL;
 common_free:
 	hv_common_free();
 }

arch/x86/hyperv/ivm.c

@@ -289,6 +289,43 @@ static void snp_cleanup_vmsa(struct sev_es_save_area *vmsa)
 		free_page((unsigned long)vmsa);
 }
 
+enum es_result hv_set_savic_backing_page(u64 gfn)
+{
+	u64 control = HV_HYPERCALL_REP_COMP_1 | HVCALL_SET_VP_REGISTERS;
+	struct hv_set_vp_registers_input *input
+		= hv_vp_early_input_arg + smp_processor_id() * PAGE_SIZE;
+	union hv_x64_register_sev_gpa_page value;
+	unsigned long flags;
+	int retry = 5;
+	u64 ret;
+
+	local_irq_save(flags);
+
+	value.enabled = 1;
+	value.reserved = 0;
+	value.pagenumber = gfn;
+
+	memset(input, 0, struct_size(input, element, 1));
+	input->header.partitionid = HV_PARTITION_ID_SELF;
+	input->header.vpindex = HV_VP_INDEX_SELF;
+	input->header.inputvtl = ms_hyperv.vtl;
+	input->element[0].name = HV_X64_REGISTER_SEV_AVIC_GPA;
+	input->element[0].value.reg64 = value.u64;
+
+	do {
+		ret = hv_do_hypercall(control, input, NULL);
+	} while (ret == HV_STATUS_TIME_OUT && retry--);
+	if (!hv_result_success(ret))
+		pr_err("Failed to set secure AVIC backing page %llx.\n", ret);
+
+	local_irq_restore(flags);
+
+	if (hv_result_success(ret))
+		return ES_OK;
+	else
+		return ES_VMM_ERROR;
+}
+
 int hv_snp_boot_ap(u32 apic_id, unsigned long start_ip, unsigned int cpu)
 {
 	struct sev_es_save_area *vmsa = (struct sev_es_save_area *)

arch/x86/include/asm/apic.h

@@ -241,6 +241,16 @@ static inline u64 native_x2apic_icr_read(void)
 	return val;
 }
 
+#if defined(CONFIG_AMD_SECURE_AVIC)
+extern void x2apic_savic_update_vector(unsigned int cpu,
+				unsigned int vector,
+				bool set);
+extern void x2apic_savic_init_backing_page(void *backing_page);
+#else
+static inline void x2apic_savic_update_vector(unsigned int cpu,
+					      unsigned int vector,								      bool set) { }
+#endif
+
 extern int x2apic_mode;
 extern int x2apic_phys;
 extern void __init x2apic_set_max_apicid(u32 apicid);
@@ -305,6 +315,7 @@ struct apic {
 
 	/* Probe, setup and smpboot functions */
 	int	(*probe)(void);
+	void	(*setup)(void);
 	int	(*acpi_madt_oem_check)(char *oem_id, char *oem_table_id);
 
 	void	(*init_apic_ldr)(void);
@@ -317,6 +328,8 @@ struct apic {
 	/* wakeup secondary CPU using 64-bit wakeup point */
 	int	(*wakeup_secondary_cpu_64)(u32 apicid, unsigned long start_eip, unsigned int cpu);
 
+	void	(*update_vector)(unsigned int cpu, unsigned int vector, bool set);
+
 	char	*name;
 };
 

arch/x86/include/asm/apicdef.h

@@ -135,6 +135,8 @@
 #define		APIC_TDR_DIV_128	0xA
 #define	APIC_EFEAT	0x400
 #define	APIC_ECTRL	0x410
+#define APIC_SEOI	0x420
+#define APIC_IER	0x480
 #define APIC_EILVTn(n)	(0x500 + 0x10 * n)
 #define		APIC_EILVT_NR_AMD_K8	1	/* # of extended interrupts */
 #define		APIC_EILVT_NR_AMD_10H	4

arch/x86/include/asm/hyperv-tlfs.h

@@ -311,6 +311,14 @@ enum hv_isolation_type {
 #define HV_MSR_STIMER0_CONFIG	(HV_X64_MSR_STIMER0_CONFIG)
 #define HV_MSR_STIMER0_COUNT	(HV_X64_MSR_STIMER0_COUNT)
 
+/*
+ * Registers are only accessible via HVCALL_GET_VP_REGISTERS hvcall and
+ * there is not associated MSR address.
+ */
+#define	HV_X64_REGISTER_VSM_VP_STATUS	0x000D0003
+#define	HV_X64_VTL_MASK			GENMASK(3, 0)
+#define HV_X64_REGISTER_SEV_AVIC_GPA    0x00090043
+
 /* Hyper-V memory host visibility */
 enum hv_mem_host_visibility {
 	VMBUS_PAGE_NOT_VISIBLE		= 0,

arch/x86/include/asm/mshyperv.h

@@ -43,6 +43,7 @@ static inline unsigned char hv_get_nmi_reason(void)
 extern bool hyperv_paravisor_present;
 
 extern void *hv_hypercall_pg;
+extern void *hv_vp_early_input_arg;
 
 extern u64 hv_current_partition_id;
 
@@ -160,7 +161,7 @@ static inline u64 _hv_do_fast_hypercall8(u64 control, u64 input1)
 				      : "cc", "edi", "esi");
 	}
 #endif
-		return hv_status;
+	return hv_status;
 }
 
 static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
@@ -265,6 +266,7 @@ int hv_unmap_ioapic_interrupt(int ioapic_id, struct hv_interrupt_entry *entry);
 bool hv_ghcb_negotiate_protocol(void);
 void __noreturn hv_ghcb_terminate(unsigned int set, unsigned int reason);
 int hv_snp_boot_ap(u32 apic_id, unsigned long start_ip, unsigned int cpu);
+enum es_result hv_set_savic_backing_page(u64 gfn);
 #else
 static inline bool hv_ghcb_negotiate_protocol(void) { return false; }
 static inline void hv_ghcb_terminate(unsigned int set, unsigned int reason) {}