Pkvm v6.18 pvvmcs part2

arch/x86/include/asm/fpu/xstate.h

@@ -131,4 +131,8 @@ static __always_inline __pure bool fpu_state_size_dynamic(void)
 }
 #endif
 
+#ifdef __PKVM_HYP__
+void pkvm_setup_xstate_cache(void);
+#endif
+
 #endif

arch/x86/include/asm/kvm_host.h

@@ -2317,6 +2317,7 @@ int kvm_emulate_xsetbv(struct kvm_vcpu *vcpu);
 
 int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
 int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
+bool pkvm_host_has_emulated_msr(struct kvm *kvm, u32 msr);
 
 unsigned long kvm_get_rflags(struct kvm_vcpu *vcpu);
 void kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags);
@@ -2516,6 +2517,7 @@ void __user *__x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa,
 				     u32 size);
 bool kvm_vcpu_is_reset_bsp(struct kvm_vcpu *vcpu);
 bool kvm_vcpu_is_bsp(struct kvm_vcpu *vcpu);
+int kvm_vcpu_x86_setup_mce(struct kvm_vcpu *vcpu, u64 mcg_cap);
 
 static inline bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
 {

arch/x86/include/asm/kvm_pkvm.h

@@ -70,6 +70,54 @@ union pkvm_hc_data {
 	struct {
 		struct pkvm_memcache memcache;
 	} vcpu_free;
+	struct {
+		u64 data;
+	} get_msr;
+	union {
+		unsigned long rsp;
+		unsigned long rip;
+		unsigned long cr0;
+		unsigned long cr3;
+		unsigned long cr4;
+		u64 pdptrs[4];
+	} cache_reg;
+	struct {
+		unsigned long data;
+	} get_rflags;
+	struct {
+		struct kvm_segment seg_val;
+		int seg;
+	} set_segment;
+	struct {
+		struct kvm_segment seg_val;
+	} get_segment;
+	struct {
+		u64 data;
+	} get_segment_base;
+	struct {
+		struct desc_ptr desc;
+	} set_idt;
+	struct {
+		struct desc_ptr desc;
+	} get_idt;
+	struct {
+		struct desc_ptr desc;
+	} set_gdt;
+	struct {
+		struct desc_ptr desc;
+	} get_gdt;
+	struct {
+		u32 data;
+	} get_interrupt_shadow;
+	struct {
+		bool data;
+	} get_nmi_mask;
+	struct {
+		struct pkvm_memcache memcache;
+	} vcpu_after_set_cpuid;
+	struct {
+		struct pkvm_memcache memcache;
+	} vcpu_add_fpstate;
 	struct {
 		u64 data[PKVM_HC_DATA_MAX_NUM];
 	} raw;
@@ -86,11 +134,16 @@ static_assert(sizeof(union pkvm_hc_data) == PKVM_HC_DATA_MAX_NUM * sizeof(u64));
 	(ALIGN(sizeof(((union pkvm_hc_data *)0)->f), sizeof(u64)) / sizeof(u64))
 
 #define PKVM_HC_OUTPUT_NUM(f)	f##_output_num
+#define PKVM_HC_INPUT_NUM(f)	f##_input_num
 
 enum {
 	#define PKVM_HC(f)	PKVM_HC_OUTPUT_NUM(f) = 0,
 	#define PKVM_HC_OUT(f)	PKVM_HC_OUTPUT_NUM(f) = PKVM_HC_DATA_NUM(f),
 	#include <asm/pkvm_hypercalls.h>
+
+	#define PKVM_HC(f)	PKVM_HC_INPUT_NUM(f) = 0,
+	#define PKVM_HC_IN(f)	PKVM_HC_INPUT_NUM(f) = PKVM_HC_DATA_NUM(f),
+	#include <asm/pkvm_hypercalls.h>
 };
 
 static inline int pkvm_hc_output_num(enum pkvm_hc hc)
@@ -103,6 +156,16 @@ static inline int pkvm_hc_output_num(enum pkvm_hc hc)
 	}
 }
 
+static inline int pkvm_hc_input_num(enum pkvm_hc hc)
+{
+	switch (hc) {
+	#define PKVM_HC(f) case TO_PKVM_HC(f): return PKVM_HC_INPUT_NUM(f);
+	#include <asm/pkvm_hypercalls.h>
+	default:
+		return 0;
+	}
+}
+
 #define PKVM_HC_IN_0()
 #define PKVM_HC_IN_1(a1)		, "b"(a1)
 #define PKVM_HC_IN_2(a1, a2)		PKVM_HC_IN_1(a1), "c"(a2)
@@ -151,6 +214,23 @@ static inline int pkvm_hc_output_num(enum pkvm_hc hc)
 			      __pkvm_hypercall(f, o, 4, ##__VA_ARGS__),			\
 	PKVM_HC_UNREACHABLE(f))))))
 
+#define pkvm_hypercall_in(f, i)								\
+	__builtin_choose_expr(PKVM_HC_INPUT_NUM(f) == 1,				\
+			      __pkvm_hypercall(f, NULL, 0, (i)->raw.data[0]),		\
+	__builtin_choose_expr(PKVM_HC_INPUT_NUM(f) == 2,				\
+			      __pkvm_hypercall(f, NULL, 0, (i)->raw.data[0],		\
+					       (i)->raw.data[1]),			\
+	__builtin_choose_expr(PKVM_HC_INPUT_NUM(f) == 3,				\
+			      __pkvm_hypercall(f, NULL, 0, (i)->raw.data[0],		\
+					       (i)->raw.data[1],			\
+					       (i)->raw.data[2]),			\
+	__builtin_choose_expr(PKVM_HC_INPUT_NUM(f) == 4,				\
+			      __pkvm_hypercall(f, NULL, 0, (i)->raw.data[0],		\
+					       (i)->raw.data[1],			\
+					       (i)->raw.data[2],			\
+					       (i)->raw.data[3]),			\
+	PKVM_HC_UNREACHABLE(f)))))
+
 static inline unsigned long pkvm_hc(struct kvm_vcpu *vcpu)
 {
 	return vcpu->arch.regs[VCPU_REGS_RAX];
@@ -160,13 +240,41 @@ static inline unsigned long pkvm_hc(struct kvm_vcpu *vcpu)
 static inline unsigned long pkvm_hc_input##n(struct kvm_vcpu *vcpu)			\
 {											\
 	return vcpu->arch.regs[VCPU_REGS_##reg];					\
+}											\
+static inline void pkvm_hc_get_input##n(struct kvm_vcpu *vcpu, union pkvm_hc_data *p)	\
+{											\
+	BUILD_BUG_ON(n == 0 || n > PKVM_HC_DATA_MAX_NUM);				\
+	p->raw.data[n - 1] = vcpu->arch.regs[VCPU_REGS_##reg];				\
 }
 
 DEFINE_PKVM_HC_INPUT(1, RBX)
 DEFINE_PKVM_HC_INPUT(2, RCX)
 DEFINE_PKVM_HC_INPUT(3, RDX)
 DEFINE_PKVM_HC_INPUT(4, RSI)
 
+static inline void pkvm_hc_get_input(struct kvm_vcpu *vcpu, enum pkvm_hc hc,
+				     union pkvm_hc_data *in)
+{
+	switch (pkvm_hc_input_num(hc)) {
+	case 4:
+		pkvm_hc_get_input4(vcpu, in);
+		fallthrough;
+	case 3:
+		pkvm_hc_get_input3(vcpu, in);
+		fallthrough;
+	case 2:
+		pkvm_hc_get_input2(vcpu, in);
+		fallthrough;
+	case 1:
+		pkvm_hc_get_input1(vcpu, in);
+		fallthrough;
+	case 0:
+		break;
+	default:
+		BUG();
+	}
+}
+
 static inline void pkvm_hc_set_ret(struct kvm_vcpu *vcpu, int ret)
 {
 	vcpu->arch.regs[VCPU_REGS_RAX] = ret;
@@ -224,8 +332,12 @@ extern struct pkvm_init_ops *pkvm_sym(init_ops);
 extern struct cpumask pkvm_sym(__cpu_possible_mask);
 extern unsigned int pkvm_sym(nr_cpu_ids);
 DECLARE_STATIC_KEY_FALSE(pkvm_sym(switch_vcpu_ibpb));
+extern u64 pkvm_sym(x86_pred_cmd);
 extern struct fpu_state_config pkvm_sym(fpu_kernel_cfg);
 extern struct fpu_state_config pkvm_sym(fpu_user_cfg);
+#ifdef CONFIG_X86_64
+DECLARE_STATIC_KEY_FALSE(pkvm_sym(__fpu_state_size_dynamic));
+#endif
 
 u64 pkvm_total_reserve_pages(void);
 PKVM_DECLARE(void *, pkvm_early_alloc_page, (void));
@@ -376,7 +488,32 @@ static inline size_t pkvm_guest_initial_fpstate_size(struct kvm *kvm)
 
 #ifdef __PKVM_HYP__
 
-#ifndef CONFIG_PKVM_X86_DEBUG
+#undef kvm_err
+#undef kvm_info
+#undef kvm_debug
+#undef kvm_debug_ratelimited
+#undef kvm_pr_unimpl
+
+#ifdef CONFIG_PKVM_X86_DEBUG
+
+#define kvm_err(fmt, ...) \
+	pr_err("pkvm: " fmt, ## __VA_ARGS__)
+#define kvm_info(fmt, ...) \
+	pr_info("pkvm: " fmt, ## __VA_ARGS__)
+#define kvm_debug(fmt, ...) \
+	pr_debug("pkvm: " fmt, ## __VA_ARGS__)
+#define kvm_debug_ratelimited(fmt, ...) \
+	pr_debug_ratelimited("pkvm: " fmt, ## __VA_ARGS__)
+#define kvm_pr_unimpl(fmt, ...) \
+	pr_err_ratelimited("pkvm: " fmt, ## __VA_ARGS__)
+
+#else /* CONFIG_PKVM_X86_DEBUG */
+
+#define kvm_err(fmt, ...) do {} while(0)
+#define kvm_info(fmt, ...) do {} while(0)
+#define kvm_debug(fmt, ...) do {} while(0)
+#define kvm_debug_ratelimited(fmt, ...) do {} while(0)
+#define kvm_pr_unimpl(fmt, ...) do {} while(0)
 
 #undef WARN_ON
 #undef WARN
@@ -402,6 +539,15 @@ static inline size_t pkvm_guest_initial_fpstate_size(struct kvm *kvm)
 
 #endif /* CONFIG_PKVM_X86_DEBUG */
 
+#undef KVM_BUG_ON
+#define KVM_BUG_ON(cond, kvm)						\
+({									\
+	bool __ret = !!(cond);						\
+									\
+	BUG_ON(__ret);							\
+	unlikely(__ret);						\
+})
+
 #endif /* __PKVM_HYP__ */
 
 #else /* !CONFIG_PKVM_X86 */

arch/x86/include/asm/pkvm_hypercalls.h

@@ -7,6 +7,10 @@ BUILD_BUG_ON(1)
 #define PKVM_HC_OUT PKVM_HC
 #endif
 
+#ifndef PKVM_HC_IN
+#define PKVM_HC_IN PKVM_HC
+#endif
+
 /* Hypercalls used only during pKVM initialization */
 PKVM_HC(init)
 PKVM_HC(init_finalize)
@@ -23,6 +27,55 @@ PKVM_HC(vm_init)
 PKVM_HC_OUT(vm_destroy)
 PKVM_HC(vcpu_create)
 PKVM_HC_OUT(vcpu_free)
+PKVM_HC(vcpu_load)
+PKVM_HC(vcpu_put)
+PKVM_HC(vcpu_reset)
+PKVM_HC(update_exception_bitmap)
+PKVM_HC(set_efer)
+PKVM_HC(set_msr)
+PKVM_HC_OUT(get_msr)
+PKVM_HC_OUT(cache_reg)
+PKVM_HC(set_cr4)
+PKVM_HC(set_cr0)
+PKVM_HC_OUT(get_rflags)
+PKVM_HC(set_rflags)
+PKVM_HC(set_dr7)
+PKVM_HC_IN(set_segment)
+PKVM_HC_OUT(get_segment)
+PKVM_HC_OUT(get_segment_base)
+PKVM_HC_IN(set_idt)
+PKVM_HC_OUT(get_idt)
+PKVM_HC_IN(set_gdt)
+PKVM_HC_OUT(get_gdt)
+PKVM_HC(flush_tlb_all)
+PKVM_HC(flush_tlb_current)
+PKVM_HC(flush_tlb_gva)
+PKVM_HC(flush_tlb_guest)
+PKVM_HC(set_interrupt_shadow)
+PKVM_HC_OUT(get_interrupt_shadow)
+PKVM_HC(enable_nmi_window)
+PKVM_HC(enable_irq_window)
+PKVM_HC(interrupt_allowed)
+PKVM_HC(nmi_allowed)
+PKVM_HC_OUT(get_nmi_mask)
+PKVM_HC(set_nmi_mask)
+PKVM_HC(inject_irq)
+PKVM_HC(inject_nmi)
+PKVM_HC(inject_exception)
+PKVM_HC(cancel_injection)
+PKVM_HC(update_cr8_intercept)
+PKVM_HC(set_virtual_apic_mode)
+PKVM_HC(refresh_apicv_exec_ctrl)
+PKVM_HC(load_eoi_exitmap)
+PKVM_HC(hwapic_isr_update)
+PKVM_HC(sync_pir_to_irr)
+PKVM_HC_OUT(vcpu_after_set_cpuid)
+PKVM_HC_OUT(vcpu_add_fpstate)
+PKVM_HC(write_tsc_offset)
+PKVM_HC(write_tsc_multiplier)
+PKVM_HC(load_mmu_pgd)
+PKVM_HC(setup_mce)
 
 #undef PKVM_HC
 #undef PKVM_HC_OUT
+#undef PKVM_HC_IN

arch/x86/include/asm/pkvm_image_vars.h

@@ -19,6 +19,7 @@ PKVM_ALIAS(__trace_bprintk);
 PKVM_ALIAS(__dynamic_pr_debug);
 PKVM_ALIAS(mem_dump_obj);
 PKVM_ALIAS(vmalloc_base);
+PKVM_ALIAS(get_cpu_entry_area);
 #endif
 
 #endif /* _ASM_x86_PKVM_IMAGE_VARS_H */