MSR

Signed-off-by: Ludvig Liljenberg <[email protected]>

Author: ludfjig

src/hyperlight_host/src/error.rs

@@ -144,6 +144,14 @@ pub enum HyperlightError {
     #[error("Memory Allocation Failed with OS Error {0:?}.")]
     MemoryAllocationFailed(Option<i32>),
 
+    /// MSR Read Violation - Guest attempted to read from a Model-Specific Register
+    #[error("Guest attempted to read from MSR {0:#x} which is not allowed")]
+    MsrReadViolation(u32),
+
+    /// MSR Write Violation - Guest attempted to write to a Model-Specific Register
+    #[error("Guest attempted to write {1:#x} to MSR {0:#x} which is not allowed")]
+    MsrWriteViolation(u32, u64),
+
     /// Memory Protection Failed
     #[error("Memory Protection Failed with OS Error {0:?}.")]
     MemoryProtectionFailed(Option<i32>),
@@ -322,7 +330,9 @@ impl HyperlightError {
             | HyperlightError::PoisonedSandbox
             | HyperlightError::ExecutionAccessViolation(_)
             | HyperlightError::StackOverflow()
-            | HyperlightError::MemoryAccessViolation(_, _, _) => true,
+            | HyperlightError::MemoryAccessViolation(_, _, _)
+            | HyperlightError::MsrReadViolation(_)
+            | HyperlightError::MsrWriteViolation(_, _) => true,
 
             // All other errors do not poison the sandbox.
             HyperlightError::AnyhowError(_)

src/hyperlight_host/src/hypervisor/hyperlight_vm.rs

@@ -46,7 +46,7 @@ use crate::hypervisor::hyperv_linux::MshvVm;
 use crate::hypervisor::hyperv_windows::WhpVm;
 #[cfg(kvm)]
 use crate::hypervisor::kvm::KvmVm;
-use crate::hypervisor::regs::CommonSpecialRegisters;
+use crate::hypervisor::regs::{CommonDebugRegs, CommonSpecialRegisters};
 use crate::hypervisor::vm::{Vm, VmExit};
 #[cfg(target_os = "windows")]
 use crate::hypervisor::wrappers::HandleWrapper;
@@ -532,6 +532,12 @@ impl HyperlightVm {
                         }
                     }
                 }
+                Ok(VmExit::MsrRead(msr_index)) => {
+                    break Err(HyperlightError::MsrReadViolation(msr_index));
+                }
+                Ok(VmExit::MsrWrite { msr_index, value }) => {
+                    break Err(HyperlightError::MsrWriteViolation(msr_index, value));
+                }
                 Ok(VmExit::Cancelled()) => {
                     // If cancellation was not requested for this specific guest function call,
                     // the vcpu was interrupted by a stale cancellation from a previous call
@@ -592,6 +598,16 @@ impl HyperlightVm {
         self.interrupt_handle.clone()
     }
 
+    pub(crate) fn reset_vcpu(&self) -> Result<()> {
+        self.vm.set_regs(&CommonRegisters::default())?;
+        self.vm.set_sregs(&CommonSpecialRegisters::default())?;
+        self.vm.set_fpu(&CommonFpu::default())?;
+        self.vm.set_xsave(&[0; 1024])?;
+        self.vm.set_debug_regs(&CommonDebugRegs::default())?;
+        // TODO reset MSRs
+        Ok(())
+    }
+
     #[cfg(gdb)]
     fn handle_debug(
         &mut self,

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -21,16 +21,22 @@ use std::sync::LazyLock;
 #[cfg(gdb)]
 use mshv_bindings::{DebugRegisters, hv_message_type_HVMSG_X64_EXCEPTION_INTERCEPT};
 use mshv_bindings::{
-    hv_message_type, hv_message_type_HVMSG_GPA_INTERCEPT, hv_message_type_HVMSG_UNMAPPED_GPA,
+    HV_INTERCEPT_ACCESS_MASK_READ, HV_INTERCEPT_ACCESS_MASK_WRITE, HV_INTERCEPT_ACCESS_READ,
+    HV_INTERCEPT_ACCESS_WRITE, XSave, hv_intercept_type_HV_INTERCEPT_TYPE_X64_MSR, hv_message_type,
+    hv_message_type_HVMSG_GPA_INTERCEPT, hv_message_type_HVMSG_UNMAPPED_GPA,
     hv_message_type_HVMSG_X64_HALT, hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT,
+    hv_message_type_HVMSG_X64_MSR_INTERCEPT,
     hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES,
     hv_partition_synthetic_processor_features, hv_register_assoc,
-    hv_register_name_HV_X64_REGISTER_RIP, hv_register_value, mshv_user_mem_region,
+    hv_register_name_HV_X64_REGISTER_RIP, hv_register_value, mshv_install_intercept,
+    mshv_user_mem_region,
 };
 use mshv_ioctls::{Mshv, VcpuFd, VmFd};
 use tracing::{Span, instrument};
 
-use crate::hypervisor::regs::{CommonFpu, CommonRegisters, CommonSpecialRegisters};
+use crate::hypervisor::regs::{
+    CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters,
+};
 use crate::hypervisor::vm::{Vm, VmExit};
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::{Result, new_error};
@@ -81,8 +87,15 @@ impl MshvVm {
             vm_fd
         };
 
-        let vcpu_fd = vm_fd.create_vcpu(0)?;
+        // Install MSR intercepts, will interrupt on all MSR accesses (READ & WRITE)
+        let intercept = mshv_install_intercept {
+            access_type_mask: HV_INTERCEPT_ACCESS_MASK_WRITE | HV_INTERCEPT_ACCESS_MASK_READ,
+            intercept_type: hv_intercept_type_HV_INTERCEPT_TYPE_X64_MSR,
+            intercept_parameter: Default::default(),
+        };
+        vm_fd.install_intercept(intercept)?;
 
+        let vcpu_fd = vm_fd.create_vcpu(0)?;
         Ok(Self { vm_fd, vcpu_fd })
     }
 }
@@ -114,12 +127,32 @@ impl Vm for MshvVm {
         Ok(())
     }
 
-    #[cfg(crashdump)]
     fn xsave(&self) -> Result<Vec<u8>> {
         let xsave = self.vcpu_fd.get_xsave()?;
         Ok(xsave.buffer.to_vec())
     }
 
+    fn set_xsave(&self, xsave: &[u32; 1024]) -> Result<()> {
+        let mut buf = XSave::default();
+        let (prefix, bytes, suffix) = unsafe { xsave.align_to() };
+        assert!(prefix.is_empty());
+        assert!(suffix.is_empty());
+        buf.buffer.copy_from_slice(bytes);
+        self.vcpu_fd.set_xsave(&buf)?;
+        Ok(())
+    }
+
+    fn debug_regs(&self) -> Result<CommonDebugRegs> {
+        let debug_regs = self.vcpu_fd.get_debug_regs()?;
+        Ok(debug_regs.into())
+    }
+
+    fn set_debug_regs(&self, drs: &CommonDebugRegs) -> Result<()> {
+        let mshv_debug_regs = drs.into();
+        self.vcpu_fd.set_debug_regs(&mshv_debug_regs)?;
+        Ok(())
+    }
+
     /// # Safety
     /// The caller must ensure that the memory region is valid and points to valid memory,
     /// and lives long enough for the VM to use it.
@@ -140,6 +173,7 @@ impl Vm for MshvVm {
         const IO_PORT: hv_message_type = hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT;
         const UNMAPPED_GPA: hv_message_type = hv_message_type_HVMSG_UNMAPPED_GPA;
         const INVALID_GPA: hv_message_type = hv_message_type_HVMSG_GPA_INTERCEPT;
+        const MSR: hv_message_type = hv_message_type_HVMSG_X64_MSR_INTERCEPT;
         #[cfg(gdb)]
         const EXCEPTION_INTERCEPT: hv_message_type = hv_message_type_HVMSG_X64_EXCEPTION_INTERCEPT;
 
@@ -181,6 +215,21 @@ impl Vm for MshvVm {
                         _ => VmExit::Unknown("Unknown MMIO access".to_string()),
                     }
                 }
+                MSR => {
+                    let msr_message = m.to_msr_info().map_err(mshv_ioctls::MshvError::from)?;
+                    let edx = msr_message.rdx;
+                    let eax = msr_message.rax;
+                    let written_value = (edx << 32) | eax;
+                    let access = msr_message.header.intercept_access_type as u32;
+                    match access {
+                        HV_INTERCEPT_ACCESS_READ => VmExit::MsrRead(msr_message.msr_number),
+                        HV_INTERCEPT_ACCESS_WRITE => VmExit::MsrWrite {
+                            msr_index: msr_message.msr_number,
+                            value: written_value,
+                        },
+                        _ => VmExit::Unknown(format!("Unknown MSR access type={}", access)),
+                    }
+                }
                 #[cfg(gdb)]
                 EXCEPTION_INTERCEPT => {
                     let exception_message = m

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -34,7 +34,9 @@ use super::vm::{Vm, VmExit};
 #[cfg(not(gdb))]
 use super::vm::{Vm, VmExit};
 use super::wrappers::HandleWrapper;
-use crate::hypervisor::regs::{CommonFpu, CommonRegisters, CommonSpecialRegisters};
+use crate::hypervisor::regs::{
+    CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters,
+};
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::{Result, log_then_return, new_error};
 
@@ -104,6 +106,18 @@ impl WhpVm {
                 &NUM_CPU as *const _ as *const _,
                 std::mem::size_of_val(&NUM_CPU) as _,
             )?;
+
+            // Enable MSR exits through Extended VM Exits - must be set BEFORE WHvSetupPartition
+            let mut extended_exits_property = WHV_PARTITION_PROPERTY::default();
+            // X64MsrExit bit position, missing from rust bindings for some reason. See https://github.com/MicrosoftDocs/Virtualization-Documentation/blob/265f685159dfd3b2fae8d1dcf1b7d206c31ee880/virtualization/api/hypervisor-platform/headers/WinHvPlatformDefs.h#L1495
+            extended_exits_property.ExtendedVmExits.AsUINT64 = 1 << 1;
+            WHvSetPartitionProperty(
+                partition,
+                WHvPartitionPropertyCodeExtendedVmExits,
+                &extended_exits_property as *const _ as *const _,
+                std::mem::size_of::<WHV_PARTITION_PROPERTY>() as _,
+            )?;
+
             WHvSetupPartition(partition)?;
             WHvCreateVirtualProcessor(partition, 0, 0)?;
             partition
@@ -392,6 +406,22 @@ impl Vm for WhpVm {
         log_then_return!("Mapping host memory into the guest not yet supported on this platform");
     }
 
+    fn xsave(&self) -> Result<Vec<u8>> {
+        todo!()
+    }
+
+    fn set_xsave(&self, xsave: &[u32; 1024]) -> Result<()> {
+        todo!()
+    }
+
+    fn debug_regs(&self) -> Result<CommonDebugRegs> {
+        todo!()
+    }
+
+    fn set_debug_regs(&self, drs: &CommonDebugRegs) -> Result<()> {
+        todo!()
+    }
+
     #[expect(non_upper_case_globals, reason = "Windows API constant are lower case")]
     fn run_vcpu(&mut self) -> Result<VmExit> {
         let mut exit_context: WHV_RUN_VP_EXIT_CONTEXT = Default::default();
@@ -438,6 +468,23 @@ impl Vm for WhpVm {
             }
             // Execution was cancelled by the host.
             WHvRunVpExitReasonCanceled => VmExit::Cancelled(),
+            // MSR access (read or write) - we configured all MSR writes to cause exits
+            WHvRunVpExitReasonX64MsrAccess => {
+                let msr_access = unsafe { exit_context.Anonymous.MsrAccess };
+                let eax = msr_access.Rax;
+                let edx = msr_access.Rdx;
+                let written_value = (edx << 32) | eax;
+                let access = unsafe { msr_access.AccessInfo.AsUINT32 } as u32;
+                // Missing from rust bindings for some reason, see https://github.com/MicrosoftDocs/Virtualization-Documentation/blob/265f685159dfd3b2fae8d1dcf1b7d206c31ee880/virtualization/api/hypervisor-platform/headers/WinHvPlatformDefs.h#L3020
+                match access {
+                    0 => VmExit::MsrRead(msr_access.MsrNumber),
+                    1 => VmExit::MsrWrite {
+                        msr_index: msr_access.MsrNumber,
+                        value: written_value,
+                    },
+                    _ => VmExit::Unknown(format!("Unknown MSR access type={}", access)),
+                }
+            }
             #[cfg(gdb)]
             WHvRunVpExitReasonException => {
                 let exception = unsafe { exit_context.Anonymous.VpException };

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -18,12 +18,14 @@ use std::sync::LazyLock;
 
 #[cfg(gdb)]
 use kvm_bindings::kvm_guest_debug;
-use kvm_bindings::kvm_userspace_memory_region;
-use kvm_ioctls::Cap::UserMemory;
+use kvm_bindings::{kvm_debugregs, kvm_userspace_memory_region, kvm_xsave};
+use kvm_ioctls::Cap::{self, UserMemory};
 use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd};
 use tracing::{Span, instrument};
 
-use crate::hypervisor::regs::{CommonFpu, CommonRegisters, CommonSpecialRegisters};
+use crate::hypervisor::regs::{
+    CommonDebugRegs, CommonFpu, CommonRegisters, CommonSpecialRegisters,
+};
 use crate::hypervisor::vm::{Vm, VmExit};
 use crate::mem::memory_region::MemoryRegion;
 use crate::{Result, new_error};
@@ -107,7 +109,6 @@ impl Vm for KvmVm {
         Ok(self.vcpu_fd.set_fpu(&fpu.into())?)
     }
 
-    #[cfg(crashdump)]
     fn xsave(&self) -> Result<Vec<u8>> {
         let xsave = self.vcpu_fd.get_xsave()?;
         Ok(xsave
@@ -117,6 +118,34 @@ impl Vm for KvmVm {
             .collect())
     }
 
+    fn set_xsave(&self, xsave: &[u32; 1024]) -> Result<()> {
+        if self.vm_fd.check_extension_int(Cap::Xsave2) as usize != xsave.len() * size_of::<u32>() {
+            return Err(new_error!(
+                "KVM_CAP_XSAVE2 not supported: {}",
+                self.vm_fd.check_extension_int(Cap::Xsave2)
+            ));
+        }
+        let xsave = kvm_xsave {
+            region: *xsave,
+            ..Default::default()
+        };
+        // we made sure above that SET_XSAVE only copies 4096 bytes
+        unsafe { self.vcpu_fd.set_xsave(&xsave)? };
+
+        Ok(())
+    }
+
+    fn debug_regs(&self) -> Result<CommonDebugRegs> {
+        let kvm_debug_regs = self.vcpu_fd.get_debug_regs()?;
+        Ok(kvm_debug_regs.into())
+    }
+
+    fn set_debug_regs(&self, drs: &CommonDebugRegs) -> Result<()> {
+        let kvm_debug_regs: kvm_debugregs = drs.into();
+        self.vcpu_fd.set_debug_regs(&kvm_debug_regs)?;
+        Ok(())
+    }
+
     unsafe fn map_memory(&mut self, (slot, region): (u32, &MemoryRegion)) -> Result<()> {
         let mut kvm_region: kvm_userspace_memory_region = region.into();
         kvm_region.slot = slot;
@@ -258,3 +287,16 @@ impl Vm for KvmVm {
         Ok(())
     }
 }
+
+#[test]
+fn test() {
+    let msr_feature_index_list = KVM.as_ref().unwrap().get_msr_index_list().unwrap();
+    msr_feature_index_list.as_slice().iter().for_each(|msr| {
+        println!("Writable MSR: 0x{:08X}", msr);
+    });
+    println!(
+        "Total Writable MSRs: {}",
+        msr_feature_index_list.as_slice().len()
+    );
+    // println!("msr_feature_index_list: {:?}", msr_feature_index_list);
+}

src/hyperlight_host/src/hypervisor/regs.rs

@@ -14,13 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+mod debug_regs;
 mod fpu;
 mod special_regs;
 mod standard_regs;
 
 #[cfg(target_os = "windows")]
 use std::collections::HashSet;
 
+pub(crate) use debug_regs::*;
 pub(crate) use fpu::*;
 pub(crate) use special_regs::*;
 pub(crate) use standard_regs::*;