[hyperlight_host/trace] Add HV interface for reading trace registers

This adds a new interface which tracing code can use to request the
values of registers from the hypervisor supervising a sandbox.

Signed-off-by: Lucy Menon <[email protected]>

Author: syntactically

src/hyperlight_host/Cargo.toml

@@ -125,6 +125,9 @@ crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors
 # include any particularly expensive instrumentation unless other
 # features are enabled.
 trace_guest = []
+# This feature enables unwinding the guest stack from the host, in
+# order to produce stack traces for debugging or profiling.
+unwind_guest = [ "trace_guest" ]
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"]
 inprocess = []

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -24,11 +24,18 @@ use mshv_bindings::{
     hv_register_name_HV_X64_REGISTER_RIP, hv_register_value, mshv_user_mem_region,
     FloatingPointUnit, SegmentRegister, SpecialRegisters, StandardRegisters,
 };
+#[cfg(feature = "unwind_guest")]
+use mshv_bindings::{
+    hv_register_name, hv_register_name_HV_X64_REGISTER_RAX, hv_register_name_HV_X64_REGISTER_RBP,
+    hv_register_name_HV_X64_REGISTER_RCX, hv_register_name_HV_X64_REGISTER_RSP,
+};
 use mshv_ioctls::{Mshv, VcpuFd, VmFd};
 use tracing::{instrument, Span};
 
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
+#[cfg(feature = "unwind_guest")]
+use super::TraceRegister;
 use super::{
     Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR,
     CR4_OSXMMEXCPT, CR4_PAE, EFER_LMA, EFER_LME, EFER_NX, EFER_SCE,
@@ -165,6 +172,19 @@ impl Debug for HypervLinuxDriver {
     }
 }
 
+#[cfg(feature = "unwind_guest")]
+impl From<TraceRegister> for hv_register_name {
+    fn from(r: TraceRegister) -> Self {
+        match r {
+            TraceRegister::RAX => hv_register_name_HV_X64_REGISTER_RAX,
+            TraceRegister::RCX => hv_register_name_HV_X64_REGISTER_RCX,
+            TraceRegister::RIP => hv_register_name_HV_X64_REGISTER_RIP,
+            TraceRegister::RSP => hv_register_name_HV_X64_REGISTER_RSP,
+            TraceRegister::RBP => hv_register_name_HV_X64_REGISTER_RBP,
+        }
+    }
+}
+
 impl Hypervisor for HypervLinuxDriver {
     #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
     fn initialise(
@@ -366,6 +386,17 @@ impl Hypervisor for HypervLinuxDriver {
         Ok(result)
     }
 
+    #[cfg(feature = "unwind_guest")]
+    fn read_trace_reg(&self, reg: TraceRegister) -> Result<u64> {
+        let mut assoc = [hv_register_assoc {
+            name: reg.into(),
+            ..Default::default()
+        }];
+        self.vcpu_fd.get_reg(&mut assoc)?;
+        // safety: all registers that we currently support are 64-bit
+        unsafe { Ok(assoc[0].value.reg64) }
+    }
+
     #[instrument(skip_all, parent = Span::current(), level = "Trace")]
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -22,10 +22,13 @@ use std::string::String;
 use hyperlight_common::mem::PAGE_SIZE_USIZE;
 use tracing::{instrument, Span};
 use windows::Win32::Foundation::HANDLE;
+#[cfg(feature = "trace_guest")]
+use windows::Win32::System::Hypervisor::WHV_REGISTER_NAME;
 use windows::Win32::System::Hypervisor::{
-    WHvX64RegisterCr0, WHvX64RegisterCr3, WHvX64RegisterCr4, WHvX64RegisterCs, WHvX64RegisterEfer,
-    WHV_MEMORY_ACCESS_TYPE, WHV_PARTITION_HANDLE, WHV_REGISTER_VALUE, WHV_RUN_VP_EXIT_CONTEXT,
-    WHV_RUN_VP_EXIT_REASON, WHV_X64_SEGMENT_REGISTER, WHV_X64_SEGMENT_REGISTER_0,
+    WHvGetVirtualProcessorRegisters, WHvX64RegisterCr0, WHvX64RegisterCr3, WHvX64RegisterCr4,
+    WHvX64RegisterCs, WHvX64RegisterEfer, WHV_MEMORY_ACCESS_TYPE, WHV_PARTITION_HANDLE,
+    WHV_REGISTER_VALUE, WHV_RUN_VP_EXIT_CONTEXT, WHV_RUN_VP_EXIT_REASON, WHV_X64_SEGMENT_REGISTER,
+    WHV_X64_SEGMENT_REGISTER_0,
 };
 
 use super::fpu::{FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
@@ -34,6 +37,8 @@ use super::surrogate_process::SurrogateProcess;
 use super::surrogate_process_manager::*;
 use super::windows_hypervisor_platform::{VMPartition, VMProcessor};
 use super::wrappers::WHvFPURegisters;
+#[cfg(feature = "unwind_guest")]
+use super::TraceRegister;
 use super::{
     windows_hypervisor_platform as whp, HyperlightExit, Hypervisor, VirtualCPU, CR0_AM, CR0_ET,
     CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP, CR4_OSFXSR, CR4_OSXMMEXCPT, CR4_PAE, EFER_LMA,
@@ -303,6 +308,19 @@ impl Debug for HypervWindowsDriver {
     }
 }
 
+#[cfg(feature = "trace_guest")]
+impl From<TraceRegister> for WHV_REGISTER_NAME {
+    fn from(r: TraceRegister) -> Self {
+        match r {
+            TraceRegister::RAX => windows::Win32::System::Hypervisor::WHvX64RegisterRax,
+            TraceRegister::RCX => windows::Win32::System::Hypervisor::WHvX64RegisterRcx,
+            TraceRegister::RIP => windows::Win32::System::Hypervisor::WHvX64RegisterRip,
+            TraceRegister::RSP => windows::Win32::System::Hypervisor::WHvX64RegisterRsp,
+            TraceRegister::RBP => windows::Win32::System::Hypervisor::WHvX64RegisterRbp,
+        }
+    }
+}
+
 impl Hypervisor for HypervWindowsDriver {
     #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
     fn initialise(
@@ -547,6 +565,23 @@ impl Hypervisor for HypervWindowsDriver {
         self.processor.get_partition_hdl()
     }
 
+    #[cfg(feature = "unwind_guest")]
+    fn read_trace_reg(&self, reg: TraceRegister) -> Result<u64> {
+        let register_names = [WHV_REGISTER_NAME::from(reg)];
+        let mut register_values: [WHV_REGISTER_VALUE; 1] = Default::default();
+        unsafe {
+            WHvGetVirtualProcessorRegisters(
+                self.get_partition_hdl(),
+                0,
+                register_names.as_ptr(),
+                register_names.len() as u32,
+                register_values.as_mut_ptr(),
+            )?;
+            // safety: all registers that we currently support are 64-bit
+            Ok(register_values[0].Reg64)
+        }
+    }
+
     #[instrument(skip_all, parent = Span::current(), level = "Trace")]
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor

src/hyperlight_host/src/hypervisor/inprocess.rs

@@ -17,6 +17,8 @@ limitations under the License.
 use std::fmt::Debug;
 use std::os::raw::c_void;
 
+#[cfg(feature = "unwind_guest")]
+use super::TraceRegister;
 use super::{HyperlightExit, Hypervisor};
 #[cfg(crashdump)]
 use crate::mem::memory_region::MemoryRegion;
@@ -122,6 +124,11 @@ impl<'a> Hypervisor for InprocessDriver<'a> {
         unimplemented!("run should not be needed since we are in in-process mode")
     }
 
+    #[cfg(feature = "unwind_guest")]
+    fn read_trace_reg(&self, _reg: TraceRegister) -> Result<u64> {
+        unimplemented!("read_trace_reg is not implemented in in-process mode")
+    }
+
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self
     }

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -24,6 +24,8 @@ use tracing::{instrument, Span};
 
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
 use super::handlers::{MemAccessHandlerWrapper, OutBHandlerWrapper};
+#[cfg(feature = "unwind_guest")]
+use super::TraceRegister;
 use super::{
     HyperlightExit, Hypervisor, VirtualCPU, CR0_AM, CR0_ET, CR0_MP, CR0_NE, CR0_PE, CR0_PG, CR0_WP,
     CR4_OSFXSR, CR4_OSXMMEXCPT, CR4_PAE, EFER_LMA, EFER_LME, EFER_NX, EFER_SCE,
@@ -337,6 +339,18 @@ impl Hypervisor for KVMDriver {
         Ok(result)
     }
 
+    #[cfg(feature = "unwind_guest")]
+    fn read_trace_reg(&self, reg: TraceRegister) -> Result<u64> {
+        let regs = self.vcpu_fd.get_regs()?;
+        Ok(match reg {
+            TraceRegister::RAX => regs.rax,
+            TraceRegister::RCX => regs.rcx,
+            TraceRegister::RIP => regs.rip,
+            TraceRegister::RSP => regs.rsp,
+            TraceRegister::RBP => regs.rbp,
+        })
+    }
+
     #[instrument(skip_all, parent = Span::current(), level = "Trace")]
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor

src/hyperlight_host/src/hypervisor/mod.rs

@@ -103,6 +103,21 @@ pub enum HyperlightExit {
     Retry(),
 }
 
+/// Registers which may be useful for tracing/stack unwinding
+#[cfg(feature = "trace_guest")]
+pub enum TraceRegister {
+    /// RAX
+    RAX,
+    /// RCX
+    RCX,
+    /// RIP
+    RIP,
+    /// RSP
+    RSP,
+    /// RBP
+    RBP,
+}
+
 /// A common set of hypervisor functionality
 ///
 /// Note: a lot of these structures take in an `Option<HypervisorHandler>`.
@@ -194,6 +209,10 @@ pub(crate) trait Hypervisor: Debug + Sync + Send {
 
     #[cfg(crashdump)]
     fn get_memory_regions(&self) -> &[MemoryRegion];
+
+    /// Read a register for trace/unwind purposes
+    #[cfg(feature = "unwind_guest")]
+    fn read_trace_reg(&self, reg: TraceRegister) -> Result<u64>;
 }
 
 /// A virtual CPU that can be run until an exit occurs