[hyperlight_host] Plumb a trace file descriptor around

This adds (unused) support for creating trace files for sandboxes and
passing them around to relevant sandbox event handler code. This will
be used for collecting debug trace and profiling information.

Signed-off-by: Lucy Menon <[email protected]>
Signed-off-by: Doru Blânzeanu <[email protected]>

Author: syntactically

Cargo.lock

@@ -44,6 +44,7 @@ anyhow = "1.0"
 metrics = "0.24.2"
 serde_json = "1.0"
 elfcore = "2.0"
+uuid = { version = "1.17.0", features = ["v4"] }
 
 [target.'cfg(windows)'.dependencies]
 windows = { version = "0.61", features = [
@@ -79,6 +80,7 @@ mshv-ioctls3 = { package="mshv-ioctls",  version = "=0.3.2", optional = true}
 [dev-dependencies]
 uuid = { version = "1.17.0", features = ["v4"] }
 signal-hook-registry = "1.4.5"
+envy = { version = "0.4.2" }
 serde = "1.0"
 proptest = "1.7.0"
 tempfile = "3.20.0"
@@ -126,6 +128,7 @@ executable_heap = []
 print_debug = []
 # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged.
 crashdump = ["dep:chrono"]
+trace_guest = []
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
 mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]

src/hyperlight_host/Cargo.toml

@@ -18,14 +18,21 @@ use std::sync::{Arc, Mutex};
 
 use tracing::{Span, instrument};
 
+#[cfg(feature = "trace_guest")]
+use super::Hypervisor;
 use crate::{Result, new_error};
 
 /// The trait representing custom logic to handle the case when
 /// a Hypervisor's virtual CPU (vCPU) informs Hyperlight the guest
 /// has initiated an outb operation.
 pub(crate) trait OutBHandlerCaller: Sync + Send {
     /// Function that gets called when an outb operation has occurred.
-    fn call(&mut self, port: u16, payload: u32) -> Result<()>;
+    fn call(
+        &mut self,
+        #[cfg(feature = "trace_guest")] hv: &mut dyn Hypervisor,
+        port: u16,
+        payload: u32,
+    ) -> Result<()>;
 }
 
 /// A convenient type representing a common way `OutBHandler` implementations
@@ -36,6 +43,10 @@ pub(crate) trait OutBHandlerCaller: Sync + Send {
 /// a &mut self).
 pub(crate) type OutBHandlerWrapper = Arc<Mutex<dyn OutBHandlerCaller>>;
 
+#[cfg(feature = "trace_guest")]
+pub(crate) type OutBHandlerFunction =
+    Box<dyn FnMut(&mut dyn Hypervisor, u16, u32) -> Result<()> + Send>;
+#[cfg(not(feature = "trace_guest"))]
 pub(crate) type OutBHandlerFunction = Box<dyn FnMut(u16, u32) -> Result<()> + Send>;
 
 /// A `OutBHandler` implementation using a `OutBHandlerFunction`
@@ -52,12 +63,22 @@ impl From<OutBHandlerFunction> for OutBHandler {
 
 impl OutBHandlerCaller for OutBHandler {
     #[instrument(err(Debug), skip_all, parent = Span::current(), level= "Trace")]
-    fn call(&mut self, port: u16, payload: u32) -> Result<()> {
+    fn call(
+        &mut self,
+        #[cfg(feature = "trace_guest")] hv: &mut dyn Hypervisor,
+        port: u16,
+        payload: u32,
+    ) -> Result<()> {
         let mut func = self
             .0
             .try_lock()
             .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?;
-        func(port, payload)
+        func(
+            #[cfg(feature = "trace_guest")]
+            hv,
+            port,
+            payload,
+        )
     }
 }
 

src/hyperlight_host/src/hypervisor/handlers.rs

@@ -72,6 +72,8 @@ use crate::HyperlightError;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
 use crate::sandbox::SandboxConfiguration;
+#[cfg(feature = "trace_guest")]
+use crate::sandbox::TraceInfo;
 #[cfg(crashdump)]
 use crate::sandbox::uninitialized::SandboxRuntimeConfig;
 use crate::{Result, log_then_return, new_error};
@@ -311,6 +313,9 @@ pub(crate) struct HypervLinuxDriver {
     gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
     #[cfg(crashdump)]
     rt_cfg: SandboxRuntimeConfig,
+    #[cfg(feature = "trace_guest")]
+    #[allow(dead_code)]
+    trace_info: TraceInfo,
 }
 
 impl HypervLinuxDriver {
@@ -322,6 +327,8 @@ impl HypervLinuxDriver {
     /// the underlying virtual CPU after this function returns. Call the
     /// `apply_registers` method to do that, or more likely call
     /// `initialise` to do it for you.
+    #[allow(clippy::too_many_arguments)]
+    // TODO: refactor this function to take fewer arguments. Add trace_info to rt_cfg
     #[instrument(skip_all, parent = Span::current(), level = "Trace")]
     pub(crate) fn new(
         mem_regions: Vec<MemoryRegion>,
@@ -331,6 +338,7 @@ impl HypervLinuxDriver {
         config: &SandboxConfiguration,
         #[cfg(gdb)] gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
         #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig,
+        #[cfg(feature = "trace_guest")] trace_info: TraceInfo,
     ) -> Result<Self> {
         let mshv = Mshv::new()?;
         let pr = Default::default();
@@ -438,6 +446,8 @@ impl HypervLinuxDriver {
             gdb_conn,
             #[cfg(crashdump)]
             rt_cfg,
+            #[cfg(feature = "trace_guest")]
+            trace_info,
         };
 
         // Send the interrupt handle to the GDB thread if debugging is enabled
@@ -646,7 +656,12 @@ impl Hypervisor for HypervLinuxDriver {
         outb_handle_fn
             .try_lock()
             .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
-            .call(port, val)?;
+            .call(
+                #[cfg(feature = "trace_guest")]
+                self,
+                port,
+                val,
+            )?;
 
         // update rip
         self.vcpu_fd.set_reg(&[hv_register_assoc {
@@ -1142,6 +1157,8 @@ mod tests {
                 #[cfg(crashdump)]
                 guest_core_dump: true,
             },
+            #[cfg(feature = "trace_guest")]
+            TraceInfo::new().unwrap(),
         )
         .unwrap();
     }

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -57,6 +57,8 @@ use crate::hypervisor::fpu::FP_CONTROL_WORD_DEFAULT;
 use crate::hypervisor::wrappers::WHvGeneralRegisters;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
+#[cfg(feature = "trace_guest")]
+use crate::sandbox::TraceInfo;
 #[cfg(crashdump)]
 use crate::sandbox::uninitialized::SandboxRuntimeConfig;
 use crate::{Result, debug, log_then_return, new_error};
@@ -280,6 +282,9 @@ pub(crate) struct HypervWindowsDriver {
     gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
     #[cfg(crashdump)]
     rt_cfg: SandboxRuntimeConfig,
+    #[cfg(feature = "trace_guest")]
+    #[allow(dead_code)]
+    trace_info: TraceInfo,
 }
 /* This does not automatically impl Send/Sync because the host
  * address of the shared memory region is a raw pointer, which are
@@ -291,6 +296,7 @@ unsafe impl Sync for HypervWindowsDriver {}
 
 impl HypervWindowsDriver {
     #[allow(clippy::too_many_arguments)]
+    // TODO: refactor this function to take fewer arguments. Add trace_info to rt_cfg
     #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
     pub(crate) fn new(
         mem_regions: Vec<MemoryRegion>,
@@ -301,6 +307,7 @@ impl HypervWindowsDriver {
         mmap_file_handle: HandleWrapper,
         #[cfg(gdb)] gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
         #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig,
+        #[cfg(feature = "trace_guest")] trace_info: TraceInfo,
     ) -> Result<Self> {
         // create and setup hypervisor partition
         let mut partition = VMPartition::new(1)?;
@@ -351,6 +358,8 @@ impl HypervWindowsDriver {
             gdb_conn,
             #[cfg(crashdump)]
             rt_cfg,
+            #[cfg(feature = "trace_guest")]
+            trace_info,
         };
 
         // Send the interrupt handle to the GDB thread if debugging is enabled
@@ -674,7 +683,12 @@ impl Hypervisor for HypervWindowsDriver {
         outb_handle_fn
             .try_lock()
             .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
-            .call(port, val)?;
+            .call(
+                #[cfg(feature = "trace_guest")]
+                self,
+                port,
+                val,
+            )?;
 
         let mut regs = self.processor.get_regs()?;
         regs.rip = rip + instruction_length;

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -46,6 +46,8 @@ use crate::HyperlightError;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
 use crate::sandbox::SandboxConfiguration;
+#[cfg(feature = "trace_guest")]
+use crate::sandbox::TraceInfo;
 #[cfg(crashdump)]
 use crate::sandbox::uninitialized::SandboxRuntimeConfig;
 use crate::{Result, log_then_return, new_error};
@@ -297,12 +299,17 @@ pub(crate) struct KVMDriver {
     gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
     #[cfg(crashdump)]
     rt_cfg: SandboxRuntimeConfig,
+    #[cfg(feature = "trace_guest")]
+    #[allow(dead_code)]
+    trace_info: TraceInfo,
 }
 
 impl KVMDriver {
     /// Create a new instance of a `KVMDriver`, with only control registers
     /// set. Standard registers will not be set, and `initialise` must
     /// be called to do so.
+    #[allow(clippy::too_many_arguments)]
+    // TODO: refactor this function to take fewer arguments. Add trace_info to rt_cfg
     #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
     pub(crate) fn new(
         mem_regions: Vec<MemoryRegion>,
@@ -312,6 +319,7 @@ impl KVMDriver {
         config: &SandboxConfiguration,
         #[cfg(gdb)] gdb_conn: Option<DebugCommChannel<DebugResponse, DebugMsg>>,
         #[cfg(crashdump)] rt_cfg: SandboxRuntimeConfig,
+        #[cfg(feature = "trace_guest")] trace_info: TraceInfo,
     ) -> Result<Self> {
         let kvm = Kvm::new()?;
 
@@ -390,6 +398,8 @@ impl KVMDriver {
             gdb_conn,
             #[cfg(crashdump)]
             rt_cfg,
+            #[cfg(feature = "trace_guest")]
+            trace_info,
         };
 
         // Send the interrupt handle to the GDB thread if debugging is enabled
@@ -570,7 +580,12 @@ impl Hypervisor for KVMDriver {
             outb_handle_fn
                 .try_lock()
                 .map_err(|e| new_error!("Error locking at {}:{}: {}", file!(), line!(), e))?
-                .call(port, value)?;
+                .call(
+                    #[cfg(feature = "trace_guest")]
+                    self,
+                    port,
+                    value,
+                )?;
         }
 
         Ok(())

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -503,10 +503,12 @@ pub(crate) mod tests {
 
     use hyperlight_testing::dummy_guest_as_string;
 
-    use super::handlers::{MemAccessHandler, OutBHandler};
+    use super::handlers::{MemAccessHandler, OutBHandler, OutBHandlerFunction};
     #[cfg(gdb)]
     use crate::hypervisor::DbgMemAccessHandlerCaller;
     use crate::mem::ptr::RawPtr;
+    #[cfg(feature = "trace_guest")]
+    use crate::sandbox::TraceInfo;
     use crate::sandbox::uninitialized::GuestBinary;
     #[cfg(any(crashdump, gdb))]
     use crate::sandbox::uninitialized::SandboxRuntimeConfig;
@@ -538,11 +540,6 @@ pub(crate) mod tests {
             return Ok(());
         }
 
-        let outb_handler: Arc<Mutex<OutBHandler>> = {
-            let func: Box<dyn FnMut(u16, u32) -> Result<()> + Send> =
-                Box::new(|_, _| -> Result<()> { Ok(()) });
-            Arc::new(Mutex::new(OutBHandler::from(func)))
-        };
         let mem_access_handler = {
             let func: Box<dyn FnMut() -> Result<()> + Send> = Box::new(|| -> Result<()> { Ok(()) });
             Arc::new(Mutex::new(MemAccessHandler::from(func)))
@@ -563,7 +560,17 @@ pub(crate) mod tests {
             &config,
             #[cfg(any(crashdump, gdb))]
             &rt_cfg,
+            #[cfg(feature = "trace_guest")]
+            TraceInfo::new()?,
         )?;
+        let outb_handler: Arc<Mutex<OutBHandler>> = {
+            #[cfg(feature = "trace_guest")]
+            #[allow(clippy::type_complexity)]
+            let func: OutBHandlerFunction = Box::new(|_, _, _| -> Result<()> { Ok(()) });
+            #[cfg(not(feature = "trace_guest"))]
+            let func: OutBHandlerFunction = Box::new(|_, _| -> Result<()> { Ok(()) });
+            Arc::new(Mutex::new(OutBHandler::from(func)))
+        };
         vm.initialise(
             RawPtr::from(0x230000),
             1234567890,

src/hyperlight_host/src/hypervisor/mod.rs

@@ -40,6 +40,9 @@ pub(crate) mod uninitialized_evolve;
 /// Trait used by the macros to paper over the differences between hyperlight and hyperlight-wasm
 mod callable;
 
+#[cfg(feature = "trace_guest")]
+use std::sync::{Arc, Mutex};
+
 /// Trait used by the macros to paper over the differences between hyperlight and hyperlight-wasm
 pub use callable::Callable;
 /// Re-export for `SandboxConfiguration` type
@@ -86,6 +89,35 @@ pub fn is_hypervisor_present() -> bool {
     hypervisor::get_available_hypervisor().is_some()
 }
 
+#[cfg(feature = "trace_guest")]
+#[derive(Clone)]
+/// The information that trace collection requires in order to write
+/// an accurate trace.
+pub(crate) struct TraceInfo {
+    /// The epoch against which trace events are timed; at least as
+    /// early as the creation of the sandbox being traced.
+    #[allow(dead_code)]
+    pub epoch: std::time::Instant,
+    /// The file to which the trace is being written
+    #[allow(dead_code)]
+    pub file: Arc<Mutex<std::fs::File>>,
+}
+#[cfg(feature = "trace_guest")]
+impl TraceInfo {
+    /// Create a new TraceInfo by saving the current time as the epoch
+    /// and generating a random filename.
+    pub fn new() -> crate::Result<Self> {
+        let mut path = std::env::current_dir()?;
+        path.push("trace");
+        path.push(uuid::Uuid::new_v4().to_string());
+        path.set_extension("trace");
+        Ok(Self {
+            epoch: std::time::Instant::now(),
+            file: Arc::new(Mutex::new(std::fs::File::create_new(path)?)),
+        })
+    }
+}
+
 pub(crate) trait WrapperGetter {
     #[allow(dead_code)]
     fn get_mgr_wrapper(&self) -> &MemMgrWrapper<HostSharedMemory>;