Rename and refactor dump_on_crash feature, clean up print_debug feature

Signed-off-by: Ludvig Liljenberg <[email protected]>

Author: ludfjig

.github/workflows/dep_rust.yml

@@ -74,6 +74,10 @@ jobs:
           # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess
           just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv' || 'kvm' }} 
 
+          # make sure certain cargo features compile
+          cargo check -p hyperlight-host --features crashdump
+          cargo check -p hyperlight-host --features print_debug
+
           # without any driver (shouldn't compile)
           just test-rust-feature-compilation-fail ${{ matrix.config }}
 

docs/debugging-hyperlight.md

@@ -39,6 +39,6 @@ cargo test --package hyperlight-host --test integration_test --features print_de
 
 ## Dumping the memory configuration, virtual processor register state and memory contents on a crash or unexpected VM Exit
 
-To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `dump_on_crash` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.
+To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.
 
 There are no tools at this time to analyze the dump file, but it can be useful for debugging.

src/hyperlight_host/Cargo.toml

@@ -118,9 +118,7 @@ function_call_metrics = []
 executable_heap = []
 # This feature enables printing of debug information to stdout in debug builds
 print_debug = []
-# This feature enables dunping of the VMs details to a file when an unexpected or error exit occurs in a VM in debug mode
-# the name of the file is output to stdout and logged.
-dump_on_crash = ["dep:tempfile"]
+crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds.
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"]
 inprocess = []

src/hyperlight_host/build.rs

@@ -94,6 +94,10 @@ fn main() -> Result<()> {
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.
         // You should never use #[cfg(feature = "inprocess")] in the codebase. Use #[cfg(inprocess)] instead.
         inprocess: { all(feature = "inprocess", debug_assertions) },
+        // crashdump feature is aliased with debug_assertions to make it only available in debug-builds.
+        crashdump: { all(feature = "crashdump", debug_assertions) },
+        // print_debug feature is aliased with debug_assertions to make it only available in debug-builds.
+        print_debug: { all(feature = "print_debug", debug_assertions) },
     }
 
     write_built_file()?;

src/hyperlight_host/src/hypervisor/crashdump.rs

@@ -0,0 +1,47 @@
+use std::io::Write;
+
+use tempfile::NamedTempFile;
+
+use super::Hypervisor;
+use crate::{new_error, Result};
+
+/// Dump registers + memory regions + raw memory to a tempfile
+#[cfg(crashdump)]
+pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
+    let mem_regions = hv.get_memory_regions();
+    let mem_size = mem_regions
+        .iter()
+        .map(|region| region.host_region.len())
+        .sum();
+    let mem_start_addr = mem_regions[0].host_region.start as *const u8;
+
+    if mem_start_addr.is_null() || mem_size == 0 {
+        return Err(new_error!(
+            "Invalid address or size while creating crashdump"
+        ));
+    }
+
+    let mut temp_file = NamedTempFile::with_prefix("mem")?;
+
+    let hv_details = format!("{:#x?}", hv);
+
+    // write hypervisor details such as registers, memory regions, etc.
+    temp_file.write_all(hv_details.as_bytes())?;
+    // write memory dump
+    temp_file.write_all(b"================ MEMORY DUMP =================\n")?;
+    // SAFETY: Address and size non-null and non-zero
+    unsafe {
+        let slice = std::slice::from_raw_parts(mem_start_addr, mem_size);
+        temp_file.write_all(slice)?;
+        temp_file.flush()?;
+    }
+    let persist_path = temp_file.path().with_extension("dmp");
+    temp_file
+        .persist(&persist_path)
+        .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?;
+
+    println!("Memory dumped to file: {:?}", persist_path);
+    log::error!("Memory dumped to file: {:?}", persist_path);
+
+    Ok(())
+}

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -37,7 +37,7 @@ use crate::hypervisor::hypervisor_handler::HypervisorHandler;
 use crate::hypervisor::HyperlightExit;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
-use crate::{debug, log_then_return, new_error, Result};
+use crate::{log_then_return, new_error, Result};
 
 /// Determine whether the HyperV for Linux hypervisor API is present
 /// and functional.
@@ -287,7 +287,7 @@ impl Hypervisor for HypervLinuxDriver {
         let result = match &self.vcpu_fd.run(hv_message) {
             Ok(m) => match m.header.message_type {
                 HALT_MESSAGE => {
-                    debug!("mshv - Halt Details : {:#?}", &self);
+                    crate::debug!("mshv - Halt Details : {:#?}", &self);
                     HyperlightExit::Halt()
                 }
                 IO_PORT_INTERCEPT_MESSAGE => {
@@ -296,7 +296,7 @@ impl Hypervisor for HypervLinuxDriver {
                     let rip = io_message.header.rip;
                     let rax = io_message.rax;
                     let instruction_length = io_message.header.instruction_length() as u64;
-                    debug!("mshv IO Details : \nPort : {}\n{:#?}", port_number, &self);
+                    crate::debug!("mshv IO Details : \nPort : {}\n{:#?}", port_number, &self);
                     HyperlightExit::IoOut(
                         port_number,
                         rax.to_le_bytes().to_vec(),
@@ -307,24 +307,22 @@ impl Hypervisor for HypervLinuxDriver {
                 UNMAPPED_GPA_MESSAGE => {
                     let mimo_message = m.to_memory_info()?;
                     let addr = mimo_message.guest_physical_address;
-                    debug!(
+                    crate::debug!(
                         "mshv MMIO unmapped GPA -Details: Address: {} \n {:#?}",
-                        addr, &self
+                        addr,
+                        &self
                     );
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     HyperlightExit::Mmio(addr)
                 }
                 INVALID_GPA_ACCESS_MESSAGE => {
                     let mimo_message = m.to_memory_info()?;
                     let gpa = mimo_message.guest_physical_address;
                     let access_info = MemoryRegionFlags::try_from(mimo_message)?;
-                    debug!(
+                    crate::debug!(
                         "mshv MMIO invalid GPA access -Details: Address: {} \n {:#?}",
-                        gpa, &self
+                        gpa,
+                        &self
                     );
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     match self.get_memory_access_violation(
                         gpa as usize,
                         &self.mem_regions,
@@ -335,9 +333,7 @@ impl Hypervisor for HypervLinuxDriver {
                     }
                 }
                 other => {
-                    debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
+                    crate::debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self);
                     log_then_return!("unknown Hyper-V run message type {:?}", other);
                 }
             },
@@ -346,9 +342,7 @@ impl Hypervisor for HypervLinuxDriver {
                 libc::EINTR => HyperlightExit::Cancelled(),
                 libc::EAGAIN => HyperlightExit::Retry(),
                 _ => {
-                    debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
+                    crate::debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self);
                     log_then_return!("Error running VCPU {:?}", e);
                 }
             },
@@ -360,6 +354,11 @@ impl Hypervisor for HypervLinuxDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 impl Drop for HypervLinuxDriver {

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -19,7 +19,6 @@ use std::fmt;
 use std::fmt::{Debug, Formatter};
 use std::string::String;
 
-use cfg_if::cfg_if;
 use hyperlight_common::mem::PAGE_SIZE_USIZE;
 use tracing::{instrument, Span};
 use windows::Win32::Foundation::HANDLE;
@@ -46,7 +45,7 @@ use crate::hypervisor::wrappers::WHvGeneralRegisters;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
 use crate::HyperlightError::{NoHypervisorFound, WindowsAPIError};
-use crate::{debug, log_then_return, new_error, Result};
+use crate::{log_then_return, new_error, Result};
 
 /// A Hypervisor driver for HyperV-on-Windows.
 pub(crate) struct HypervWindowsDriver {
@@ -463,19 +462,11 @@ impl Hypervisor for HypervWindowsDriver {
                 // see https://learn.microsoft.com/en-us/virtualization/api/hypervisor-platform/funcs/whvexitcontextdatatypes)
                 let instruction_length = exit_context.VpContext._bitfield & 0xF;
                 unsafe {
-                    cfg_if! {
-                        if #[cfg(all(feature = "print_debug", debug_assertions))] {
-                            println!(
-                                "HyperV IO Details :\n Port: {:#x} \n {:#?}",
-                                exit_context.Anonymous.IoPortAccess.PortNumber, &self
-                            );
-                        } else {
-                            debug!(
-                                "HyperV IO Details :\n Port: {:#x} \n {:#?}",
-                                exit_context.Anonymous.IoPortAccess.PortNumber, &self
-                            );
-                        }
-                    }
+                    crate::debug!(
+                        "HyperV IO Details :\n Port: {:#x} \n {:#?}",
+                        exit_context.Anonymous.IoPortAccess.PortNumber,
+                        &self
+                    );
                     HyperlightExit::IoOut(
                         exit_context.Anonymous.IoPortAccess.PortNumber,
                         exit_context
@@ -491,7 +482,7 @@ impl Hypervisor for HypervWindowsDriver {
             }
             // HvRunVpExitReasonX64Halt
             WHV_RUN_VP_EXIT_REASON(8i32) => {
-                debug!("HyperV Halt Details :\n {:#?}", &self);
+                crate::debug!("HyperV Halt Details :\n {:#?}", &self);
                 HyperlightExit::Halt()
             }
             // WHvRunVpExitReasonMemoryAccess
@@ -504,22 +495,12 @@ impl Hypervisor for HypervWindowsDriver {
                     )
                 };
                 let access_info = MemoryRegionFlags::try_from(access_info)?;
-                debug!(
+                crate::debug!(
                     "HyperV Memory Access Details :\n GPA: {:#?}\n Access Info :{:#?}\n {:#?} ",
-                    gpa, access_info, &self
+                    gpa,
+                    access_info,
+                    &self
                 );
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                {
-                    if let Err(e) = unsafe {
-                        self.write_dump_file(
-                            self.mem_regions.clone(),
-                            self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
-                            self.size,
-                        )
-                    } {
-                        println!("Error dumping memory: {}", e);
-                    }
-                }
 
                 match self.get_memory_access_violation(gpa as usize, &self.mem_regions, access_info)
                 {
@@ -531,26 +512,15 @@ impl Hypervisor for HypervWindowsDriver {
             //  Execution was cancelled by the host.
             //  This will happen when guest code runs for too long
             WHV_RUN_VP_EXIT_REASON(8193i32) => {
-                debug!("HyperV Cancelled Details :\n {:#?}", &self);
+                crate::debug!("HyperV Cancelled Details :\n {:#?}", &self);
                 HyperlightExit::Cancelled()
             }
             WHV_RUN_VP_EXIT_REASON(_) => {
-                debug!(
+                crate::debug!(
                     "HyperV Unexpected Exit Details :#nReason {:#?}\n {:#?}",
-                    exit_context.ExitReason, &self
+                    exit_context.ExitReason,
+                    &self
                 );
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                {
-                    if let Err(e) = unsafe {
-                        self.write_dump_file(
-                            self.mem_regions.clone(),
-                            self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
-                            self.size,
-                        )
-                    } {
-                        println!("Error dumping memory: {}", e);
-                    }
-                }
                 match self.get_exit_details(exit_context.ExitReason) {
                     Ok(error) => HyperlightExit::Unknown(error),
                     Err(e) => HyperlightExit::Unknown(format!("Error getting exit details: {}", e)),
@@ -569,6 +539,11 @@ impl Hypervisor for HypervWindowsDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 #[cfg(test)]

src/hyperlight_host/src/hypervisor/inprocess.rs

@@ -18,6 +18,8 @@ use std::fmt::Debug;
 use std::os::raw::c_void;
 
 use super::{HyperlightExit, Hypervisor};
+#[cfg(crashdump)]
+use crate::mem::memory_region::MemoryRegion;
 use crate::sandbox::leaked_outb::LeakedOutBWrapper;
 use crate::Result;
 
@@ -123,4 +125,9 @@ impl<'a> Hypervisor for InprocessDriver<'a> {
     fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE {
         unimplemented!("get_partition_handle should not be needed since we are in in-process mode")
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        unimplemented!("get_memory_regions is not supported since we are in in-process mode")
+    }
 }