Fix cargo feature 'dump_on_crash'

.github/workflows/dep_rust.yml

@@ -74,6 +74,10 @@ jobs:
           # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess
           just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv' || 'kvm' }} 
 
+          # make sure certain cargo features compile
+          cargo check -p hyperlight-host --features crashdump
+          cargo check -p hyperlight-host --features print_debug
+
           # without any driver (shouldn't compile)
           just test-rust-feature-compilation-fail ${{ matrix.config }}
 

docs/debugging-hyperlight.md

@@ -39,6 +39,6 @@ cargo test --package hyperlight-host --test integration_test --features print_de
 
 ## Dumping the memory configuration, virtual processor register state and memory contents on a crash or unexpected VM Exit
 
-To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `dump_on_crash` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.
+To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.
 
 There are no tools at this time to analyze the dump file, but it can be useful for debugging.

src/hyperlight_host/Cargo.toml

@@ -118,9 +118,7 @@ function_call_metrics = []
 executable_heap = []
 # This feature enables printing of debug information to stdout in debug builds
 print_debug = []
-# This feature enables dunping of the VMs details to a file when an unexpected or error exit occurs in a VM in debug mode
-# the name of the file is output to stdout and logged.
-dump_on_crash = ["dep:tempfile"]
+crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds.
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"]
 inprocess = []

src/hyperlight_host/build.rs

@@ -94,6 +94,10 @@ fn main() -> Result<()> {
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.
         // You should never use #[cfg(feature = "inprocess")] in the codebase. Use #[cfg(inprocess)] instead.
         inprocess: { all(feature = "inprocess", debug_assertions) },
+        // crashdump feature is aliased with debug_assertions to make it only available in debug-builds.
+        crashdump: { all(feature = "crashdump", debug_assertions) },
+        // print_debug feature is aliased with debug_assertions to make it only available in debug-builds.
+        print_debug: { all(feature = "print_debug", debug_assertions) },
     }
 
     write_built_file()?;

src/hyperlight_host/src/hypervisor/crashdump.rs

@@ -0,0 +1,44 @@
+use std::io::Write;
+
+use tempfile::NamedTempFile;
+
+use super::Hypervisor;
+use crate::{new_error, Result};
+
+/// Dump registers + memory regions + raw memory to a tempfile
+#[cfg(crashdump)]
+pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
+    let mut temp_file = NamedTempFile::with_prefix("mem")?;
+    let hv_details = format!("{:#x?}", hv);
+
+    // write hypervisor details such as registers, info about mapped memory regions, etc.
+    temp_file.write_all(hv_details.as_bytes())?;
+    temp_file.write_all(b"================ MEMORY DUMP =================\n")?;
+
+    // write the raw memory dump for each memory region
+    for region in hv.get_memory_regions() {
+        if region.host_region.start == 0 || region.host_region.is_empty() {
+            continue;
+        }
+        // SAFETY: we got this memory region from the hypervisor so should never be invalid
+        let region_slice = unsafe {
+            std::slice::from_raw_parts(
+                region.host_region.start as *const u8,
+                region.host_region.len(),
+            )
+        };
+        temp_file.write_all(region_slice)?;
+    }
+    temp_file.flush()?;
+
+    // persist the tempfile to disk
+    let persist_path = temp_file.path().with_extension("dmp");
+    temp_file
+        .persist(&persist_path)
+        .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?;
+
+    println!("Memory dumped to file: {:?}", persist_path);
+    log::error!("Memory dumped to file: {:?}", persist_path);
+
+    Ok(())
+}

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -37,7 +37,7 @@ use crate::hypervisor::hypervisor_handler::HypervisorHandler;
 use crate::hypervisor::HyperlightExit;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
-use crate::{debug, log_then_return, new_error, Result};
+use crate::{log_then_return, new_error, Result};
 
 /// Determine whether the HyperV for Linux hypervisor API is present
 /// and functional.
@@ -287,7 +287,7 @@ impl Hypervisor for HypervLinuxDriver {
         let result = match &self.vcpu_fd.run(hv_message) {
             Ok(m) => match m.header.message_type {
                 HALT_MESSAGE => {
-                    debug!("mshv - Halt Details : {:#?}", &self);
+                    crate::debug!("mshv - Halt Details : {:#?}", &self);
                     HyperlightExit::Halt()
                 }
                 IO_PORT_INTERCEPT_MESSAGE => {
@@ -296,7 +296,7 @@ impl Hypervisor for HypervLinuxDriver {
                     let rip = io_message.header.rip;
                     let rax = io_message.rax;
                     let instruction_length = io_message.header.instruction_length() as u64;
-                    debug!("mshv IO Details : \nPort : {}\n{:#?}", port_number, &self);
+                    crate::debug!("mshv IO Details : \nPort : {}\n{:#?}", port_number, &self);
                     HyperlightExit::IoOut(
                         port_number,
                         rax.to_le_bytes().to_vec(),
@@ -307,24 +307,22 @@ impl Hypervisor for HypervLinuxDriver {
                 UNMAPPED_GPA_MESSAGE => {
                     let mimo_message = m.to_memory_info()?;
                     let addr = mimo_message.guest_physical_address;
-                    debug!(
+                    crate::debug!(
                         "mshv MMIO unmapped GPA -Details: Address: {} \n {:#?}",
-                        addr, &self
+                        addr,
+                        &self
                     );
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     HyperlightExit::Mmio(addr)
                 }
                 INVALID_GPA_ACCESS_MESSAGE => {
                     let mimo_message = m.to_memory_info()?;
                     let gpa = mimo_message.guest_physical_address;
                     let access_info = MemoryRegionFlags::try_from(mimo_message)?;
-                    debug!(
+                    crate::debug!(
                         "mshv MMIO invalid GPA access -Details: Address: {} \n {:#?}",
-                        gpa, &self
+                        gpa,
+                        &self
                     );
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     match self.get_memory_access_violation(
                         gpa as usize,
                         &self.mem_regions,
@@ -335,9 +333,7 @@ impl Hypervisor for HypervLinuxDriver {
                     }
                 }
                 other => {
-                    debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
+                    crate::debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self);
                     log_then_return!("unknown Hyper-V run message type {:?}", other);
                 }
             },
@@ -346,9 +342,7 @@ impl Hypervisor for HypervLinuxDriver {
                 libc::EINTR => HyperlightExit::Cancelled(),
                 libc::EAGAIN => HyperlightExit::Retry(),
                 _ => {
-                    debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
+                    crate::debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self);
                     log_then_return!("Error running VCPU {:?}", e);
                 }
             },
@@ -360,6 +354,11 @@ impl Hypervisor for HypervLinuxDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 impl Drop for HypervLinuxDriver {

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -19,7 +19,6 @@ use std::fmt;
 use std::fmt::{Debug, Formatter};
 use std::string::String;
 
-use cfg_if::cfg_if;
 use hyperlight_common::mem::PAGE_SIZE_USIZE;
 use tracing::{instrument, Span};
 use windows::Win32::Foundation::HANDLE;
@@ -463,19 +462,10 @@ impl Hypervisor for HypervWindowsDriver {
                 // see https://learn.microsoft.com/en-us/virtualization/api/hypervisor-platform/funcs/whvexitcontextdatatypes)
                 let instruction_length = exit_context.VpContext._bitfield & 0xF;
                 unsafe {
-                    cfg_if! {
-                        if #[cfg(all(feature = "print_debug", debug_assertions))] {
-                            println!(
-                                "HyperV IO Details :\n Port: {:#x} \n {:#?}",
-                                exit_context.Anonymous.IoPortAccess.PortNumber, &self
-                            );
-                        } else {
-                            debug!(
-                                "HyperV IO Details :\n Port: {:#x} \n {:#?}",
-                                exit_context.Anonymous.IoPortAccess.PortNumber, &self
-                            );
-                        }
-                    }
+                    debug!(
+                        "HyperV IO Details :\n Port: {:#x} \n {:#?}",
+                        exit_context.Anonymous.IoPortAccess.PortNumber, &self
+                    );
                     HyperlightExit::IoOut(
                         exit_context.Anonymous.IoPortAccess.PortNumber,
                         exit_context
@@ -508,18 +498,6 @@ impl Hypervisor for HypervWindowsDriver {
                     "HyperV Memory Access Details :\n GPA: {:#?}\n Access Info :{:#?}\n {:#?} ",
                     gpa, access_info, &self
                 );
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                {
-                    if let Err(e) = unsafe {
-                        self.write_dump_file(
-                            self.mem_regions.clone(),
-                            self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
-                            self.size,
-                        )
-                    } {
-                        println!("Error dumping memory: {}", e);
-                    }
-                }
 
                 match self.get_memory_access_violation(gpa as usize, &self.mem_regions, access_info)
                 {
@@ -539,18 +517,6 @@ impl Hypervisor for HypervWindowsDriver {
                     "HyperV Unexpected Exit Details :#nReason {:#?}\n {:#?}",
                     exit_context.ExitReason, &self
                 );
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                {
-                    if let Err(e) = unsafe {
-                        self.write_dump_file(
-                            self.mem_regions.clone(),
-                            self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
-                            self.size,
-                        )
-                    } {
-                        println!("Error dumping memory: {}", e);
-                    }
-                }
                 match self.get_exit_details(exit_context.ExitReason) {
                     Ok(error) => HyperlightExit::Unknown(error),
                     Err(e) => HyperlightExit::Unknown(format!("Error getting exit details: {}", e)),
@@ -569,6 +535,11 @@ impl Hypervisor for HypervWindowsDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 #[cfg(test)]

src/hyperlight_host/src/hypervisor/inprocess.rs

@@ -18,6 +18,8 @@ use std::fmt::Debug;
 use std::os::raw::c_void;
 
 use super::{HyperlightExit, Hypervisor};
+#[cfg(crashdump)]
+use crate::mem::memory_region::MemoryRegion;
 use crate::sandbox::leaked_outb::LeakedOutBWrapper;
 use crate::Result;
 
@@ -123,4 +125,9 @@ impl<'a> Hypervisor for InprocessDriver<'a> {
     fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE {
         unimplemented!("get_partition_handle should not be needed since we are in in-process mode")
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        unimplemented!("get_memory_regions is not supported since we are in in-process mode")
+    }
 }

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -17,7 +17,6 @@ limitations under the License.
 use std::convert::TryFrom;
 use std::fmt::Debug;
 
-use cfg_if::cfg_if;
 use kvm_bindings::{kvm_fpu, kvm_regs, kvm_userspace_memory_region, KVM_MEM_READONLY};
 use kvm_ioctls::Cap::UserMemory;
 use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd};
@@ -32,7 +31,7 @@ use super::{
 use crate::hypervisor::hypervisor_handler::HypervisorHandler;
 use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::mem::ptr::{GuestPtr, RawPtr};
-use crate::{debug, log_then_return, new_error, Result};
+use crate::{log_then_return, new_error, Result};
 
 /// Return `true` if the KVM API is available, version 12, and has UserMemory capability, or `false` otherwise
 #[instrument(skip_all, parent = Span::current(), level = "Trace")]
@@ -272,22 +271,20 @@ impl Hypervisor for KVMDriver {
         let exit_reason = self.vcpu_fd.run();
         let result = match exit_reason {
             Ok(VcpuExit::Hlt) => {
-                debug!("KVM - Halt Details : {:#?}", &self);
+                crate::debug!("KVM - Halt Details : {:#?}", &self);
                 HyperlightExit::Halt()
             }
             Ok(VcpuExit::IoOut(port, data)) => {
-                // because vcpufd.run() mutably borrows self we cannot pass self to debug! macro here
-                debug!("KVM IO Details : \nPort : {}\nData : {:?}", port, data);
+                // because vcpufd.run() mutably borrows self we cannot pass self to crate::debug! macro here
+                crate::debug!("KVM IO Details : \nPort : {}\nData : {:?}", port, data);
                 // KVM does not need to set RIP or instruction length so these are set to 0
                 HyperlightExit::IoOut(port, data.to_vec(), 0, 0)
             }
             Ok(VcpuExit::MmioRead(addr, _)) => {
-                debug!("KVM MMIO Read -Details: Address: {} \n {:#?}", addr, &self);
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                self.dump_on_crash(self.mem_regions.clone());
-                let gpa = addr as usize;
+                crate::debug!("KVM MMIO Read -Details: Address: {} \n {:#?}", addr, &self);
+
                 match self.get_memory_access_violation(
-                    gpa,
+                    addr as usize,
                     &self.mem_regions,
                     MemoryRegionFlags::READ,
                 ) {
@@ -296,12 +293,10 @@ impl Hypervisor for KVMDriver {
                 }
             }
             Ok(VcpuExit::MmioWrite(addr, _)) => {
-                debug!("KVM MMIO Write -Details: Address: {} \n {:#?}", addr, &self);
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                self.dump_on_crash(self.mem_regions.clone());
-                let gpa = addr as usize;
+                crate::debug!("KVM MMIO Write -Details: Address: {} \n {:#?}", addr, &self);
+
                 match self.get_memory_access_violation(
-                    gpa,
+                    addr as usize,
                     &self.mem_regions,
                     MemoryRegionFlags::WRITE,
                 ) {
@@ -314,26 +309,13 @@ impl Hypervisor for KVMDriver {
                 libc::EINTR => HyperlightExit::Cancelled(),
                 libc::EAGAIN => HyperlightExit::Retry(),
                 _ => {
-                    debug!("KVM Error -Details: Address: {} \n {:#?}", e, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
+                    crate::debug!("KVM Error -Details: Address: {} \n {:#?}", e, &self);
                     log_then_return!("Error running VCPU {:?}", e);
                 }
             },
             Ok(other) => {
-                cfg_if! {
-                    if #[cfg(all(feature = "print_debug", debug_assertions))] {
-                        let _ = other;
-                        debug!("KVM Other Exit: \n {:#?}", &self);
-                        HyperlightExit::Unknown("Unexpected KVM Exit".to_string())
-                    } else if #[cfg(all(feature = "dump_on_crash", debug_assertions))] {
-                            self.dump_on_crash(self.mem_regions.clone());
-                            HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other))
-                    } else{
-                        debug!("KVM Other Exit {:?}", other);
-                        HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other))
-                    }
-                }
+                crate::debug!("KVM Other Exit {:?}", other);
+                HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other))
             }
         };
         Ok(result)
@@ -343,6 +325,11 @@ impl Hypervisor for KVMDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 #[cfg(test)]