Fix cargo feature 'crashdump'

Signed-off-by: Ludvig Liljenberg <[email protected]>

Author: ludfjig

.github/workflows/dep_rust.yml

@@ -74,6 +74,9 @@ jobs:
           # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess
           just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv' || 'kvm' }} 
 
+          # make sure certain cargo features compile
+          cargo check -p hyperlight-host --features crashdump
+
           # without any driver (shouldn't compile)
           just test-rust-feature-compilation-fail ${{ matrix.config }}
 

docs/debugging-hyperlight.md

@@ -39,6 +39,6 @@ cargo test --package hyperlight-host --test integration_test --features print_de
 
 ## Dumping the memory configuration, virtual processor register state and memory contents on a crash or unexpected VM Exit
 
-To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `dump_on_crash` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.
+To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.
 
 There are no tools at this time to analyze the dump file, but it can be useful for debugging.

src/hyperlight_host/Cargo.toml

@@ -118,9 +118,7 @@ function_call_metrics = []
 executable_heap = []
 # This feature enables printing of debug information to stdout in debug builds
 print_debug = []
-# This feature enables dunping of the VMs details to a file when an unexpected or error exit occurs in a VM in debug mode
-# the name of the file is output to stdout and logged.
-dump_on_crash = ["dep:tempfile"]
+crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds.
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"]
 inprocess = []

src/hyperlight_host/build.rs

@@ -94,6 +94,8 @@ fn main() -> Result<()> {
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.
         // You should never use #[cfg(feature = "inprocess")] in the codebase. Use #[cfg(inprocess)] instead.
         inprocess: { all(feature = "inprocess", debug_assertions) },
+        // crashdump feature is aliased with debug_assertions to make it only available in debug-builds.
+        crashdump: { all(feature = "crashdump", debug_assertions) },
     }
 
     write_built_file()?;

src/hyperlight_host/src/hypervisor/crashdump.rs

@@ -0,0 +1,47 @@
+use std::io::Write;
+
+use tempfile::NamedTempFile;
+
+use super::Hypervisor;
+use crate::{new_error, Result};
+
+/// Dump registers + memory regions + raw memory to a tempfile
+#[cfg(crashdump)]
+pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
+    let mem_regions = hv.get_memory_regions();
+    let mem_size = mem_regions
+        .iter()
+        .map(|region| region.host_region.len())
+        .sum();
+    let mem_start_addr = mem_regions[0].host_region.start as *const u8;
+
+    if mem_start_addr.is_null() || mem_size == 0 {
+        return Err(new_error!(
+            "Invalid address or size while creating crashdump"
+        ));
+    }
+
+    let mut temp_file = NamedTempFile::with_prefix("mem")?;
+
+    let hv_details = format!("{:#x?}", hv);
+
+    // write hypervisor details such as registers, memory regions, etc.
+    temp_file.write_all(hv_details.as_bytes())?;
+    // write memory dump
+    temp_file.write_all(b"================ MEMORY DUMP =================\n")?;
+    // SAFETY: Address and size non-null and non-zero
+    unsafe {
+        let slice = std::slice::from_raw_parts(mem_start_addr, mem_size);
+        temp_file.write_all(slice)?;
+        temp_file.flush()?;
+    }
+    let persist_path = temp_file.path().with_extension("dmp");
+    temp_file
+        .persist(&persist_path)
+        .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?;
+
+    println!("Memory dumped to file: {:?}", persist_path);
+    log::error!("Memory dumped to file: {:?}", persist_path);
+
+    Ok(())
+}

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -311,8 +311,6 @@ impl Hypervisor for HypervLinuxDriver {
                         "mshv MMIO unmapped GPA -Details: Address: {} \n {:#?}",
                         addr, &self
                     );
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     HyperlightExit::Mmio(addr)
                 }
                 INVALID_GPA_ACCESS_MESSAGE => {
@@ -323,8 +321,6 @@ impl Hypervisor for HypervLinuxDriver {
                         "mshv MMIO invalid GPA access -Details: Address: {} \n {:#?}",
                         gpa, &self
                     );
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     match self.get_memory_access_violation(
                         gpa as usize,
                         &self.mem_regions,
@@ -336,8 +332,6 @@ impl Hypervisor for HypervLinuxDriver {
                 }
                 other => {
                     debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     log_then_return!("unknown Hyper-V run message type {:?}", other);
                 }
             },
@@ -347,8 +341,6 @@ impl Hypervisor for HypervLinuxDriver {
                 libc::EAGAIN => HyperlightExit::Retry(),
                 _ => {
                     debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     log_then_return!("Error running VCPU {:?}", e);
                 }
             },
@@ -360,6 +352,11 @@ impl Hypervisor for HypervLinuxDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 impl Drop for HypervLinuxDriver {

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -508,18 +508,6 @@ impl Hypervisor for HypervWindowsDriver {
                     "HyperV Memory Access Details :\n GPA: {:#?}\n Access Info :{:#?}\n {:#?} ",
                     gpa, access_info, &self
                 );
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                {
-                    if let Err(e) = unsafe {
-                        self.write_dump_file(
-                            self.mem_regions.clone(),
-                            self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
-                            self.size,
-                        )
-                    } {
-                        println!("Error dumping memory: {}", e);
-                    }
-                }
 
                 match self.get_memory_access_violation(gpa as usize, &self.mem_regions, access_info)
                 {
@@ -539,18 +527,6 @@ impl Hypervisor for HypervWindowsDriver {
                     "HyperV Unexpected Exit Details :#nReason {:#?}\n {:#?}",
                     exit_context.ExitReason, &self
                 );
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                {
-                    if let Err(e) = unsafe {
-                        self.write_dump_file(
-                            self.mem_regions.clone(),
-                            self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
-                            self.size,
-                        )
-                    } {
-                        println!("Error dumping memory: {}", e);
-                    }
-                }
                 match self.get_exit_details(exit_context.ExitReason) {
                     Ok(error) => HyperlightExit::Unknown(error),
                     Err(e) => HyperlightExit::Unknown(format!("Error getting exit details: {}", e)),
@@ -569,6 +545,11 @@ impl Hypervisor for HypervWindowsDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 #[cfg(test)]

src/hyperlight_host/src/hypervisor/inprocess.rs

@@ -18,6 +18,8 @@ use std::fmt::Debug;
 use std::os::raw::c_void;
 
 use super::{HyperlightExit, Hypervisor};
+#[cfg(crashdump)]
+use crate::mem::memory_region::MemoryRegion;
 use crate::sandbox::leaked_outb::LeakedOutBWrapper;
 use crate::Result;
 
@@ -123,4 +125,9 @@ impl<'a> Hypervisor for InprocessDriver<'a> {
     fn get_partition_handle(&self) -> windows::Win32::System::Hypervisor::WHV_PARTITION_HANDLE {
         unimplemented!("get_partition_handle should not be needed since we are in in-process mode")
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        unimplemented!("get_memory_regions is not supported since we are in in-process mode")
+    }
 }

src/hyperlight_host/src/hypervisor/kvm.rs

@@ -17,7 +17,6 @@ limitations under the License.
 use std::convert::TryFrom;
 use std::fmt::Debug;
 
-use cfg_if::cfg_if;
 use kvm_bindings::{kvm_fpu, kvm_regs, kvm_userspace_memory_region, KVM_MEM_READONLY};
 use kvm_ioctls::Cap::UserMemory;
 use kvm_ioctls::{Kvm, VcpuExit, VcpuFd, VmFd};
@@ -283,11 +282,9 @@ impl Hypervisor for KVMDriver {
             }
             Ok(VcpuExit::MmioRead(addr, _)) => {
                 debug!("KVM MMIO Read -Details: Address: {} \n {:#?}", addr, &self);
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                self.dump_on_crash(self.mem_regions.clone());
-                let gpa = addr as usize;
+
                 match self.get_memory_access_violation(
-                    gpa,
+                    addr as usize,
                     &self.mem_regions,
                     MemoryRegionFlags::READ,
                 ) {
@@ -297,11 +294,9 @@ impl Hypervisor for KVMDriver {
             }
             Ok(VcpuExit::MmioWrite(addr, _)) => {
                 debug!("KVM MMIO Write -Details: Address: {} \n {:#?}", addr, &self);
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                self.dump_on_crash(self.mem_regions.clone());
-                let gpa = addr as usize;
+
                 match self.get_memory_access_violation(
-                    gpa,
+                    addr as usize,
                     &self.mem_regions,
                     MemoryRegionFlags::WRITE,
                 ) {
@@ -315,25 +310,12 @@ impl Hypervisor for KVMDriver {
                 libc::EAGAIN => HyperlightExit::Retry(),
                 _ => {
                     debug!("KVM Error -Details: Address: {} \n {:#?}", e, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     log_then_return!("Error running VCPU {:?}", e);
                 }
             },
             Ok(other) => {
-                cfg_if! {
-                    if #[cfg(all(feature = "print_debug", debug_assertions))] {
-                        let _ = other;
-                        debug!("KVM Other Exit: \n {:#?}", &self);
-                        HyperlightExit::Unknown("Unexpected KVM Exit".to_string())
-                    } else if #[cfg(all(feature = "dump_on_crash", debug_assertions))] {
-                            self.dump_on_crash(self.mem_regions.clone());
-                            HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other))
-                    } else{
-                        debug!("KVM Other Exit {:?}", other);
-                        HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other))
-                    }
-                }
+                debug!("KVM Other Exit {:?}", other);
+                HyperlightExit::Unknown(format!("Unexpected KVM Exit {:?}", other))
             }
         };
         Ok(result)
@@ -343,6 +325,11 @@ impl Hypervisor for KVMDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 #[cfg(test)]