Fix cargo feature 'crashdump'

Signed-off-by: Ludvig Liljenberg <[email protected]>

Author: ludfjig

.github/workflows/Benchmarks.yml

@@ -17,33 +17,12 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        include:
-          - build: windows-2022-release-amd
-            os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-amd"]
-            hypervisor: hyperv
-            arch: amd
-          - build: linux-kvm-release-amd
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd"]
-            hypervisor: kvm
-            arch: amd
-          - build: linux-mshv-release-amd
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-amd"]
-            hypervisor: mshv
-            arch: amd
-          - build: windows-2022-release-intel
-            os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-intel"]
-            hypervisor: hyperv
-            arch: intel
-          - build: linux-kvm-release-intel
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-intel"]
-            hypervisor: kvm
-            arch: intel
-          - build: linux-mshv-release-intel
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-intel"]
-            hypervisor: mshv
-            arch: intel
+        hypervisor: [hyperv, mshv, kvm] # hyperv is windows, mshv and kvm are linux
+        cpu: [amd, intel]
+        config: [release] # don't want to benchmark debug-builds
 
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor, matrix.cpu)) }} 
+    
     steps:
       ### Setup ###
 
@@ -82,7 +61,7 @@ jobs:
         run: git fetch --tags origin
 
       - name: Download benchmarks from most recent release
-        run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.arch }} # skip tag parameter to compare to latest stable release
+        run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.cpu }} # skip tag parameter to compare to latest stable release
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -92,6 +71,6 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: benchmarks_${{runner.os}}_${{matrix.hypervisor}}_${{ matrix.arch }}
+          name: benchmarks_${{runner.os}}_${{matrix.hypervisor}}_${{ matrix.cpu }}
           path: ./target/criterion/
           if-no-files-found: error

.github/workflows/ValidatePullRequest.yml

@@ -7,6 +7,12 @@ on:
     branches: [main, "release/**"]
   merge_group:
 
+# Cancels old running job if a new one is triggered (e.g. by a push onto the same branch).
+# This will cancel dependent jobs as well, such as dep_rust and dep_fuzzing
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true    
+
 permissions:
   id-token: write
   contents: read
@@ -42,6 +48,7 @@ jobs:
     secrets: inherit
     with: 
       docs_only: ${{needs.docs-pr.outputs.docs-only}}
+
   fuzzing:
     needs:
       - docs-pr
@@ -50,6 +57,7 @@ jobs:
       max_total_time: 300 # 5 minutes in seconds
       docs_only: ${{needs.docs-pr.outputs.docs-only}}
     secrets: inherit
+
   spelling:
     name: spell check with typos
     runs-on: ubuntu-latest

.github/workflows/dep_rust.yml

@@ -26,69 +26,11 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        include:
-          - build: windows-2022-debug-amd
-            os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-amd"]
-            hypervisor: hyperv
-            config: debug
-            arch: amd
-          - build: linux-kvm-debug-amd
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd"]
-            hypervisor: kvm
-            config: debug
-            arch: amd
-          - build: linux-mshv-debug-amd
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-amd"]
-            hypervisor: mshv
-            config: debug
-            arch: amd
-          - build: windows-2022-release-amd
-            os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-amd"]
-            hypervisor: hyperv
-            config: release
-            arch: amd
-          - build: linux-kvm-release-amd
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-amd"]
-            hypervisor: kvm
-            config: release
-            arch: amd
-          - build: linux-mshv-release-amd
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-amd"]
-            hypervisor: mshv
-            config: release
-            arch: amd
-          - build: windows-2022-debug-intel
-            os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-intel"]
-            hypervisor: hyperv
-            config: debug
-            arch: intel
-          - build: linux-kvm-debug-intel
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-intel"]
-            hypervisor: kvm
-            config: debug
-            arch: intel
-          - build: linux-mshv-debug-intel
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-intel"]
-            hypervisor: mshv
-            config: debug
-            arch: intel
-          - build: windows-2022-release-intel
-            os: [self-hosted, Windows, X64, "1ES.Pool=hld-win2022-intel"]
-            hypervisor: hyperv
-            config: release
-            arch: intel
-          - build: linux-kvm-release-intel
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-kvm-intel"]
-            hypervisor: kvm
-            config: release
-            arch: intel
-          - build: linux-mshv-release-intel
-            os: [self-hosted, Linux, X64, "1ES.Pool=hld-mshv-intel"]
-            hypervisor: mshv
-            config: release
-            arch: intel
-
-    runs-on: ${{ matrix.os }}
+        hypervisor: [hyperv, mshv, kvm] # hyperv is windows, mshv and kvm are linux
+        cpu: [amd, intel]
+        config: [debug, release]
+
+    runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor, matrix.cpu)) }} 
     steps:
       - uses: actions/checkout@v4
 
@@ -132,6 +74,9 @@ jobs:
           # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess
           just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv' || 'kvm' }} 
 
+          # make sure certain cargo features compile
+          cargo check -p hyperlight-host --features crashdump
+
           # without any driver (shouldn't compile)
           just test-rust-feature-compilation-fail ${{ matrix.config }}
 
@@ -160,7 +105,7 @@ jobs:
         run: sudo apt install gh -y
 
       - name: Download benchmarks from "latest"
-        run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.arch}} dev-latest # compare to prerelease
+        run: just bench-download ${{ runner.os }} ${{ matrix.hypervisor }} ${{ matrix.cpu}} dev-latest # compare to prerelease
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         continue-on-error: true

Justfile

@@ -165,13 +165,13 @@ run-rust-examples-linux target=default-target: (build-rust target) (run-rust-exa
 # Downloads the benchmarks result from the given release tag.
 # If tag is not given, defaults to latest release
 # Options for os: "Windows", or "Linux"
-# Options for Linux hypervisor: "kvm", "hyperv"
+# Options for Linux hypervisor: "kvm", "mshv"
 # Options for Windows hypervisor: "hyperv"
-# Options for arch: "amd", "intel"
-bench-download os hypervisor arch tag="":
-    gh release download {{ tag }} -D ./target/ -p benchmarks_{{ os }}_{{ hypervisor }}_{{ arch }}.tar.gz
+# Options for cpu: "amd", "intel"
+bench-download os hypervisor cpu tag="":
+    gh release download {{ tag }} -D ./target/ -p benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz
     mkdir -p target/criterion {{ if os() == "windows" { "-Force" } else { "" } }}
-    tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ arch }}.tar.gz -C target/criterion/ --strip-components=1
+    tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1
 
 # Warning: compares to and then OVERWRITES the given baseline
 bench-ci baseline target=default-target:

docs/debugging-hyperlight.md

@@ -39,6 +39,6 @@ cargo test --package hyperlight-host --test integration_test --features print_de
 
 ## Dumping the memory configuration, virtual processor register state and memory contents on a crash or unexpected VM Exit
 
-To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `dump_on_crash` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.
+To dump the details of the memory configuration, the virtual processors register state and the contents of the VM memory set the feature `crashdump` and run a debug build. This will result in a dump file being created in the temporary directory. The name and location of the dump file will be printed to the console and logged as an error message.
 
 There are no tools at this time to analyze the dump file, but it can be useful for debugging.

src/hyperlight_host/Cargo.toml

@@ -118,9 +118,7 @@ function_call_metrics = []
 executable_heap = []
 # This feature enables printing of debug information to stdout in debug builds
 print_debug = []
-# This feature enables dunping of the VMs details to a file when an unexpected or error exit occurs in a VM in debug mode
-# the name of the file is output to stdout and logged.
-dump_on_crash = ["dep:tempfile"]
+crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds.
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"]
 inprocess = []

src/hyperlight_host/build.rs

@@ -94,6 +94,8 @@ fn main() -> Result<()> {
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.
         // You should never use #[cfg(feature = "inprocess")] in the codebase. Use #[cfg(inprocess)] instead.
         inprocess: { all(feature = "inprocess", debug_assertions) },
+        // crashdump feature is aliased with debug_assertions to make it only available in debug-builds.
+        crashdump: { all(feature = "crashdump", debug_assertions) },
     }
 
     write_built_file()?;

src/hyperlight_host/src/hypervisor/crashdump.rs

@@ -0,0 +1,47 @@
+use std::io::Write;
+
+use tempfile::NamedTempFile;
+
+use super::Hypervisor;
+use crate::{new_error, Result};
+
+/// Dump registers + memory regions + raw memory to a tempfile
+#[cfg(crashdump)]
+pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
+    let mem_regions = hv.get_memory_regions();
+    let mem_size = mem_regions
+        .iter()
+        .map(|region| region.host_region.len())
+        .sum();
+    let mem_start_addr = mem_regions[0].host_region.start as *const u8;
+
+    if mem_start_addr.is_null() || mem_size == 0 {
+        return Err(new_error!(
+            "Invalid address or size while creating crashdump"
+        ));
+    }
+
+    let mut temp_file = NamedTempFile::with_prefix("mem")?;
+
+    let hv_details = format!("{:#x?}", hv);
+
+    // write hypervisor details such as registers, memory regions, etc.
+    temp_file.write_all(hv_details.as_bytes())?;
+    // write memory dump
+    temp_file.write_all(b"================ MEMORY DUMP =================\n")?;
+    // SAFETY: Address and size non-null and non-zero
+    unsafe {
+        let slice = std::slice::from_raw_parts(mem_start_addr, mem_size);
+        temp_file.write_all(slice)?;
+        temp_file.flush()?;
+    }
+    let persist_path = temp_file.path().with_extension("dmp");
+    temp_file
+        .persist(&persist_path)
+        .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?;
+
+    println!("Memory dumped to file: {:?}", persist_path);
+    log::error!("Memory dumped to file: {:?}", persist_path);
+
+    Ok(())
+}

src/hyperlight_host/src/hypervisor/hyperv_linux.rs

@@ -311,8 +311,6 @@ impl Hypervisor for HypervLinuxDriver {
                         "mshv MMIO unmapped GPA -Details: Address: {} \n {:#?}",
                         addr, &self
                     );
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     HyperlightExit::Mmio(addr)
                 }
                 INVALID_GPA_ACCESS_MESSAGE => {
@@ -323,8 +321,6 @@ impl Hypervisor for HypervLinuxDriver {
                         "mshv MMIO invalid GPA access -Details: Address: {} \n {:#?}",
                         gpa, &self
                     );
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     match self.get_memory_access_violation(
                         gpa as usize,
                         &self.mem_regions,
@@ -336,8 +332,6 @@ impl Hypervisor for HypervLinuxDriver {
                 }
                 other => {
                     debug!("mshv Other Exit: Exit: {:#?} \n {:#?}", other, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     log_then_return!("unknown Hyper-V run message type {:?}", other);
                 }
             },
@@ -347,8 +341,6 @@ impl Hypervisor for HypervLinuxDriver {
                 libc::EAGAIN => HyperlightExit::Retry(),
                 _ => {
                     debug!("mshv Error - Details: Error: {} \n {:#?}", e, &self);
-                    #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                    self.dump_on_crash(self.mem_regions.clone());
                     log_then_return!("Error running VCPU {:?}", e);
                 }
             },
@@ -360,6 +352,11 @@ impl Hypervisor for HypervLinuxDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 impl Drop for HypervLinuxDriver {

src/hyperlight_host/src/hypervisor/hyperv_windows.rs

@@ -508,18 +508,6 @@ impl Hypervisor for HypervWindowsDriver {
                     "HyperV Memory Access Details :\n GPA: {:#?}\n Access Info :{:#?}\n {:#?} ",
                     gpa, access_info, &self
                 );
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                {
-                    if let Err(e) = unsafe {
-                        self.write_dump_file(
-                            self.mem_regions.clone(),
-                            self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
-                            self.size,
-                        )
-                    } {
-                        println!("Error dumping memory: {}", e);
-                    }
-                }
 
                 match self.get_memory_access_violation(gpa as usize, &self.mem_regions, access_info)
                 {
@@ -539,18 +527,6 @@ impl Hypervisor for HypervWindowsDriver {
                     "HyperV Unexpected Exit Details :#nReason {:#?}\n {:#?}",
                     exit_context.ExitReason, &self
                 );
-                #[cfg(all(debug_assertions, feature = "dump_on_crash"))]
-                {
-                    if let Err(e) = unsafe {
-                        self.write_dump_file(
-                            self.mem_regions.clone(),
-                            self.source_address.add(PAGE_SIZE_USIZE) as *const u8,
-                            self.size,
-                        )
-                    } {
-                        println!("Error dumping memory: {}", e);
-                    }
-                }
                 match self.get_exit_details(exit_context.ExitReason) {
                     Ok(error) => HyperlightExit::Unknown(error),
                     Err(e) => HyperlightExit::Unknown(format!("Error getting exit details: {}", e)),
@@ -569,6 +545,11 @@ impl Hypervisor for HypervWindowsDriver {
     fn as_mut_hypervisor(&mut self) -> &mut dyn Hypervisor {
         self as &mut dyn Hypervisor
     }
+
+    #[cfg(crashdump)]
+    fn get_memory_regions(&self) -> &[MemoryRegion] {
+        &self.mem_regions
+    }
 }
 
 #[cfg(test)]