crashdump: create core dump file when a guest crashes

- the core dump file is an ELF file with special segments that describe
the guest's memory when it crashed, the CPU register's values and other
special notes that tell the debugger how to set up a debugging session
starting from the core dump

Signed-off-by: Doru Blânzeanu <[email protected]>

Author: dblnz

Cargo.lock

@@ -46,6 +46,7 @@ tempfile = { version = "3.19", optional = true }
 serde_yaml = "0.9"
 anyhow = "1.0"
 metrics = "0.24.2"
+elfcore = { git = "https://github.com/dblnz/elfcore.git", branch = "split-linux-impl-from-elfcore" }
 
 [target.'cfg(windows)'.dependencies]
 windows = { version = "0.61", features = [
@@ -118,7 +119,7 @@ cfg_aliases = "0.2.1"
 built = { version = "0.7.7", features = ["chrono", "git2"] }
 
 [features]
-default = ["kvm", "mshv2", "seccomp"]
+default = ["crashdump", "kvm", "mshv2", "seccomp"]
 seccomp = ["dep:seccompiler"]
 function_call_metrics = []
 executable_heap = []

src/hyperlight_host/Cargo.toml

@@ -1,44 +1,240 @@
-use std::io::Write;
+use std::cmp::min;
 
+use elfcore::{
+    ArchComponentState, ArchState, CoreDumpBuilder, CoreError, Elf64_Auxv, ProcessInfoSource,
+    ReadProcessMemory, ThreadView, VaProtection, VaRegion,
+};
 use tempfile::NamedTempFile;
 
 use super::Hypervisor;
+use crate::mem::memory_region::{MemoryRegion, MemoryRegionFlags};
 use crate::{new_error, Result};
 
-/// Dump registers + memory regions + raw memory to a tempfile
-#[cfg(crashdump)]
-pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
-    let mut temp_file = NamedTempFile::with_prefix("mem")?;
-    let hv_details = format!("{:#x?}", hv);
+// amd64 notes
+pub const NT_X86_XSTATE: u32 = 0x202;
 
-    // write hypervisor details such as registers, info about mapped memory regions, etc.
-    temp_file.write_all(hv_details.as_bytes())?;
-    temp_file.write_all(b"================ MEMORY DUMP =================\n")?;
+#[derive(Debug)]
+pub(crate) struct CrashDumpContext<'a> {
+    regions: &'a [MemoryRegion],
+    regs: [u64; 27],
+    xsave: Vec<u8>,
+    entry: u64,
+}
 
-    // write the raw memory dump for each memory region
-    for region in hv.get_memory_regions() {
-        if region.host_region.start == 0 || region.host_region.is_empty() {
-            continue;
+impl<'a> CrashDumpContext<'a> {
+    pub(crate) fn new(
+        regions: &'a [MemoryRegion],
+        regs: [u64; 27],
+        xsave: Vec<u8>,
+        entry: u64,
+    ) -> Self {
+        Self {
+            regions,
+            regs,
+            xsave,
+            entry,
         }
-        // SAFETY: we got this memory region from the hypervisor so should never be invalid
-        let region_slice = unsafe {
-            std::slice::from_raw_parts(
-                region.host_region.start as *const u8,
-                region.host_region.len(),
-            )
+    }
+}
+
+struct GuestView {
+    regions: Vec<VaRegion>,
+    threads: Vec<ThreadView>,
+    aux_vector: Vec<elfcore::Elf64_Auxv>,
+}
+
+impl GuestView {
+    fn new(ctx: &CrashDumpContext) -> Self {
+        let regions = ctx
+            .regions
+            .iter()
+            .filter(|r| !r.host_region.is_empty())
+            .map(|r| VaRegion {
+                begin: r.guest_region.start as u64,
+                end: r.guest_region.end as u64,
+                offset: r.host_region.start as u64,
+                protection: VaProtection {
+                    is_private: false,
+                    read: r.flags.contains(MemoryRegionFlags::READ),
+                    write: r.flags.contains(MemoryRegionFlags::WRITE),
+                    execute: r.flags.contains(MemoryRegionFlags::EXECUTE),
+                },
+                mapped_file_name: None,
+            })
+            .collect();
+
+        let mut components = vec![];
+        if !ctx.xsave.is_empty() {
+            components.push(ArchComponentState {
+                name: "XSAVE",
+                note_type: NT_X86_XSTATE,
+                note_name: b"LINUX",
+                data: ctx.xsave.clone(),
+            });
+        }
+        let thread = ThreadView {
+            flags: 0, // Kernel flags for the process
+            tid: 1,
+            uid: 0, // User ID
+            gid: 0, // Group ID
+            comm: "\0".to_string(),
+            ppid: 0,    // Parent PID
+            pgrp: 0,    // Process group ID
+            nice: 0,    // Nice value
+            state: 0,   // Process state
+            utime: 0,   // User time
+            stime: 0,   // System time
+            cutime: 0,  // Children User time
+            cstime: 0,  // Children User time
+            cursig: 0,  // Current signal
+            session: 0, // Session ID of the process
+            sighold: 0, // Blocked signal
+            sigpend: 0, // Pending signal
+            cmd_line: "\0".to_string(),
+
+            arch_state: Box::new(ArchState {
+                gpr_state: ctx.regs.to_vec(),
+                components,
+            }),
         };
-        temp_file.write_all(region_slice)?;
+
+        // Create the auxv vector
+        // The first entry is AT_ENTRY, which is the entry point of the program
+        // The entry point is the address where the program starts executing
+        // This helps the debugger to know that the entry is changed by an offset
+        // so the symbols can be loaded correctly.
+        // The second entry is AT_NULL, which marks the end of the vector
+        let auxv = vec![
+            Elf64_Auxv {
+                a_type: 9, // AT_ENTRY
+                a_val: ctx.entry,
+            },
+            Elf64_Auxv {
+                a_type: 0, // AT_NULL
+                a_val: 0,
+            },
+        ];
+
+        Self {
+            regions,
+            threads: vec![thread],
+            aux_vector: auxv,
+        }
+    }
+}
+
+impl ProcessInfoSource for GuestView {
+    fn pid(&self) -> i32 {
+        1
+    }
+    fn threads(&self) -> &[elfcore::ThreadView] {
+        &self.threads
+    }
+    fn page_size(&self) -> usize {
+        0x1000
+    }
+    fn aux_vector(&self) -> Option<&[elfcore::Elf64_Auxv]> {
+        Some(&self.aux_vector)
+    }
+    fn va_regions(&self) -> &[elfcore::VaRegion] {
+        &self.regions
+    }
+    fn mapped_files(&self) -> Option<&[elfcore::MappedFile]> {
+        None
     }
-    temp_file.flush()?;
+}
+
+struct GuestMemReader {
+    regions: Vec<MemoryRegion>,
+}
+
+impl GuestMemReader {
+    fn new(ctx: &CrashDumpContext) -> Self {
+        Self {
+            regions: ctx.regions.to_vec(),
+        }
+    }
+}
+
+impl ReadProcessMemory for GuestMemReader {
+    fn read_process_memory(
+        &mut self,
+        base: usize,
+        buf: &mut [u8],
+    ) -> std::result::Result<usize, CoreError> {
+        for r in self.regions.iter() {
+            if base >= r.guest_region.start && base < r.guest_region.end {
+                let offset = base - r.guest_region.start;
+                let region_slice = unsafe {
+                    std::slice::from_raw_parts(
+                        r.host_region.start as *const u8,
+                        r.host_region.len(),
+                    )
+                };
+
+                // Calculate how much we can copy
+                let copy_size = min(buf.len(), region_slice.len() - offset);
+                if copy_size == 0 {
+                    return std::result::Result::Ok(0);
+                }
+
+                // Only copy the amount that fits in both buffers
+                buf[..copy_size].copy_from_slice(&region_slice[offset..offset + copy_size]);
+                return std::result::Result::Ok(copy_size);
+            }
+        }
+
+        // If we reach here, we didn't find a matching region
+        std::result::Result::Ok(0)
+    }
+}
+
+/// Create core dump file from the hypervisor information
+///
+/// This function generates an ELF core dump file capturing the hypervisor's state,
+/// which can be used for debugging when crashes occur. The file is created in the
+/// system's temporary directory with extension '.dmp' and the path is printed to stdout and logs.
+///
+/// # Arguments
+/// * `hv`: Reference to the hypervisor implementation
+///
+/// # Returns
+/// * `Result<()>`: Success or error
+pub(crate) fn crashdump_to_tempfile(hv: &dyn Hypervisor) -> Result<()> {
+    log::info!("Creating core dump file...");
+
+    // Create a temporary file with a recognizable prefix
+    let temp_file = NamedTempFile::with_prefix("hl_core_")
+        .map_err(|e| new_error!("Failed to create temporary file: {:?}", e))?;
+
+    // Get crash context from hypervisor
+    let ctx = hv
+        .crashdump_context()
+        .map_err(|e| new_error!("Failed to get crashdump context: {:?}", e))?;
+
+    // Set up data sources for the core dump
+    let guest_view = GuestView::new(&ctx);
+    let memory_reader = GuestMemReader::new(&ctx);
+
+    // Create and write core dump
+    let core_builder = CoreDumpBuilder::from_source(
+        Box::new(guest_view) as Box<dyn ProcessInfoSource>,
+        Box::new(memory_reader) as Box<dyn ReadProcessMemory>,
+    );
+
+    core_builder
+        .write(&temp_file)
+        .map_err(|e| new_error!("Failed to write core dump: {:?}", e))?;
 
-    // persist the tempfile to disk
+    // Persist the file with a .dmp extension
     let persist_path = temp_file.path().with_extension("dmp");
     temp_file
         .persist(&persist_path)
-        .map_err(|e| new_error!("Failed to persist crashdump file: {:?}", e))?;
+        .map_err(|e| new_error!("Failed to persist core dump file: {:?}", e))?;
 
-    println!("Memory dumped to file: {:?}", persist_path);
-    log::error!("Memory dumped to file: {:?}", persist_path);
+    let path_string = persist_path.to_string_lossy().to_string();
+    println!("Core dump created successfully: {}", path_string);
+    log::error!("Core dump file: {}", path_string);
 
     Ok(())
 }

src/hyperlight_host/src/hypervisor/crashdump.rs

@@ -49,6 +49,8 @@ use mshv_bindings::{
 use mshv_ioctls::{Mshv, VcpuFd, VmFd};
 use tracing::{instrument, Span};
 
+#[cfg(crashdump)]
+use super::crashdump;
 use super::fpu::{FP_CONTROL_WORD_DEFAULT, FP_TAG_WORD_DEFAULT, MXCSR_DEFAULT};
 #[cfg(gdb)]
 use super::gdb::{DebugCommChannel, DebugMsg, DebugResponse, GuestDebug, MshvDebug};
@@ -658,8 +660,47 @@ impl Hypervisor for HypervLinuxDriver {
     }
 
     #[cfg(crashdump)]
-    fn get_memory_regions(&self) -> &[MemoryRegion] {
-        &self.mem_regions
+    fn crashdump_context(&self) -> Result<super::crashdump::CrashDumpContext> {
+        let mut regs = [0; 27];
+
+        let vcpu_regs = self.vcpu_fd.get_regs()?;
+        let sregs = self.vcpu_fd.get_sregs()?;
+        let xsave = self.vcpu_fd.get_xsave()?;
+
+        regs[0] = vcpu_regs.r15; // r15
+        regs[1] = vcpu_regs.r14; // r14
+        regs[2] = vcpu_regs.r13; // r13
+        regs[3] = vcpu_regs.r12; // r12
+        regs[4] = vcpu_regs.rbp; // rbp
+        regs[5] = vcpu_regs.rbx; // rbx
+        regs[6] = vcpu_regs.r11; // r11
+        regs[7] = vcpu_regs.r10; // r10
+        regs[8] = vcpu_regs.r9; // r9
+        regs[9] = vcpu_regs.r8; // r8
+        regs[10] = vcpu_regs.rax; // rax
+        regs[11] = vcpu_regs.rcx; // rcx
+        regs[12] = vcpu_regs.rdx; // rdx
+        regs[13] = vcpu_regs.rsi; // rsi
+        regs[14] = vcpu_regs.rdi; // rdi
+        regs[15] = 0; // orig rax
+        regs[16] = vcpu_regs.rip; // rip
+        regs[17] = sregs.cs.selector as u64; // cs
+        regs[18] = vcpu_regs.rflags; // eflags
+        regs[19] = vcpu_regs.rsp; // rsp
+        regs[20] = sregs.ss.selector as u64; // ss
+        regs[21] = sregs.fs.base; // fs_base
+        regs[22] = sregs.gs.base; // gs_base
+        regs[23] = sregs.ds.selector as u64; // ds
+        regs[24] = sregs.es.selector as u64; // es
+        regs[25] = sregs.fs.selector as u64; // fs
+        regs[26] = sregs.gs.selector as u64; // gs
+
+        Ok(crashdump::CrashDumpContext::new(
+            &self.mem_regions,
+            regs,
+            xsave.buffer.to_vec(),
+            self.entrypoint,
+        ))
     }
 
     #[cfg(gdb)]