[hyperlight_host/trace] Support collecting guest stacktraces

Signed-off-by: Lucy Menon <[email protected]>
Signed-off-by: Doru Blânzeanu <[email protected]>

Author: syntactically

Cargo.lock

@@ -25,11 +25,12 @@ spin = "0.10.0"
 [features]
 default = ["tracing"]
 fuzzing = ["dep:arbitrary"]
+unwind_guest = []
 std = []
 
 [dev-dependencies]
 hyperlight-testing = { workspace = true }
 
 [lib]
 bench = false # see https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options
-doctest = false # reduce noise in test output
+doctest = false # reduce noise in test output

src/hyperlight_common/Cargo.toml

@@ -90,11 +90,14 @@ impl TryFrom<u8> for Exception {
 /// - CallFunction: makes a call to a host function,
 /// - Abort: aborts the execution of the guest,
 /// - DebugPrint: prints a message to the host
+/// - TraceRecordStack: records the stack trace of the guest
 pub enum OutBAction {
     Log = 99,
     CallFunction = 101,
     Abort = 102,
     DebugPrint = 103,
+    #[cfg(feature = "unwind_guest")]
+    TraceRecordStack = 104,
 }
 
 impl TryFrom<u16> for OutBAction {
@@ -105,6 +108,8 @@ impl TryFrom<u16> for OutBAction {
             101 => Ok(OutBAction::CallFunction),
             102 => Ok(OutBAction::Abort),
             103 => Ok(OutBAction::DebugPrint),
+            #[cfg(feature = "unwind_guest")]
+            104 => Ok(OutBAction::TraceRecordStack),
             _ => Err(anyhow::anyhow!("Invalid OutBAction value: {}", val)),
         }
     }

src/hyperlight_common/src/outb.rs

@@ -28,6 +28,8 @@ rand = { version = "0.9" }
 cfg-if = { version = "1.0.1" }
 libc = { version = "0.2.174" }
 flatbuffers = "25.2.10"
+framehop = { version = "0.13.1", optional = true }
+fallible-iterator = { version = "0.3.0", optional = true }
 page_size = "0.6.0"
 termcolor = "1.2.0"
 bitflags = "2.9.1"
@@ -45,6 +47,7 @@ metrics = "0.24.2"
 serde_json = "1.0"
 elfcore = "2.0"
 uuid = { version = "1.17.0", features = ["v4"] }
+blake3 = "1.8.2"
 
 [target.'cfg(windows)'.dependencies]
 windows = { version = "0.61", features = [
@@ -131,7 +134,7 @@ crashdump = ["dep:chrono"]
 trace_guest = []
 # This feature enables unwinding the guest stack from the host, in
 # order to produce stack traces for debugging or profiling.
-unwind_guest = [ "trace_guest" ]
+unwind_guest = [ "trace_guest", "dep:framehop", "dep:fallible-iterator", "hyperlight-common/unwind_guest" ]
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
 mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
 mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]

src/hyperlight_host/Cargo.toml

@@ -269,7 +269,7 @@ pub struct VirtualCPU {}
 impl VirtualCPU {
     /// Run the given hypervisor until a halt instruction is reached
     #[instrument(err(Debug), skip_all, parent = Span::current(), level = "Trace")]
-    pub fn run(
+    pub(crate) fn run(
         hv: &mut dyn Hypervisor,
         outb_handle_fn: Arc<Mutex<dyn OutBHandlerCaller>>,
         mem_access_fn: Arc<Mutex<dyn MemAccessHandlerCaller>>,

src/hyperlight_host/src/hypervisor/mod.rs

@@ -14,6 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+#[cfg(feature = "unwind_guest")]
+use std::sync::Arc;
+
 #[cfg(target_arch = "aarch64")]
 use goblin::elf::reloc::{R_AARCH64_NONE, R_AARCH64_RELATIVE};
 #[cfg(target_arch = "x86_64")]
@@ -26,13 +29,85 @@ use goblin::elf64::program_header::PT_LOAD;
 
 use crate::{Result, log_then_return, new_error};
 
+#[cfg(feature = "unwind_guest")]
+struct ResolvedSectionHeader {
+    name: String,
+    addr: u64,
+    offset: u64,
+    size: u64,
+}
+
 pub(crate) struct ElfInfo {
     payload: Vec<u8>,
     phdrs: ProgramHeaders,
+    #[cfg(feature = "unwind_guest")]
+    shdrs: Vec<ResolvedSectionHeader>,
     entry: u64,
     relocs: Vec<Reloc>,
 }
 
+#[cfg(feature = "unwind_guest")]
+struct UnwindInfo {
+    payload: Vec<u8>,
+    load_addr: u64,
+    va_size: u64,
+    base_svma: u64,
+    shdrs: Vec<ResolvedSectionHeader>,
+}
+
+#[cfg(feature = "unwind_guest")]
+impl super::exe::UnwindInfo for UnwindInfo {
+    fn as_module(&self) -> framehop::Module<Vec<u8>> {
+        framehop::Module::new(
+            // TODO: plumb through a name from from_file if this
+            // came from a file
+            "guest".to_string(),
+            self.load_addr..self.load_addr + self.va_size,
+            self.load_addr,
+            self,
+        )
+    }
+    fn hash(&self) -> blake3::Hash {
+        blake3::hash(&self.payload)
+    }
+}
+
+#[cfg(feature = "unwind_guest")]
+impl UnwindInfo {
+    fn resolved_section_header(&self, name: &[u8]) -> Option<&ResolvedSectionHeader> {
+        self.shdrs
+            .iter()
+            .find(|&sh| sh.name.as_bytes()[0..core::cmp::min(name.len(), sh.name.len())] == *name)
+    }
+}
+
+#[cfg(feature = "unwind_guest")]
+impl framehop::ModuleSectionInfo<Vec<u8>> for &UnwindInfo {
+    fn base_svma(&self) -> u64 {
+        self.base_svma
+    }
+    fn section_svma_range(&mut self, name: &[u8]) -> Option<std::ops::Range<u64>> {
+        let shdr = self.resolved_section_header(name)?;
+        Some(shdr.addr..shdr.addr + shdr.size)
+    }
+    fn section_data(&mut self, name: &[u8]) -> Option<Vec<u8>> {
+        if name == b".eh_frame" && self.resolved_section_header(b".debug_frame").is_some() {
+            /* Rustc does not always emit enough information for stack
+             * unwinding in .eh_frame, presumably because we use panic =
+             * abort in the guest. Framehop defaults to ignoring
+             * .debug_frame if .eh_frame exists, but we want the opposite
+             * behaviour here, since .debug_frame will actually contain
+             * frame information whereas .eh_frame often doesn't because
+             * of the aforementioned behaviour.  Consequently, we hack
+             * around this by pretending that .eh_frame doesn't exist if
+             * .debug_frame does. */
+            return None;
+        }
+        let shdr = self.resolved_section_header(name)?;
+        Some(self.payload[shdr.offset as usize..(shdr.offset + shdr.size) as usize].to_vec())
+    }
+}
+
 impl ElfInfo {
     pub(crate) fn new(bytes: &[u8]) -> Result<Self> {
         let elf = Elf::parse(bytes)?;
@@ -47,6 +122,19 @@ impl ElfInfo {
         Ok(ElfInfo {
             payload: bytes.to_vec(),
             phdrs: elf.program_headers,
+            #[cfg(feature = "unwind_guest")]
+            shdrs: elf
+                .section_headers
+                .iter()
+                .filter_map(|sh| {
+                    Some(ResolvedSectionHeader {
+                        name: elf.shdr_strtab.get_at(sh.sh_name)?.to_string(),
+                        addr: sh.sh_addr,
+                        offset: sh.sh_offset,
+                        size: sh.sh_size,
+                    })
+                })
+                .collect(),
             entry: elf.entry,
             relocs,
         })
@@ -73,7 +161,11 @@ impl ElfInfo {
             .unwrap();
         (max_phdr.p_vaddr + max_phdr.p_memsz - self.get_base_va()) as usize
     }
-    pub(crate) fn load_at(&mut self, load_addr: usize, target: &mut [u8]) -> Result<()> {
+    pub(crate) fn load_at(
+        self,
+        load_addr: usize,
+        target: &mut [u8],
+    ) -> Result<super::exe::LoadInfo> {
         let base_va = self.get_base_va();
         for phdr in self.phdrs.iter().filter(|phdr| phdr.p_type == PT_LOAD) {
             let start_va = (phdr.p_vaddr - base_va) as usize;
@@ -113,6 +205,20 @@ impl ElfInfo {
                 }
             }
         }
-        Ok(())
+        cfg_if::cfg_if! {
+            if #[cfg(feature = "unwind_guest")] {
+                let va_size = self.get_va_size() as u64;
+                let base_svma = self.get_base_va();
+                Ok(Arc::new(UnwindInfo {
+                    payload: self.payload,
+                    load_addr: load_addr as u64,
+                    va_size,
+                    base_svma,
+                    shdrs: self.shdrs,
+                }))
+            } else {
+                Ok(())
+            }
+        }
     }
 }

src/hyperlight_host/src/mem/elf.rs

@@ -16,6 +16,8 @@ limitations under the License.
 
 use std::fs::File;
 use std::io::Read;
+#[cfg(feature = "unwind_guest")]
+use std::sync::Arc;
 use std::vec::Vec;
 
 use super::elf::ElfInfo;
@@ -37,6 +39,41 @@ pub enum ExeInfo {
 const DEFAULT_ELF_STACK_RESERVE: u64 = 65536;
 const DEFAULT_ELF_HEAP_RESERVE: u64 = 131072;
 
+#[cfg(feature = "unwind_guest")]
+pub(crate) trait UnwindInfo: Send + Sync {
+    fn as_module(&self) -> framehop::Module<Vec<u8>>;
+    fn hash(&self) -> blake3::Hash;
+}
+
+#[cfg(feature = "unwind_guest")]
+pub(crate) struct DummyUnwindInfo {}
+#[cfg(feature = "unwind_guest")]
+impl UnwindInfo for DummyUnwindInfo {
+    fn as_module(&self) -> framehop::Module<Vec<u8>> {
+        framehop::Module::new("unsupported".to_string(), 0..0, 0, self)
+    }
+    fn hash(&self) -> blake3::Hash {
+        blake3::Hash::from_bytes([0; 32])
+    }
+}
+#[cfg(feature = "unwind_guest")]
+impl<A> framehop::ModuleSectionInfo<A> for &DummyUnwindInfo {
+    fn base_svma(&self) -> u64 {
+        0
+    }
+    fn section_svma_range(&mut self, _name: &[u8]) -> Option<std::ops::Range<u64>> {
+        None
+    }
+    fn section_data(&mut self, _name: &[u8]) -> Option<A> {
+        None
+    }
+}
+
+#[cfg(feature = "unwind_guest")]
+pub(crate) type LoadInfo = Arc<dyn UnwindInfo>;
+#[cfg(not(feature = "unwind_guest"))]
+pub(crate) type LoadInfo = ();
+
 impl ExeInfo {
     pub fn from_file(path: &str) -> Result<Self> {
         let mut file = File::open(path)?;
@@ -71,12 +108,9 @@ impl ExeInfo {
     // copying into target, but the PE loader chooses to apply
     // relocations in its owned representation of the PE contents,
     // which requires it to be &mut.
-    pub fn load(self, load_addr: usize, target: &mut [u8]) -> Result<()> {
+    pub fn load(self, load_addr: usize, target: &mut [u8]) -> Result<LoadInfo> {
         match self {
-            ExeInfo::Elf(mut elf) => {
-                elf.load_at(load_addr, target)?;
-            }
+            ExeInfo::Elf(elf) => elf.load_at(load_addr, target),
         }
-        Ok(())
     }
 }

src/hyperlight_host/src/mem/exe.rs

@@ -338,7 +338,7 @@ impl SandboxMemoryManager<ExclusiveSharedMemory> {
         cfg: SandboxConfiguration,
         exe_info: ExeInfo,
         guest_blob: Option<&GuestBlob>,
-    ) -> Result<Self> {
+    ) -> Result<(Self, super::exe::LoadInfo)> {
         let guest_blob_size = guest_blob.map(|b| b.data.len()).unwrap_or(0);
         let guest_blob_mem_flags = guest_blob.map(|b| b.permissions);
 
@@ -364,18 +364,21 @@ impl SandboxMemoryManager<ExclusiveSharedMemory> {
             shared_mem.write_u64(offset, load_addr_u64)?;
         }
 
-        exe_info.load(
+        let load_info = exe_info.load(
             load_addr.clone().try_into()?,
             &mut shared_mem.as_mut_slice()[layout.get_guest_code_offset()..],
         )?;
 
-        Ok(Self::new(
-            layout,
-            shared_mem,
-            load_addr,
-            entrypoint_offset,
-            #[cfg(target_os = "windows")]
-            None,
+        Ok((
+            Self::new(
+                layout,
+                shared_mem,
+                load_addr,
+                entrypoint_offset,
+                #[cfg(target_os = "windows")]
+                None,
+            ),
+            load_info,
         ))
     }