Wasmtime: implement debug instrumentation and basic host API to examine runtime state.

This PR implements ideas from the [recent RFC] to serve as the basis
for Wasm (guest) debugging: it adds a stackslot to each function
translated from Wasm, stores to replicate Wasm VM state in the
stackslot as the program runs, and metadata to describe the format of
that state and allow reading it out at runtime.

As an initial user of this state, this PR adds a basic "stack view"
API that, from host code that has been called from Wasm, can examine
Wasm frames currently on the stack and read out all of their locals
and stack slots.

Note in particular that this PR does not include breakpoints,
watchpoints, stepped execution, or any sort of user interface for any
of this; it is only a foundation.

This PR still has a few unsatisfying bits that I intend to address:

- The "stack view" performs some O(n) work when the view is initially
  taken, computing some internal data per frame. This is forced by the
  current design of `Backtrace`, which takes a closure and walks that
  closure over stack frames eagerly (rather than work as an
  iterator). It's got some impressive iterator-chain stuff going on
  internally, so refactoring it to the latter approach might not
  be *too* bad, but I haven't tackled it yet.

  A O(1) stack view, that is, one that does work only for frames as
  the host API is used to walk up the stack, is desirable because some
  use-cases may want to quickly examine e.g. only the deepest
  frame (say, running with a breakpoint condition that needs to read a
  particular local's value after each step).

- It includes a new `Config::compiler_force_inlining()` option that is
  used only for testing that we get the correct frames after
  inlining. I couldn't get the existing flags to work on a Wasmtime
  config level and suspect there may be an existing bug there; I will
  try to split out a fix for it.

This PR renames the existing `debug` option to `native_debug`, to
distinguish it from the new approach.

[recent RFC]: bytecodealliance/rfcs#44

Author: cfallin

.github/workflows/main.yml

@@ -918,7 +918,7 @@ jobs:
         sudo mkdir -p /usr/lib/local/lib/python3.10/dist-packages/lldb
         sudo ln -s /usr/lib/llvm-15/lib/python3.10/dist-packages/lldb/* /usr/lib/python3/dist-packages/lldb/
         # Only testing release since it is more likely to expose issues with our low-level symbol handling.
-        cargo test --release --test all -- --ignored --test-threads 1 debug::
+        cargo test --release --test all -- --ignored --test-threads 1 native_debug::
       env:
         LLDB: lldb-18
         WASI_SDK_PATH: /tmp/wasi-sdk

Cargo.toml

@@ -469,6 +469,7 @@ default = [
   "stack-switching",
   "winch",
   "pulley",
+  "debug",
 
   # Enable some nice features of clap by default, but they come at a binary size
   # cost, so allow disabling this through disabling of our own `default`
@@ -531,6 +532,7 @@ gc-drc = ["gc", "wasmtime/gc-drc", "wasmtime-cli-flags/gc-drc"]
 gc-null = ["gc", "wasmtime/gc-null", "wasmtime-cli-flags/gc-null"]
 pulley = ["wasmtime-cli-flags/pulley"]
 stack-switching = ["wasmtime/stack-switching", "wasmtime-cli-flags/stack-switching"]
+debug = ["wasmtime-cli-flags/debug", "wasmtime/debug"]
 
 # CLI subcommands for the `wasmtime` executable. See `wasmtime $cmd --help`
 # for more information on each subcommand.

crates/c-api/src/config.rs

@@ -55,7 +55,7 @@ pub extern "C" fn wasm_config_new() -> Box<wasm_config_t> {
 
 #[unsafe(no_mangle)]
 pub extern "C" fn wasmtime_config_debug_info_set(c: &mut wasm_config_t, enable: bool) {
-    c.config.debug_info(enable);
+    c.config.native_debug_info(enable);
 }
 
 #[unsafe(no_mangle)]

crates/cli-flags/Cargo.toml

@@ -40,3 +40,4 @@ threads = ["wasmtime/threads"]
 memory-protection-keys = ["wasmtime/memory-protection-keys"]
 pulley = ["wasmtime/pulley"]
 stack-switching = ["wasmtime/stack-switching"]
+debug = ["wasmtime/debug"]

crates/cli-flags/src/lib.rs

@@ -263,7 +263,9 @@ wasmtime_option_group! {
     #[serde(rename_all = "kebab-case", deny_unknown_fields)]
     pub struct DebugOptions {
         /// Enable generation of DWARF debug information in compiled code.
-        pub debug_info: Option<bool>,
+        pub native_debug_info: Option<bool>,
+        /// Enable debug instrumentation for perfect value reconstruction.
+        pub debug_instrumentation: Option<bool>,
         /// Configure whether compiled code can map native addresses to wasm.
         pub address_map: Option<bool>,
         /// Configure whether logging is enabled.
@@ -701,8 +703,13 @@ impl CommonOptions {
             enable => config.cranelift_debug_verifier(enable),
             true => err,
         }
-        if let Some(enable) = self.debug.debug_info {
-            config.debug_info(enable);
+        if let Some(enable) = self.debug.native_debug_info {
+            config.native_debug_info(enable);
+        }
+        match_feature! {
+            ["debug" : self.debug.debug_instrumentation]
+            enable => config.debug_instrumentation(enable),
+            _ => err,
         }
         if self.debug.coredump.is_some() {
             #[cfg(feature = "coredump")]

crates/cranelift/src/compiled_function.rs

@@ -1,7 +1,7 @@
 use crate::{Relocation, mach_reloc_to_reloc, mach_trap_to_trap};
 use cranelift_codegen::{
-    Final, MachBufferFinalized, MachSrcLoc, ValueLabelsRanges, ir, isa::unwind::CfaUnwindInfo,
-    isa::unwind::UnwindInfo,
+    Final, MachBufferFinalized, MachBufferFrameLayout, MachSrcLoc, ValueLabelsRanges, ir,
+    isa::unwind::CfaUnwindInfo, isa::unwind::UnwindInfo,
 };
 use wasmtime_environ::{FilePos, InstructionAddressMap, PrimaryMap, TrapInformation};
 
@@ -44,8 +44,6 @@ pub struct CompiledFunctionMetadata {
     pub cfa_unwind_info: Option<CfaUnwindInfo>,
     /// Mapping of value labels and their locations.
     pub value_labels_ranges: ValueLabelsRanges,
-    /// Allocated stack slots.
-    pub sized_stack_slots: ir::StackSlots,
     /// Start source location.
     pub start_srcloc: FilePos,
     /// End source location.
@@ -155,9 +153,11 @@ impl CompiledFunction {
         self.metadata.cfa_unwind_info = Some(unwind);
     }
 
-    /// Set the sized stack slots.
-    pub fn set_sized_stack_slots(&mut self, slots: ir::StackSlots) {
-        self.metadata.sized_stack_slots = slots;
+    /// Returns the frame-layout metadata for this function.
+    pub fn frame_layout(&self) -> &MachBufferFrameLayout {
+        self.buffer
+            .frame_layout()
+            .expect("Single-function MachBuffer must have frame layout information")
     }
 }
 

crates/cranelift/src/compiler.rs

@@ -14,7 +14,10 @@ use cranelift_codegen::isa::{
     unwind::{UnwindInfo, UnwindInfoKind},
 };
 use cranelift_codegen::print_errors::pretty_error;
-use cranelift_codegen::{CompiledCode, Context, FinalizedMachCallSite};
+use cranelift_codegen::{
+    CompiledCode, Context, FinalizedMachCallSite, MachBufferDebugTagList, MachBufferFrameLayout,
+    MachDebugTagPos,
+};
 use cranelift_entity::PrimaryMap;
 use cranelift_frontend::FunctionBuilder;
 use object::write::{Object, StandardSegment, SymbolId};
@@ -28,13 +31,13 @@ use std::ops::Range;
 use std::path;
 use std::sync::{Arc, Mutex};
 use wasmparser::{FuncValidatorAllocations, FunctionBody};
-use wasmtime_environ::obj::ELF_WASMTIME_EXCEPTIONS;
+use wasmtime_environ::obj::{ELF_WASMTIME_EXCEPTIONS, ELF_WASMTIME_FRAMES};
 use wasmtime_environ::{
     Abi, AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, CompiledFunctionBody,
-    DefinedFuncIndex, FlagValue, FuncKey, FunctionBodyData, FunctionLoc, HostCall,
-    InliningCompiler, ModuleTranslation, ModuleTypesBuilder, PtrSize, StackMapSection,
-    StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, TripleExt, Tunables, VMOffsets,
-    WasmFuncType, WasmValType,
+    DefinedFuncIndex, FlagValue, FrameInstPos, FrameStackShape, FrameTableBuilder, FuncKey,
+    FunctionBodyData, FunctionLoc, HostCall, InliningCompiler, ModuleTranslation,
+    ModuleTypesBuilder, PtrSize, StackMapSection, StaticModuleIndex, TrapEncodingBuilder,
+    TrapSentinel, TripleExt, Tunables, VMOffsets, WasmFuncType, WasmValType,
 };
 use wasmtime_unwinder::ExceptionTableBuilder;
 
@@ -252,7 +255,7 @@ impl wasmtime_environ::Compiler for Compiler {
             context.func.collect_debug_info();
         }
 
-        let mut func_env = FuncEnvironment::new(self, translation, types, wasm_func_ty);
+        let mut func_env = FuncEnvironment::new(self, translation, types, wasm_func_ty, key);
 
         // The `stack_limit` global value below is the implementation of stack
         // overflow checks in Wasmtime.
@@ -575,6 +578,7 @@ impl wasmtime_environ::Compiler for Compiler {
         let mut traps = TrapEncodingBuilder::default();
         let mut stack_maps = StackMapSection::default();
         let mut exception_tables = ExceptionTableBuilder::default();
+        let mut frame_tables = FrameTableBuilder::default();
 
         let mut ret = Vec::with_capacity(funcs.len());
         for (i, (sym, func)) in funcs.iter().enumerate() {
@@ -602,6 +606,16 @@ impl wasmtime_environ::Compiler for Compiler {
                 range.clone(),
                 func.buffer.call_sites(),
             )?;
+            if self.tunables.debug_instrumentation
+                && let Some(frame_layout) = func.buffer.frame_layout()
+            {
+                clif_to_env_frame_tables(
+                    &mut frame_tables,
+                    range.clone(),
+                    func.buffer.debug_tags(),
+                    frame_layout,
+                )?;
+            }
             builder.append_padding(self.linkopts.padding_between_functions);
 
             let info = FunctionLoc {
@@ -628,6 +642,17 @@ impl wasmtime_environ::Compiler for Compiler {
             obj.append_section_data(exception_section, bytes, 1);
         });
 
+        if self.tunables.debug_instrumentation {
+            let frame_table_section = obj.add_section(
+                obj.segment_name(StandardSegment::Data).to_vec(),
+                ELF_WASMTIME_FRAMES.as_bytes().to_vec(),
+                SectionKind::ReadOnlyData,
+            );
+            frame_tables.serialize(|bytes| {
+                obj.append_section_data(frame_table_section, bytes, 1);
+            });
+        }
+
         Ok(ret)
     }
 
@@ -1402,8 +1427,6 @@ impl FunctionCompiler<'_> {
             }
         }
 
-        compiled_function
-            .set_sized_stack_slots(std::mem::take(&mut context.func.sized_stack_slots));
         self.compiler.contexts.lock().unwrap().push(self.cx);
 
         Ok(compiled_function)
@@ -1448,6 +1471,56 @@ fn clif_to_env_exception_tables<'a>(
     builder.add_func(CodeOffset::try_from(range.start).unwrap(), call_sites)
 }
 
+/// Convert from Cranelift's representation of frame state slots and
+/// debug tags to Wasmtime's serialized metadata.
+fn clif_to_env_frame_tables<'a>(
+    builder: &mut FrameTableBuilder,
+    range: Range<u64>,
+    tag_sites: impl Iterator<Item = MachBufferDebugTagList<'a>>,
+    frame_layout: &MachBufferFrameLayout,
+) -> anyhow::Result<()> {
+    let mut frame_descriptors = HashMap::new();
+    for tag_site in tag_sites {
+        // Split into frames; each has three debug tags.
+        let mut frames = vec![];
+        for frame_tags in tag_site.tags.chunks_exact(3) {
+            let &[
+                ir::DebugTag::StackSlot(slot),
+                ir::DebugTag::User(wasm_pc),
+                ir::DebugTag::User(stack_shape),
+            ] = frame_tags
+            else {
+                panic!("Invalid tags");
+            };
+
+            let frame_descriptor = *frame_descriptors.entry(slot).or_insert_with(|| {
+                let slot_to_fp_offset =
+                    frame_layout.frame_to_fp_offset - frame_layout.stackslots[slot].offset;
+                let descriptor = frame_layout.stackslots[slot].descriptor.clone();
+                builder.add_frame_descriptor(slot_to_fp_offset, descriptor)
+            });
+
+            frames.push((
+                wasm_pc,
+                frame_descriptor,
+                FrameStackShape::from_raw(stack_shape),
+            ));
+        }
+
+        let native_pc_in_code_section = u32::try_from(range.start)
+            .unwrap()
+            .checked_add(tag_site.offset)
+            .unwrap();
+        let pos = match tag_site.pos {
+            MachDebugTagPos::Post => FrameInstPos::Post,
+            MachDebugTagPos::Pre => FrameInstPos::Pre,
+        };
+        builder.add_program_point(native_pc_in_code_section, pos, &frames);
+    }
+
+    Ok(())
+}
+
 fn save_last_wasm_entry_context(
     builder: &mut FunctionBuilder,
     pointer_type: ir::Type,