diff --git a/Cargo.lock b/Cargo.lock index f4de6613af55..bb416f937c2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -796,6 +796,7 @@ dependencies = [ "target-lexicon", "thiserror 2.0.12", "toml", + "wasmtime-unwinder", "wat", ] @@ -864,6 +865,7 @@ dependencies = [ "region", "target-lexicon", "wasmtime-jit-icache-coherence", + "wasmtime-unwinder", "windows-sys 0.59.0", ] @@ -2628,15 +2630,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "psm" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "871372391786ccec00d3c5d3d6608905b3d4db263639cfe075d3b60a736d115a" -dependencies = [ - "cc", -] - [[package]] name = "pulley-interpreter" version = "35.0.0" @@ -4173,7 +4166,6 @@ dependencies = [ "once_cell", "postcard", "proptest", - "psm", "pulley-interpreter", "rand", "rayon", @@ -4202,6 +4194,7 @@ dependencies = [ "wasmtime-math", "wasmtime-slab", "wasmtime-test-util", + "wasmtime-unwinder", "wasmtime-versioned-export-macros", "wasmtime-winch", "wasmtime-wmemcheck", @@ -4612,6 +4605,17 @@ dependencies = [ "wasmtime-environ", ] +[[package]] +name = "wasmtime-unwinder" +version = "35.0.0" +dependencies = [ + "anyhow", + "cfg-if", + "cranelift-codegen", + "log", + "object", +] + [[package]] name = "wasmtime-versioned-export-macros" version = "35.0.0" diff --git a/Cargo.toml b/Cargo.toml index b69b5d95c929..683f79141a4a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -249,6 +249,7 @@ wasmtime-fuzzing = { path = "crates/fuzzing" } wasmtime-jit-icache-coherence = { path = "crates/jit-icache-coherence", version = "=35.0.0" } wasmtime-wit-bindgen = { path = "crates/wit-bindgen", version = "=35.0.0" } wasmtime-math = { path = "crates/math", version = "=35.0.0" } +wasmtime-unwinder = { path = "crates/unwinder", version = "=35.0.0" } test-programs-artifacts = { path = 'crates/test-programs/artifacts' } wasmtime-test-util = { path = "crates/test-util" } diff --git a/cranelift/codegen/src/ir/extname.rs b/cranelift/codegen/src/ir/extname.rs index 452652baf28a..f29d854b1a44 100644 --- a/cranelift/codegen/src/ir/extname.rs +++ b/cranelift/codegen/src/ir/extname.rs @@ -113,6 +113,11 @@ impl TestcaseName { pub(crate) fn new>(v: T) -> Self { Self(v.as_ref().into()) } + + /// Get the raw test case name as bytes. + pub fn raw(&self) -> &[u8] { + &self.0 + } } /// The name of an external is either a reference to a user-defined symbol diff --git a/cranelift/codegen/src/lib.rs b/cranelift/codegen/src/lib.rs index af7a11477317..f45e9a9b39be 100644 --- a/cranelift/codegen/src/lib.rs +++ b/cranelift/codegen/src/lib.rs @@ -59,8 +59,8 @@ pub mod write; pub use crate::entity::packed_option; pub use crate::machinst::buffer::{ - FinalizedMachReloc, FinalizedRelocTarget, MachCallSite, MachSrcLoc, MachTextSectionBuilder, - MachTrap, OpenPatchRegion, PatchRegion, + FinalizedMachCallSite, FinalizedMachReloc, FinalizedRelocTarget, MachCallSite, MachSrcLoc, + MachTextSectionBuilder, MachTrap, OpenPatchRegion, PatchRegion, }; pub use crate::machinst::{ CallInfo, CompiledCode, Final, MachBuffer, MachBufferFinalized, MachInst, MachInstEmit, diff --git a/cranelift/filetests/Cargo.toml b/cranelift/filetests/Cargo.toml index 7736ff247903..c70a15237a4b 100644 --- a/cranelift/filetests/Cargo.toml +++ b/cranelift/filetests/Cargo.toml @@ -19,9 +19,10 @@ cranelift-frontend = { workspace = true } cranelift-interpreter = { workspace = true } cranelift-native = { workspace = true } cranelift-reader = { workspace = true } -cranelift-jit = { workspace = true, features = ["selinux-fix"] } +cranelift-jit = { workspace = true, features = ["selinux-fix", "wasmtime-unwinder"] } cranelift-module = { workspace = true } cranelift-control = { workspace = true } +wasmtime-unwinder = { workspace = true, features = ["cranelift"] } file-per-thread-logger = { workspace = true } filecheck = { workspace = true } gimli = { workspace = true, features = ['std'] } diff --git a/cranelift/filetests/filetests/runtests/throw.clif b/cranelift/filetests/filetests/runtests/throw.clif new file mode 100644 index 000000000000..676768749a84 --- /dev/null +++ b/cranelift/filetests/filetests/runtests/throw.clif @@ -0,0 +1,50 @@ +test run +set preserve_frame_pointers=true +target x86_64 +target aarch64 +target riscv64 +target s390x + +function %entry() -> i64 tail { + fn0 = %main(i64) -> i64 tail + +block0: + v1 = get_frame_pointer.i64 + v2 = call fn0(v1) + return v2 +} + +; run: %entry() == 58 + +function %main(i64) -> i64 tail { + sig0 = (i64, i32, i64, i64) tail + fn0 = %throw(i64, i32, i64, i64) tail + +block0(v0: i64): + v1 = iconst.i64 42 + v2 = iconst.i64 100 + v3 = iconst.i32 1 + try_call fn0(v0, v3, v1, v2), sig0, block1(), [ tag1: block2(exn0, exn1) ] + +block1: + v4 = iconst.i64 1 + return v4 + +block2(v5: i64, v6: i64): + v7 = isub.i64 v6, v5 + return v7 +} + + +function %throw(i64, i32, i64, i64) tail { + sig0 = (i64, i64, i64, i32, i64, i64) + fn0 = %__cranelift_throw(i64, i64, i64, i32, i64, i64) + +block0(v0: i64, v1: i32, v2: i64, v3: i64): + v4 = get_frame_pointer.i64 + v5 = get_return_address.i64 + v6 = load.i64 v5 ; get caller's FP + v7 = func_addr.i64 fn0 + call_indirect sig0, v7(v0, v4, v6, v1, v2, v3) + return +} diff --git a/cranelift/filetests/src/function_runner.rs b/cranelift/filetests/src/function_runner.rs index 20d7d4c7545c..f9ab1e2bcdbd 100644 --- a/cranelift/filetests/src/function_runner.rs +++ b/cranelift/filetests/src/function_runner.rs @@ -1,9 +1,12 @@ //! Provides functionality for compiling and running CLIF IR for `run` tests. use anyhow::{Result, anyhow}; use core::mem; +use cranelift::prelude::Imm64; +use cranelift_codegen::cursor::{Cursor, FuncCursor}; use cranelift_codegen::data_value::DataValue; use cranelift_codegen::ir::{ - ExternalName, Function, InstBuilder, Signature, UserExternalName, UserFuncName, + ExternalName, Function, InstBuilder, InstructionData, LibCall, Opcode, Signature, + UserExternalName, UserFuncName, }; use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa}; use cranelift_codegen::{CodegenError, Context, ir, settings}; @@ -14,9 +17,10 @@ use cranelift_module::{FuncId, Linkage, Module, ModuleError}; use cranelift_native::builder_with_options; use cranelift_reader::TestFile; use pulley_interpreter::interp as pulley; +use std::cell::Cell; use std::cmp::max; -use std::collections::HashMap; use std::collections::hash_map::Entry; +use std::collections::{HashMap, HashSet}; use std::ptr::NonNull; use target_lexicon::Architecture; use thiserror::Error; @@ -67,7 +71,7 @@ struct DefinedFunction { /// let compiled = compiler.compile().unwrap(); /// let trampoline = compiled.get_trampoline(&func).unwrap(); /// -/// let returned = trampoline.call(&vec![DataValue::I32(2), DataValue::I32(40)]); +/// let returned = trampoline.call(&compiled, &vec![DataValue::I32(2), DataValue::I32(40)]); /// assert_eq!(vec![DataValue::I32(42)], returned); /// ``` pub struct TestFileCompiler { @@ -255,7 +259,13 @@ impl TestFileCompiler { } /// Defines the body of a function - pub fn define_function(&mut self, func: Function, ctrl_plane: &mut ControlPlane) -> Result<()> { + pub fn define_function( + &mut self, + mut func: Function, + ctrl_plane: &mut ControlPlane, + ) -> Result<()> { + Self::replace_hostcall_references(&mut func); + let defined_func = self .defined_functions .get(&func.name) @@ -271,6 +281,47 @@ impl TestFileCompiler { Ok(()) } + fn replace_hostcall_references(func: &mut Function) { + // For every `func_addr` referring to a hostcall that we + // define, replace with an `iconst` with the actual + // address. Then modify the external func references to + // harmless libcall references (that will be unused so + // ignored). + let mut funcrefs_to_remove = HashSet::new(); + let mut cursor = FuncCursor::new(func); + while let Some(_block) = cursor.next_block() { + while let Some(inst) = cursor.next_inst() { + match &cursor.func.dfg.insts[inst] { + InstructionData::FuncAddr { + opcode: Opcode::FuncAddr, + func_ref, + } => { + let ext_func = &cursor.func.dfg.ext_funcs[*func_ref]; + let hostcall_addr = match &ext_func.name { + ExternalName::TestCase(tc) if tc.raw() == b"__cranelift_throw" => { + Some(__cranelift_throw as usize) + } + _ => None, + }; + + if let Some(addr) = hostcall_addr { + funcrefs_to_remove.insert(*func_ref); + cursor.func.dfg.insts[inst] = InstructionData::UnaryImm { + opcode: Opcode::Iconst, + imm: Imm64::new(addr as i64), + }; + } + } + _ => {} + } + } + } + + for to_remove in funcrefs_to_remove { + func.dfg.ext_funcs[to_remove].name = ExternalName::LibCall(LibCall::Probestack); + } + } + /// Creates and registers a trampoline for a function if none exists. pub fn create_trampoline_for_function( &mut self, @@ -356,6 +407,13 @@ impl Drop for CompiledTestFile { } } +std::thread_local! { + /// TLS slot used to store a CompiledTestFile reference so that it + /// can be recovered when a hostcall (such as the exception-throw + /// handler) is invoked. + pub static COMPILED_TEST_FILE: Cell<*const CompiledTestFile> = Cell::new(std::ptr::null()); +} + /// A callable trampoline pub struct Trampoline<'a> { module: &'a JITModule, @@ -366,16 +424,18 @@ pub struct Trampoline<'a> { impl<'a> Trampoline<'a> { /// Call the target function of this trampoline, passing in [DataValue]s using a compiled trampoline. - pub fn call(&self, arguments: &[DataValue]) -> Vec { + pub fn call(&self, compiled: &CompiledTestFile, arguments: &[DataValue]) -> Vec { let mut values = UnboxedValues::make_arguments(arguments, &self.func_signature); let arguments_address = values.as_mut_ptr(); let function_ptr = self.module.get_finalized_function(self.func_id); let trampoline_ptr = self.module.get_finalized_function(self.trampoline_id); + COMPILED_TEST_FILE.set(compiled as *const _); unsafe { self.call_raw(trampoline_ptr, function_ptr, arguments_address); } + COMPILED_TEST_FILE.set(std::ptr::null()); values.collect_returns(&self.func_signature) } @@ -563,6 +623,71 @@ fn make_trampoline(name: UserFuncName, signature: &ir::Signature, isa: &dyn Targ func } +/// Hostcall invoked directly from a compiled function body to test +/// exception throws. +/// +/// This function does not return normally: it either uses the +/// unwinder to jump directly to a Cranelift frame further up the +/// stack, if a handler is found; or it panics, if not. +#[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "riscv64" +))] +extern "C-unwind" fn __cranelift_throw( + entry_fp: usize, + exit_fp: usize, + exit_pc: usize, + tag: u32, + payload1: usize, + payload2: usize, +) -> ! { + let compiled_test_file = unsafe { &*COMPILED_TEST_FILE.get() }; + let unwind_host = wasmtime_unwinder::UnwindHost; + let module_lookup = |pc| { + compiled_test_file + .module + .as_ref() + .unwrap() + .lookup_wasmtime_exception_data(pc) + }; + unsafe { + match wasmtime_unwinder::compute_throw_action( + &unwind_host, + module_lookup, + exit_pc, + exit_fp, + entry_fp, + tag, + ) { + wasmtime_unwinder::ThrowAction::Handler { pc, sp, fp } => { + wasmtime_unwinder::resume_to_exception_handler(pc, sp, fp, payload1, payload2); + } + wasmtime_unwinder::ThrowAction::None => { + panic!("Expected a handler to exit for throw of tag {tag} at pc {exit_pc:x}"); + } + } + } +} + +#[cfg(not(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "riscv64" +)))] +extern "C-unwind" fn __cranelift_throw( + _entry_fp: usize, + _exit_fp: usize, + _exit_pc: usize, + _tag: u32, + _payload1: usize, + _payload2: usize, +) -> ! { + panic!("Throw not implemented on platforms without native backends."); +} + #[cfg(target_arch = "x86_64")] use std::arch::x86_64::__m128i; #[cfg(target_arch = "x86_64")] @@ -655,7 +780,7 @@ mod test { .unwrap(); let compiled = compiler.compile().unwrap(); let trampoline = compiled.get_trampoline(&function).unwrap(); - let returned = trampoline.call(&[]); + let returned = trampoline.call(&compiled, &[]); assert_eq!(returned, vec![DataValue::I8(-1)]) } diff --git a/cranelift/filetests/src/test_run.rs b/cranelift/filetests/src/test_run.rs index d12d1557c06c..dac83ab6d4c5 100644 --- a/cranelift/filetests/src/test_run.rs +++ b/cranelift/filetests/src/test_run.rs @@ -191,7 +191,7 @@ fn run_test( args.extend_from_slice(run_args); let trampoline = testfile.get_trampoline(func).unwrap(); - Ok(trampoline.call(&args)) + Ok(trampoline.call(&testfile, &args)) }) .map_err(|s| anyhow::anyhow!("{}", s))?; } diff --git a/cranelift/jit/Cargo.toml b/cranelift/jit/Cargo.toml index 9543428463cb..576a6313f709 100644 --- a/cranelift/jit/Cargo.toml +++ b/cranelift/jit/Cargo.toml @@ -19,6 +19,7 @@ cranelift-native = { workspace = true } cranelift-codegen = { workspace = true, features = ["std"] } cranelift-entity = { workspace = true } cranelift-control = { workspace = true } +wasmtime-unwinder = { workspace = true, optional = true, features = ["cranelift"] } anyhow = { workspace = true } region = "3.0.2" libc = { workspace = true } @@ -39,6 +40,8 @@ features = [ selinux-fix = ['memmap2'] default = [] +wasmtime-unwinder = ["dep:wasmtime-unwinder"] + [dev-dependencies] cranelift = { path = "../umbrella" } cranelift-frontend = { workspace = true } diff --git a/cranelift/jit/src/backend.rs b/cranelift/jit/src/backend.rs index 0f52b08361bb..f8c5667e2fa3 100644 --- a/cranelift/jit/src/backend.rs +++ b/cranelift/jit/src/backend.rs @@ -176,6 +176,7 @@ pub struct JITModule { declarations: ModuleDeclarations, compiled_functions: SecondaryMap>, compiled_data_objects: SecondaryMap>, + code_ranges: Vec<(usize, usize, FuncId)>, functions_to_finalize: Vec, data_objects_to_finalize: Vec, } @@ -328,6 +329,9 @@ impl JITModule { data.perform_relocations(|name| self.get_address(name)); } + self.code_ranges + .sort_unstable_by_key(|(start, _end, _)| *start); + // Now that we're done patching, prepare the memory for execution! let branch_protection = if cfg!(target_arch = "aarch64") && use_bti(&self.isa.isa_flags()) { BranchProtection::BTI @@ -358,10 +362,50 @@ impl JITModule { declarations: ModuleDeclarations::default(), compiled_functions: SecondaryMap::new(), compiled_data_objects: SecondaryMap::new(), + code_ranges: Vec::new(), functions_to_finalize: Vec::new(), data_objects_to_finalize: Vec::new(), } } + + /// Look up the Wasmtime unwind ExceptionTable and corresponding + /// base PC, if any, for a given PC that may be within one of the + /// CompiledBlobs in this module. + #[cfg(feature = "wasmtime-unwinder")] + pub fn lookup_wasmtime_exception_data<'a>( + &'a self, + pc: usize, + ) -> Option<(usize, wasmtime_unwinder::ExceptionTable<'a>)> { + // Search the sorted code-ranges for the PC. + let idx = match self + .code_ranges + .binary_search_by_key(&pc, |(start, _end, _func)| *start) + { + Ok(exact_start_match) => Some(exact_start_match), + Err(least_upper_bound) if least_upper_bound > 0 => { + let last_range_before_pc = &self.code_ranges[least_upper_bound - 1]; + if last_range_before_pc.0 <= pc && pc < last_range_before_pc.1 { + Some(least_upper_bound - 1) + } else { + None + } + } + _ => None, + }?; + + let (start, _, func) = self.code_ranges[idx]; + + // Get the ExceptionTable. The "parse" here simply reads two + // u32s for lengths and constructs borrowed slices, so it's + // cheap. + let data = self.compiled_functions[func] + .as_ref() + .unwrap() + .exception_data + .as_ref()?; + let exception_table = wasmtime_unwinder::ExceptionTable::parse(data).ok()?; + Some((start, exception_table)) + } } impl Module for JITModule { @@ -460,7 +504,32 @@ impl Module for JITModule { .collect(); self.record_function_for_perf(ptr, size, &decl.linkage_name(id)); - self.compiled_functions[id] = Some(CompiledBlob { ptr, size, relocs }); + self.compiled_functions[id] = Some(CompiledBlob { + ptr, + size, + relocs, + #[cfg(feature = "wasmtime-unwinder")] + exception_data: None, + }); + + let range_start = ptr as usize; + let range_end = range_start + size; + // These will be sorted when we finalize. + self.code_ranges.push((range_start, range_end, id)); + + #[cfg(feature = "wasmtime-unwinder")] + { + let mut exception_builder = wasmtime_unwinder::ExceptionTableBuilder::default(); + exception_builder + .add_func(0, compiled_code.buffer.call_sites()) + .map_err(|_| { + ModuleError::Compilation(cranelift_codegen::CodegenError::Unsupported( + "Invalid exception data".into(), + )) + })?; + self.compiled_functions[id].as_mut().unwrap().exception_data = + Some(exception_builder.to_vec()); + } self.functions_to_finalize.push(id); @@ -509,6 +578,8 @@ impl Module for JITModule { ptr, size, relocs: relocs.to_owned(), + #[cfg(feature = "wasmtime-unwinder")] + exception_data: None, }); self.functions_to_finalize.push(id); @@ -599,6 +670,8 @@ impl Module for JITModule { ptr, size: init.size(), relocs, + #[cfg(feature = "wasmtime-unwinder")] + exception_data: None, }); self.data_objects_to_finalize.push(id); diff --git a/cranelift/jit/src/compiled_blob.rs b/cranelift/jit/src/compiled_blob.rs index 4313c9f7ff87..157d64e2a913 100644 --- a/cranelift/jit/src/compiled_blob.rs +++ b/cranelift/jit/src/compiled_blob.rs @@ -15,6 +15,8 @@ pub(crate) struct CompiledBlob { pub(crate) ptr: *mut u8, pub(crate) size: usize, pub(crate) relocs: Vec, + #[cfg(feature = "wasmtime-unwinder")] + pub(crate) exception_data: Option>, } unsafe impl Send for CompiledBlob {} diff --git a/cranelift/src/run.rs b/cranelift/src/run.rs index b9d0b0b9a2c8..e78b7cb638fb 100644 --- a/cranelift/src/run.rs +++ b/cranelift/src/run.rs @@ -96,7 +96,7 @@ fn run_file_contents(file_contents: String) -> Result<()> { let trampoline = compiled.get_trampoline(&func).unwrap(); command - .run(|_, args| Ok(trampoline.call(args))) + .run(|_, args| Ok(trampoline.call(&compiled, args))) .map_err(|s| anyhow::anyhow!("{}", s))?; } } diff --git a/crates/unwinder/Cargo.toml b/crates/unwinder/Cargo.toml new file mode 100644 index 000000000000..316005c3aff6 --- /dev/null +++ b/crates/unwinder/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "wasmtime-unwinder" +authors.workspace = true +version.workspace = true +description = "Wasmtime's unwind format and unwinder" +license = "Apache-2.0 WITH LLVM-exception" +documentation = "https://docs.rs/wasmtime-unwinder" +repository = "https://github.com/bytecodealliance/wasmtime" +categories = ["no-std"] +readme = "README.md" +keywords = ["unwind", "exceptions"] +edition.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +cranelift-codegen = { workspace = true, optional = true } +log = { workspace = true } +cfg-if = { workspace = true } +object = { workspace = true } +anyhow = { workspace = true } + +[features] +default = [] + +# Enable generation of unwind info from Cranelift metadata. +cranelift = ["dep:cranelift-codegen"] diff --git a/crates/unwinder/LICENSE b/crates/unwinder/LICENSE new file mode 100644 index 000000000000..f9d81955f4bc --- /dev/null +++ b/crates/unwinder/LICENSE @@ -0,0 +1,220 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + diff --git a/crates/unwinder/README.md b/crates/unwinder/README.md new file mode 100644 index 000000000000..f9b0c7e8f348 --- /dev/null +++ b/crates/unwinder/README.md @@ -0,0 +1,13 @@ +# `wasmtime-unwinder` + +This crate implements an unwind info format, stack walking, and +unwinding for Wasmtime. It includes logic that: + +- Can walk the Wasmstack and visit each frame; +- Can find exception handlers using an efficient format serialized + from Cranelift compilation metadata that can be mapped and used + in-place from disk; +- Provides a "throw" helper that, when called from host code that has + been invoked from Wasmcode, can find a handler; and a "resume" stub + that can be invoked to transfer control to the corresponding + handler. diff --git a/crates/wasmtime/src/runtime/vm/arch/aarch64.rs b/crates/unwinder/src/arch/aarch64.rs similarity index 77% rename from crates/wasmtime/src/runtime/vm/arch/aarch64.rs rename to crates/unwinder/src/arch/aarch64.rs index 8d2e6daefd9c..3cd1ec4dc5b6 100644 --- a/crates/wasmtime/src/runtime/vm/arch/aarch64.rs +++ b/crates/unwinder/src/arch/aarch64.rs @@ -47,9 +47,40 @@ pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { pc } +pub unsafe fn resume_to_exception_handler( + pc: usize, + sp: usize, + fp: usize, + payload1: usize, + payload2: usize, +) -> ! { + unsafe { + core::arch::asm!( + "mov x0, {}", + "mov x1, {}", + "mov sp, {}", + "mov fp, {}", + "br {}", + in(reg) payload1, + in(reg) payload2, + in(reg) sp, + in(reg) fp, + in(reg) pc, + out("x0") _, + out("x1") _, + options(nostack, nomem), + ); + + core::hint::unreachable_unchecked() + } +} + // And the current frame pointer points to the next older frame pointer. pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; +// SP of caller is FP in callee plus size of FP/return address pair. +pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = 16; + pub fn assert_fp_is_aligned(_fp: usize) { // From AAPCS64, section 6.2.3 The Frame Pointer[0]: // diff --git a/crates/unwinder/src/arch/mod.rs b/crates/unwinder/src/arch/mod.rs new file mode 100644 index 000000000000..714657a4a7fc --- /dev/null +++ b/crates/unwinder/src/arch/mod.rs @@ -0,0 +1,109 @@ +//! Architecture-specific runtime support corresponding to details of +//! Cranelift codegen or ABI support. +//! +//! This crate houses any architecture-specific tidbits required when +//! building a runtime that executes Cranelift-produced code. +//! +//! All architectures have the same interface when exposed to the rest of the +//! crate. + +cfg_if::cfg_if! { + if #[cfg(target_arch = "x86_64")] { + mod x86; + use x86 as imp; + } else if #[cfg(target_arch = "aarch64")] { + mod aarch64; + use aarch64 as imp; + } else if #[cfg(target_arch = "s390x")] { + mod s390x; + use s390x as imp; + } else if #[cfg(target_arch = "riscv64")] { + mod riscv64; + use riscv64 as imp; + } +} + +// Re re-export functions from the `imp` module with one set of `pub +// use` declarations here so we can share doc-comments. + +cfg_if::cfg_if! { + if #[cfg(any( + target_arch = "x86_64", + target_arch = "aarch64", + target_arch = "s390x", + target_arch = "riscv64" + ))] { + /// Get the current stack pointer (at the time this function is + /// executing). This may be used to check, e.g., approximate space + /// remaining on a stack, but cannot be relied upon for anything exact + /// because the stack pointer from *within this function* is read and + /// the frame is later popped. + pub use imp::get_stack_pointer; + + /// Resume execution at the given PC, SP, and FP, with the given + /// payload values, according to the tail-call ABI's exception + /// scheme. Note that this scheme does not restore any other + /// registers, so the given state is all that we need. + /// + /// # Safety + /// + /// This method requires: + /// + /// - the `sp` and `fp` to correspond to an active stack frame + /// (above the current function), in code using Cranelift's + /// `tail` calling convention. + /// + /// - The `pc` to correspond to a `try_call` handler + /// destination, as emitted in Cranelift metadata, or + /// otherwise a target that is expecting the tail-call ABI's + /// exception ABI. + /// + /// - The Rust frames between the unwind destination and this + /// frame to be unwind-safe: that is, they cannot have `Drop` + /// handlers for which safety requires that they run. + pub use imp::resume_to_exception_handler; + + /// Get the return address in the function at the next-older + /// frame from the given FP. + /// + /// # Safety + /// + /// - Requires that `fp` is a valid frame-pointer value for an + /// active stack frame (above the current function), in code + /// using Cranelift's `tail` calling convention. + pub use imp::get_next_older_pc_from_fp; + + + /// The offset of the saved old-FP value in a frame, from the + /// location pointed to by a given FP. + pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = imp::NEXT_OLDER_FP_FROM_FP_OFFSET; + + /// The offset of the next older SP value, from the value of a + /// given FP. + pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = imp::NEXT_OLDER_SP_FROM_FP_OFFSET; + + /// Assert that the given `fp` is aligned as expected by the + /// host platform's implementation of the Cranelift tail-call + /// ABI. + pub use imp::assert_fp_is_aligned; + + /// If we have the above host-specific implementations, we can + /// implement `Unwind`. + pub struct UnwindHost; + + unsafe impl crate::stackwalk::Unwind for UnwindHost { + fn next_older_fp_from_fp_offset(&self) -> usize { + NEXT_OLDER_FP_FROM_FP_OFFSET + } + fn next_older_sp_from_fp_offset(&self) -> usize { + NEXT_OLDER_SP_FROM_FP_OFFSET + } + unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize { + get_next_older_pc_from_fp(fp) + } + fn assert_fp_is_aligned(&self, fp: usize) { + assert_fp_is_aligned(fp) + } + } + } +} diff --git a/crates/unwinder/src/arch/riscv64.rs b/crates/unwinder/src/arch/riscv64.rs new file mode 100644 index 000000000000..735fa13c985e --- /dev/null +++ b/crates/unwinder/src/arch/riscv64.rs @@ -0,0 +1,57 @@ +//! Riscv64-specific definitions of architecture-specific functions in Wasmtime. + +#[inline] +#[allow(missing_docs)] +pub fn get_stack_pointer() -> usize { + let stack_pointer: usize; + unsafe { + core::arch::asm!( + "mv {}, sp", + out(reg) stack_pointer, + options(nostack,nomem), + ); + } + stack_pointer +} + +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + *(fp as *mut usize).offset(1) +} + +pub unsafe fn resume_to_exception_handler( + pc: usize, + sp: usize, + fp: usize, + payload1: usize, + payload2: usize, +) -> ! { + unsafe { + core::arch::asm!( + "mv a0, {}", + "mv a1, {}", + "mv sp, {}", + "mv fp, {}", + "jr {}", + in(reg) payload1, + in(reg) payload2, + in(reg) sp, + in(reg) fp, + in(reg) pc, + out("a0") _, + out("a1") _, + options(nostack, nomem), + ); + + core::hint::unreachable_unchecked() + } +} + +// And the current frame pointer points to the next older frame pointer. +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +// SP of caller is FP in callee plus size of FP/return address pair. +pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = 16; + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); +} diff --git a/crates/unwinder/src/arch/s390x.rs b/crates/unwinder/src/arch/s390x.rs new file mode 100644 index 000000000000..25d9ecd3d0e9 --- /dev/null +++ b/crates/unwinder/src/arch/s390x.rs @@ -0,0 +1,59 @@ +//! s390x-specific definitions of architecture-specific functions in Wasmtime. + +#[inline] +#[allow(missing_docs)] +pub fn get_stack_pointer() -> usize { + let mut sp; + unsafe { + core::arch::asm!( + "lgr {}, %r15", + out(reg) sp, + options(nostack, nomem), + ); + } + sp +} + +pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { + // The next older PC can be found in register %r14 at function entry, which + // was saved into slot 14 of the register save area pointed to by "FP" (the + // backchain pointer). + *(fp as *mut usize).offset(14) +} + +pub unsafe fn resume_to_exception_handler( + pc: usize, + sp: usize, + _fp: usize, + payload1: usize, + payload2: usize, +) -> ! { + unsafe { + core::arch::asm!( + "lgr %r6, {}", + "lgr %r7, {}", + "lgr %r15, {}", + "br {}", + in(reg) payload1, + in(reg) payload2, + in(reg) sp, + in(reg) pc, + out("r6") _, + out("r7") _, + options(nostack, nomem), + ); + + core::hint::unreachable_unchecked() + } +} + +// The next older "FP" (backchain pointer) was saved in the slot pointed to +// by the current "FP". +pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; + +// SP of caller is "FP" (backchain pointer) in callee. +pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = 0; + +pub fn assert_fp_is_aligned(fp: usize) { + assert_eq!(fp % 8, 0, "stack should always be aligned to 8"); +} diff --git a/crates/wasmtime/src/runtime/vm/arch/x86.rs b/crates/unwinder/src/arch/x86.rs similarity index 57% rename from crates/wasmtime/src/runtime/vm/arch/x86.rs rename to crates/unwinder/src/arch/x86.rs index ef8dbb4bf7f7..cf3d6328bd53 100644 --- a/crates/wasmtime/src/runtime/vm/arch/x86.rs +++ b/crates/unwinder/src/arch/x86.rs @@ -21,9 +21,40 @@ pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { *(fp as *mut usize).offset(1) } +pub unsafe fn resume_to_exception_handler( + pc: usize, + sp: usize, + fp: usize, + payload1: usize, + payload2: usize, +) -> ! { + unsafe { + core::arch::asm!( + "mov rax, {}", + "mov rdx, {}", + "mov rsp, {}", + "mov rbp, {}", + "jmp {}", + in(reg) payload1, + in(reg) payload2, + in(reg) sp, + in(reg) fp, + in(reg) pc, + out("rax") _, + out("rdx") _, + options(nostack, nomem), + ); + + core::hint::unreachable_unchecked() + } +} + // And the current frame pointer points to the next older frame pointer. pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; +// SP of caller is FP in callee plus size of FP/return address pair. +pub const NEXT_OLDER_SP_FROM_FP_OFFSET: usize = 16; + /// Frame pointers are aligned if they're aligned to twice the size of a /// pointer. pub fn assert_fp_is_aligned(fp: usize) { diff --git a/crates/unwinder/src/exception_table.rs b/crates/unwinder/src/exception_table.rs new file mode 100644 index 000000000000..ab36ce079d50 --- /dev/null +++ b/crates/unwinder/src/exception_table.rs @@ -0,0 +1,300 @@ +//! Compact representation of exception handlers associated with +//! callsites, for use when searching a Cranelift stack for a handler. +//! +//! This module implements (i) conversion from the metadata provided +//! alongside Cranelift's compilation result (as provided by +//! [`cranelift_codegen::MachBufferFinalized::call_sites`]) to its +//! format, and (ii) use of its format to find a handler efficiently. +//! +//! The format has been designed so that it can be mapped in from disk +//! and used without post-processing; this enables efficient +//! module-loading in runtimes such as Wasmtime. + +use object::{Bytes, LittleEndian, U32Bytes}; + +#[cfg(feature = "cranelift")] +use alloc::{vec, vec::Vec}; +#[cfg(feature = "cranelift")] +use cranelift_codegen::{FinalizedMachCallSite, binemit::CodeOffset}; + +/// Collector struct for exception handlers per call site. +/// +/// # Format +/// +/// We keep four different arrays (`Vec`s) that we build as we visit +/// callsites, in ascending offset (address relative to beginning of +/// code segment) order: tags, destination offsets, callsite offsets, +/// and tag/destination ranges. +/// +/// The callsite offsets and tag/destination ranges logically form a +/// sorted lookup array, allowing us to find information for any +/// single callsite. The range denotes a range of indices in the tag +/// and destination offset arrays, and those are sorted by tag per +/// callsite. Ranges are stored with the (exclusive) *end* index only; +/// the start index is implicit as the previous end, or zero if first +/// element. +/// +/// # Example +/// +/// An example of this data format: +/// +/// ```plain +/// callsites: [0x10, 0x50, 0xf0] // callsites (return addrs) at offsets 0x10, 0x50, 0xf0 +/// ranges: [2, 4, 5] // corresponding ranges for each callsite +/// tags: [1, 5, 1, -1, -1] // tags for each handler at each callsite +/// handlers: [0x40, 0x42, 0x6f, 0x71, 0xf5] // handler destinations at each callsite +/// ``` +/// +/// Expanding this out: +/// +/// ```plain +/// callsites: [0x10, 0x50, 0xf0], # PCs relative to some start of return-points. +/// ranges: [ +/// 2, # callsite 0x10 has tags/handlers indices 0..2 +/// 4, # callsite 0x50 has tags/handlers indices 2..4 +/// 5, # callsite 0xf0 has tags/handlers indices 4..5 +/// ], +/// tags: [ +/// # tags for callsite 0x10: +/// 1, +/// 5, +/// # tags for callsite 0x50: +/// 1, +/// -1, # "catch-all" +/// # tags for callsite 0xf0: +/// -1, # "catch-all" +/// ] +/// handlers: [ +/// # handlers for callsite 0x10: +/// 0x40, # relative PC to handle tag 1 (above) +/// 0x42, # relative PC to handle tag 5 +/// # handlers for callsite 0x50: +/// 0x6f, # relative PC to handle tag 1 +/// 0x71, # relative PC to handle all other tags +/// # handlers for callsite 0xf0: +/// 0xf5, # relative PC to handle all other tags +/// ] +/// ``` +#[cfg(feature = "cranelift")] +#[derive(Clone, Debug, Default)] +pub struct ExceptionTableBuilder { + pub callsites: Vec>, + pub ranges: Vec>, + pub tags: Vec>, + pub handlers: Vec>, + last_start_offset: CodeOffset, +} + +#[cfg(feature = "cranelift")] +impl ExceptionTableBuilder { + /// Add a function at a given offset from the start of the + /// compiled code section, recording information about its call + /// sites. + /// + /// Functions must be added in ascending offset order. + pub fn add_func<'a>( + &mut self, + start_offset: CodeOffset, + call_sites: impl Iterator>, + ) -> anyhow::Result<()> { + // Ensure that we see functions in offset order. + assert!(start_offset >= self.last_start_offset); + self.last_start_offset = start_offset; + + // Visit each callsite in turn, translating offsets from + // function-local to section-local. + let mut handlers = vec![]; + for call_site in call_sites { + let ret_addr = call_site.ret_addr.checked_add(start_offset).unwrap(); + handlers.extend(call_site.exception_handlers.iter().cloned()); + handlers.sort_by_key(|(tag, _dest)| *tag); + + if handlers.windows(2).any(|parts| parts[0].0 == parts[1].0) { + anyhow::bail!("Duplicate handler tag"); + } + + let start_idx = u32::try_from(self.tags.len()).unwrap(); + for (tag, dest) in handlers.drain(..) { + self.tags.push(U32Bytes::new( + LittleEndian, + tag.expand().map(|t| t.as_u32()).unwrap_or(u32::MAX), + )); + self.handlers.push(U32Bytes::new( + LittleEndian, + dest.checked_add(start_offset).unwrap(), + )); + } + let end_idx = u32::try_from(self.tags.len()).unwrap(); + + // Omit empty callsites for compactness. + if end_idx > start_idx { + self.ranges.push(U32Bytes::new(LittleEndian, end_idx)); + self.callsites.push(U32Bytes::new(LittleEndian, ret_addr)); + } + } + + Ok(()) + } + + /// Serialize the exception-handler data section, taking a closure + /// to consume slices. + pub fn serialize(&self, mut f: F) { + // Serialize the length of `callsites` / `ranges`. + let callsite_count = u32::try_from(self.callsites.len()).unwrap(); + f(&callsite_count.to_le_bytes()); + // Serialize the length of `tags` / `handlers`. + let handler_count = u32::try_from(self.handlers.len()).unwrap(); + f(&handler_count.to_le_bytes()); + + // Serialize `callsites`, `ranges`, `tags`, and `handlers` in + // that order. + f(object::bytes_of_slice(&self.callsites)); + f(object::bytes_of_slice(&self.ranges)); + f(object::bytes_of_slice(&self.tags)); + f(object::bytes_of_slice(&self.handlers)); + } + + /// Serialize the exception-handler data section to a vector of + /// bytes. + pub fn to_vec(&self) -> Vec { + let mut bytes = vec![]; + self.serialize(|slice| bytes.extend(slice.iter().cloned())); + bytes + } +} + +/// ExceptionTable deserialized from a serialized slice. +/// +/// This struct retains borrows of the various serialized parts of the +/// exception table data as produced by +/// [`ExceptionTableBuilder::serialize`]. +#[derive(Clone, Debug)] +pub struct ExceptionTable<'a> { + callsites: &'a [U32Bytes], + ranges: &'a [U32Bytes], + tags: &'a [U32Bytes], + handlers: &'a [U32Bytes], +} + +impl<'a> ExceptionTable<'a> { + /// Parse exception tables from a byte-slice as produced by + /// [`ExceptionTableBuilder::serialize`]. + pub fn parse(data: &'a [u8]) -> anyhow::Result> { + let mut data = Bytes(data); + let callsite_count = data + .read::>() + .map_err(|_| anyhow::anyhow!("Unable to read callsite count prefix"))?; + let callsite_count = usize::try_from(callsite_count.get(LittleEndian))?; + let handler_count = data + .read::>() + .map_err(|_| anyhow::anyhow!("Unable to read handler count prefix"))?; + let handler_count = usize::try_from(handler_count.get(LittleEndian))?; + let (callsites, data) = + object::slice_from_bytes::>(data.0, callsite_count) + .map_err(|_| anyhow::anyhow!("Unable to read callsites slice"))?; + let (ranges, data) = + object::slice_from_bytes::>(data, callsite_count) + .map_err(|_| anyhow::anyhow!("Unable to read ranges slice"))?; + let (tags, data) = object::slice_from_bytes::>(data, handler_count) + .map_err(|_| anyhow::anyhow!("Unable to read tags slice"))?; + let (handlers, data) = + object::slice_from_bytes::>(data, handler_count) + .map_err(|_| anyhow::anyhow!("Unable to read handlers slice"))?; + + if !data.is_empty() { + anyhow::bail!("Unexpected data at end of serialized exception table"); + } + + Ok(ExceptionTable { + callsites, + ranges, + tags, + handlers, + }) + } + + /// Look up the handler destination, if any, for a given return + /// address (as an offset into the code section) and exception + /// tag. + /// + /// Note: we use raw `u32` types for code offsets and tags here to + /// avoid dependencies on `cranelift-codegen` when this crate is + /// built without compiler backend support (runtime-only config). + pub fn lookup(&self, pc: u32, tag: u32) -> Option { + // First, look up the callsite in the sorted callsites list. + let callsite_idx = self + .callsites + .binary_search_by_key(&pc, |callsite| callsite.get(LittleEndian)) + .ok()?; + // Now get the range. + let end_idx = self.ranges[callsite_idx].get(LittleEndian); + let start_idx = if callsite_idx > 0 { + self.ranges[callsite_idx - 1].get(LittleEndian) + } else { + 0 + }; + + // Take the subslices of `tags` and `handlers` corresponding + // to this callsite. + let start_idx = usize::try_from(start_idx).unwrap(); + let end_idx = usize::try_from(end_idx).unwrap(); + let tags = &self.tags[start_idx..end_idx]; + let handlers = &self.handlers[start_idx..end_idx]; + + // Is there any handler with an exact tag match? + if let Ok(handler_idx) = tags.binary_search_by_key(&tag, |tag| tag.get(LittleEndian)) { + return Some(handlers[handler_idx].get(LittleEndian)); + } + + // If not, is there a fallback handler? Note that we serialize + // it with the tag `u32::MAX`, so it is always last in sorted + // order. + if tags.last().map(|v| v.get(LittleEndian)) == Some(u32::MAX) { + return Some(handlers.last().unwrap().get(LittleEndian)); + } + + None + } +} + +#[cfg(all(test, feature = "cranelift"))] +mod test { + use super::*; + use cranelift_codegen::entity::EntityRef; + use cranelift_codegen::ir::ExceptionTag; + + #[test] + fn serialize_exception_table() { + let callsites = [ + FinalizedMachCallSite { + ret_addr: 0x10, + exception_handlers: &[ + (Some(ExceptionTag::new(1)).into(), 0x20), + (Some(ExceptionTag::new(2)).into(), 0x30), + (None.into(), 0x40), + ], + }, + FinalizedMachCallSite { + ret_addr: 0x48, + exception_handlers: &[], + }, + FinalizedMachCallSite { + ret_addr: 0x50, + exception_handlers: &[(None.into(), 0x60)], + }, + ]; + + let mut builder = ExceptionTableBuilder::default(); + builder.add_func(0x100, callsites.into_iter()).unwrap(); + let mut bytes = vec![]; + builder.serialize(|slice| bytes.extend(slice.iter().cloned())); + + let deserialized = ExceptionTable::parse(&bytes).unwrap(); + + assert_eq!(deserialized.lookup(0x148, 1), None); + assert_eq!(deserialized.lookup(0x110, 1), Some(0x120)); + assert_eq!(deserialized.lookup(0x110, 2), Some(0x130)); + assert_eq!(deserialized.lookup(0x110, 42), Some(0x140)); + assert_eq!(deserialized.lookup(0x150, 100), Some(0x160)); + } +} diff --git a/crates/unwinder/src/lib.rs b/crates/unwinder/src/lib.rs new file mode 100644 index 000000000000..ddd1523bfd84 --- /dev/null +++ b/crates/unwinder/src/lib.rs @@ -0,0 +1,18 @@ +//! Cranelift unwinder. +#![doc = include_str!("../README.md")] +#![no_std] +#![expect(unsafe_op_in_unsafe_fn, reason = "crate isn't migrated yet")] +#![expect(clippy::allow_attributes_without_reason, reason = "crate not migrated")] + +#[cfg(feature = "cranelift")] +extern crate alloc; + +mod stackwalk; +pub use stackwalk::*; +mod arch; +#[allow(unused_imports)] // `arch` becomes empty on platforms without native-code backends. +pub use arch::*; +mod exception_table; +pub use exception_table::*; +mod throw; +pub use throw::*; diff --git a/crates/unwinder/src/stackwalk.rs b/crates/unwinder/src/stackwalk.rs new file mode 100644 index 000000000000..b2b65cc6bb8f --- /dev/null +++ b/crates/unwinder/src/stackwalk.rs @@ -0,0 +1,175 @@ +//! Stack-walking of a Wasm stack. +//! +//! A stack walk requires a first and last frame pointer (FP), and it +//! only works on code that has been compiled with frame pointers +//! enabled (`preserve_frame_pointers` Cranelift option enabled). The +//! stack walk follows the singly-linked list of saved frame pointer +//! and return address pairs on the stack that is naturally built by +//! function prologues. +//! +//! This crate makes use of the fact that Wasmtime surrounds Wasm +//! frames by trampolines both at entry and exit, and is "up the +//! stack" from the point doing the unwinding: in other words, host +//! code invokes Wasm code via an entry trampoline, that code may call +//! other Wasm code, and ultimately it calls back to host code via an +//! exit trampoline. That exit trampoline is able to provide the +//! "start FP" (FP at exit trampoline) and "end FP" (FP at entry +//! trampoline) and this stack-walker can visit all Wasm frames +//! active on the stack between those two. +//! +//! This module provides a visitor interface to frames, but is +//! agnostic to the desired use-case or consumer of the frames, and to +//! the overall runtime structure. + +use core::ops::ControlFlow; + +/// Implementation necessary to unwind the stack, used by `Backtrace`. +pub unsafe trait Unwind { + /// Returns the offset, from the current frame pointer, of where to get to + /// the previous frame pointer on the stack. + fn next_older_fp_from_fp_offset(&self) -> usize; + + /// Returns the offset, from the current frame pointer, of the + /// stack pointer of the next older frame. + fn next_older_sp_from_fp_offset(&self) -> usize; + + /// Load the return address of a frame given the frame pointer for that + /// frame. + unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize; + + /// Debug assertion that the frame pointer is aligned. + fn assert_fp_is_aligned(&self, fp: usize); +} + +/// A stack frame within a Wasm stack trace. +#[derive(Debug)] +pub struct Frame { + /// The program counter in this frame. Because every frame in the + /// stack-walk is paused at a call (as we are in host code called + /// by Wasm code below these frames), the PC is at the return + /// address, i.e., points to the instruction after the call + /// instruction. + pc: usize, + /// The frame pointer value corresponding to this frame. + fp: usize, +} + +impl Frame { + /// Get this frame's program counter. + pub fn pc(&self) -> usize { + self.pc + } + + /// Get this frame's frame pointer. + pub fn fp(&self) -> usize { + self.fp + } +} + +/// Walk through a contiguous sequence of Wasm frames starting with +/// the frame at the given PC and FP and ending at +/// `trampoline_fp`. This FP should correspond to that of a trampoline +/// that was used to enter the Wasm code. +/// +/// We require that the initial PC, FP, and `trampoline_fp` values are +/// non-null (non-zero). +pub unsafe fn visit_frames( + unwind: &dyn Unwind, + mut pc: usize, + mut fp: usize, + trampoline_fp: usize, + mut f: impl FnMut(Frame) -> ControlFlow, +) -> ControlFlow { + log::trace!("=== Tracing through contiguous sequence of Wasm frames ==="); + log::trace!("trampoline_fp = 0x{:016x}", trampoline_fp); + log::trace!(" initial pc = 0x{:016x}", pc); + log::trace!(" initial fp = 0x{:016x}", fp); + + // Safety requirements documented above. + assert_ne!(pc, 0); + assert_ne!(fp, 0); + assert_ne!(trampoline_fp, 0); + + // This loop will walk the linked list of frame pointers starting + // at `fp` and going up until `trampoline_fp`. We know that both + // `fp` and `trampoline_fp` are "trusted values" aka generated and + // maintained by Wasmtime. This means that it should be safe to + // walk the linked list of pointers and inspect Wasm frames. + // + // Note, though, that any frames outside of this range are not + // guaranteed to have valid frame pointers. For example native code + // might be using the frame pointer as a general purpose register. Thus + // we need to be careful to only walk frame pointers in this one + // contiguous linked list. + // + // To know when to stop iteration all architectures' stacks currently + // look something like this: + // + // | ... | + // | Native Frames | + // | ... | + // |-------------------| + // | ... | <-- Trampoline FP | + // | Trampoline Frame | | + // | ... | <-- Trampoline SP | + // |-------------------| Stack + // | Return Address | Grows + // | Previous FP | <-- Wasm FP Down + // | ... | | + // | Cranelift Frames | | + // | ... | V + // + // The trampoline records its own frame pointer (`trampoline_fp`), + // which is guaranteed to be above all Wasm code. To check when + // we've reached the trampoline frame, it is therefore sufficient + // to check when the next frame pointer is equal to + // `trampoline_fp`. Once that's hit then we know that the entire + // linked list has been traversed. + // + // Note that it might be possible that this loop doesn't execute + // at all. For example if the entry trampoline called Wasm code + // which `return_call`'d an exit trampoline, then `fp == + // trampoline_fp` on the entry of this function, meaning the loop + // won't actually execute anything. + while fp != trampoline_fp { + // At the start of each iteration of the loop, we know that + // `fp` is a frame pointer from Wasm code. Therefore, we know + // it is not being used as an extra general-purpose register, + // and it is safe dereference to get the PC and the next older + // frame pointer. + // + // The stack also grows down, and therefore any frame pointer + // we are dealing with should be less than the frame pointer + // on entry to Wasm code. Finally also assert that it's + // aligned correctly as an additional sanity check. + assert!(trampoline_fp > fp, "{trampoline_fp:#x} > {fp:#x}"); + unwind.assert_fp_is_aligned(fp); + + log::trace!("--- Tracing through one Wasm frame ---"); + log::trace!("pc = {:p}", pc as *const ()); + log::trace!("fp = {:p}", fp as *const ()); + + f(Frame { pc, fp })?; + + pc = unwind.get_next_older_pc_from_fp(fp); + + // We rely on this offset being zero for all supported + // architectures in + // `crates/cranelift/src/component/compiler.rs` when we set + // the Wasm exit FP. If this ever changes, we will need to + // update that code as well! + assert_eq!(unwind.next_older_fp_from_fp_offset(), 0); + + // Get the next older frame pointer from the current Wasm + // frame pointer. + let next_older_fp = *(fp as *mut usize).add(unwind.next_older_fp_from_fp_offset()); + + // Because the stack always grows down, the older FP must be greater + // than the current FP. + assert!(next_older_fp > fp, "{next_older_fp:#x} > {fp:#x}"); + fp = next_older_fp; + } + + log::trace!("=== Done tracing contiguous sequence of Wasm frames ==="); + ControlFlow::Continue(()) +} diff --git a/crates/unwinder/src/throw.rs b/crates/unwinder/src/throw.rs new file mode 100644 index 000000000000..ea1652b1d622 --- /dev/null +++ b/crates/unwinder/src/throw.rs @@ -0,0 +1,80 @@ +//! Generation of the throw-stub. +//! +//! In order to throw exceptions from within Cranelift-compiled code, +//! we provide a runtime function helper meant to be called by host +//! code that is invoked by guest code. +//! +//! The helper below must be provided a delimited range on the stack +//! corresponding to Cranelift frames above the current host code. It +//! will look for any handlers in this code, given a closure that +//! knows how to use an absolute PC to look up a module's exception +//! table and its start-of-code-segment. If a handler is found, the +//! helper below will return the SP, FP and PC that must be +//! restored. Architecture-specific helpers are provided to jump to +//! this new context with payload values. Otherwise, if no handler is +//! found, the return type indicates this, and it is the caller's +//! responsibility to invoke alternative behavior (e.g., abort the +//! program or unwind all the way to initial Cranelift-code entry). + +use crate::{ExceptionTable, Unwind}; +use core::ops::ControlFlow; + +/// Throw action to perform. +#[derive(Clone, Debug)] +pub enum ThrowAction { + /// Jump to the given handler with the given SP and FP values. + Handler { + /// Program counter of handler return point. + pc: usize, + /// Stack pointer to restore before jumping to handler. + sp: usize, + /// Frame pointer to restore before jumping to handler. + fp: usize, + }, + /// No handler found. + None, +} + +/// Implementation of stack-walking to find a handler. +/// +/// This function searches for a handler in the given range of stack +/// frames, starting from the throw stub and up to a specified entry +/// frame. +pub unsafe fn compute_throw_action<'a, F: Fn(usize) -> Option<(usize, ExceptionTable<'a>)>>( + unwind: &dyn Unwind, + module_lookup: F, + exit_pc: usize, + exit_frame: usize, + entry_frame: usize, + tag: u32, +) -> ThrowAction { + let mut last_fp = exit_frame; + match crate::stackwalk::visit_frames(unwind, exit_pc, exit_frame, entry_frame, |frame| { + if let Some((base, table)) = module_lookup(frame.pc()) { + let relative_pc = u32::try_from( + frame + .pc() + .checked_sub(base) + .expect("module lookup did not return a module base below the PC"), + ) + .expect("module larger than 4GiB"); + + if let Some(handler) = table.lookup(relative_pc, tag) { + let abs_handler_pc = base + .checked_add(usize::try_from(handler).unwrap()) + .expect("Handler address computation overflowed"); + + return ControlFlow::Break(ThrowAction::Handler { + pc: abs_handler_pc, + sp: last_fp + unwind.next_older_sp_from_fp_offset(), + fp: frame.fp(), + }); + } + } + last_fp = frame.fp(); + ControlFlow::Continue(()) + }) { + ControlFlow::Break(action) => action, + ControlFlow::Continue(()) => ThrowAction::None, + } +} diff --git a/crates/wasmtime/Cargo.toml b/crates/wasmtime/Cargo.toml index 2f5105525cd1..02188fa8fd91 100644 --- a/crates/wasmtime/Cargo.toml +++ b/crates/wasmtime/Cargo.toml @@ -24,6 +24,7 @@ wasmtime-jit-icache-coherence = { workspace = true, optional = true } wasmtime-cache = { workspace = true, optional = true } wasmtime-fiber = { workspace = true, optional = true } wasmtime-cranelift = { workspace = true, optional = true, features = ['pulley'] } +wasmtime-unwinder = { workspace = true, optional = true } wasmtime-winch = { workspace = true, optional = true } wasmtime-component-macro = { workspace = true, optional = true } wasmtime-component-util = { workspace = true, optional = true } @@ -94,9 +95,6 @@ mach2 = { workspace = true, optional = true } [target.'cfg(unix)'.dependencies] rustix = { workspace = true, optional = true, features = ["mm", "param"] } -[target.'cfg(target_arch = "s390x")'.dependencies] -psm = { workspace = true, optional = true } - [dev-dependencies] env_logger = { workspace = true } proptest = { workspace = true } @@ -151,7 +149,7 @@ default = [ # with the Cranelift compiler. Cranelift is the default compilation backend of # Wasmtime. If disabled then WebAssembly modules can only be created from # precompiled WebAssembly modules. -cranelift = ["dep:wasmtime-cranelift", "std"] +cranelift = ["dep:wasmtime-cranelift", "std", "wasmtime-unwinder/cranelift"] # Enables support for Winch, the WebAssembly baseline compiler. The Winch compiler # strategy in `Config` will be available. It is currently in active development @@ -256,11 +254,11 @@ runtime = [ "dep:wasmtime-slab", "dep:wasmtime-versioned-export-macros", "dep:windows-sys", - "dep:psm", "dep:rustix", "rustix/mm", "pulley-interpreter/interp", "dep:wasmtime-jit-icache-coherence", + "dep:wasmtime-unwinder", ] # Enable support for garbage collection-related things. diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 260c698acb48..c2f016fe2f0a 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -1911,7 +1911,7 @@ at https://bytecodealliance.org/security. pub(crate) fn unwinder(&self) -> &'static dyn Unwind { match &self.executor { - Executor::Interpreter(_) => &vm::UnwindPulley, + Executor::Interpreter(i) => i.unwinder(), #[cfg(has_host_compiler_backend)] Executor::Native => &vm::UnwindHost, } diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index a4bd468a5a1e..83cadee596ca 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -48,8 +48,6 @@ use wasmtime_environ::{ #[cfg(feature = "gc")] use wasmtime_environ::ModuleInternedTypeIndex; -#[cfg(has_host_compiler_backend)] -mod arch; #[cfg(feature = "component-model")] pub mod component; mod const_expr; @@ -66,7 +64,6 @@ mod store_box; mod sys; mod table; mod traphandlers; -mod unwind; mod vmcontext; #[cfg(feature = "threads")] @@ -90,8 +87,6 @@ pub(crate) use interpreter_disabled as interpreter; #[cfg(feature = "debug-builtins")] pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration; -#[cfg(has_host_compiler_backend)] -pub use crate::runtime::vm::arch::get_stack_pointer; pub use crate::runtime::vm::export::*; pub use crate::runtime::vm::gc::*; pub use crate::runtime::vm::imports::Imports; @@ -119,7 +114,6 @@ pub use crate::runtime::vm::sys::mmap::open_file_for_mmap; pub use crate::runtime::vm::sys::unwind::UnwindRegistration; pub use crate::runtime::vm::table::{Table, TableElement}; pub use crate::runtime::vm::traphandlers::*; -pub use crate::runtime::vm::unwind::*; #[cfg(feature = "component-model")] pub use crate::runtime::vm::vmcontext::VMTableDefinition; pub use crate::runtime::vm::vmcontext::{ @@ -130,6 +124,10 @@ pub use crate::runtime::vm::vmcontext::{ }; pub use send_sync_ptr::SendSyncPtr; +pub use wasmtime_unwinder::Unwind; + +#[cfg(has_host_compiler_backend)] +pub use wasmtime_unwinder::{UnwindHost, get_stack_pointer}; mod module_id; pub use module_id::CompiledModuleId; diff --git a/crates/wasmtime/src/runtime/vm/arch/mod.rs b/crates/wasmtime/src/runtime/vm/arch/mod.rs deleted file mode 100644 index 780759a49024..000000000000 --- a/crates/wasmtime/src/runtime/vm/arch/mod.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! Architecture-specific support required by Wasmtime. -//! -//! This crate houses any architecture-specific tidbits required when running -//! Wasmtime. Each architecture has its own file in the `arch` folder which is -//! referenced here. -//! -//! All architectures have the same interface when exposed to the rest of the -//! crate. - -cfg_if::cfg_if! { - if #[cfg(target_arch = "x86_64")] { - mod x86; - use x86 as imp; - } else if #[cfg(target_arch = "aarch64")] { - mod aarch64; - use aarch64 as imp; - } else if #[cfg(target_arch = "s390x")] { - mod s390x; - use s390x as imp; - } else if #[cfg(target_arch = "riscv64")] { - mod riscv64; - use riscv64 as imp; - } else { - mod unsupported; - use unsupported as imp; - } -} - -// Functions defined in this module but all the implementations delegate to each -// `imp` module. This exists to assert that each module internally provides the -// same set of functionality with the same types for all architectures. - -pub fn get_stack_pointer() -> usize { - imp::get_stack_pointer() -} - -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - imp::get_next_older_pc_from_fp(fp) -} - -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = imp::NEXT_OLDER_FP_FROM_FP_OFFSET; - -pub fn assert_fp_is_aligned(fp: usize) { - imp::assert_fp_is_aligned(fp) -} diff --git a/crates/wasmtime/src/runtime/vm/arch/riscv64.rs b/crates/wasmtime/src/runtime/vm/arch/riscv64.rs deleted file mode 100644 index 0ca7f8209af6..000000000000 --- a/crates/wasmtime/src/runtime/vm/arch/riscv64.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! Riscv64-specific definitions of architecture-specific functions in Wasmtime. - -#[inline] -#[allow(missing_docs)] -pub fn get_stack_pointer() -> usize { - let stack_pointer: usize; - unsafe { - core::arch::asm!( - "mv {}, sp", - out(reg) stack_pointer, - options(nostack,nomem), - ); - } - stack_pointer -} - -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - *(fp as *mut usize).offset(1) -} - -// And the current frame pointer points to the next older frame pointer. -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 16, 0, "stack should always be aligned to 16"); -} diff --git a/crates/wasmtime/src/runtime/vm/arch/s390x.rs b/crates/wasmtime/src/runtime/vm/arch/s390x.rs deleted file mode 100644 index 2773381243ab..000000000000 --- a/crates/wasmtime/src/runtime/vm/arch/s390x.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! s390x-specific definitions of architecture-specific functions in Wasmtime. - -#[inline] -#[allow(missing_docs)] -pub fn get_stack_pointer() -> usize { - psm::stack_pointer() as usize -} - -pub unsafe fn get_next_older_pc_from_fp(fp: usize) -> usize { - // The next older PC can be found in register %r14 at function entry, which - // was saved into slot 14 of the register save area pointed to by "FP" (the - // backchain pointer). - *(fp as *mut usize).offset(14) -} - -// The next older "FP" (backchain pointer) was saved in the slot pointed to -// by the current "FP". -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn assert_fp_is_aligned(fp: usize) { - assert_eq!(fp % 8, 0, "stack should always be aligned to 8"); -} diff --git a/crates/wasmtime/src/runtime/vm/arch/unsupported.rs b/crates/wasmtime/src/runtime/vm/arch/unsupported.rs deleted file mode 100644 index 19aca0a3761f..000000000000 --- a/crates/wasmtime/src/runtime/vm/arch/unsupported.rs +++ /dev/null @@ -1,28 +0,0 @@ -compile_error!("Wasmtime's runtime is being compiled for an architecture that it does not support"); - -cfg_if::cfg_if! { - if #[cfg(target_arch = "riscv32")] { - compile_error!("\ -the tracking issue for riscv32 support is https://github.com/bytecodealliance/wasmtime/issues/8768 \ -"); - } else { - compile_error!("\ -if you'd like feel free to file an issue for platform support at -https://github.com/bytecodealliance/wasmtime/issues/new -"); - } -} - -pub fn get_stack_pointer() -> usize { - panic!() -} - -pub unsafe fn get_next_older_pc_from_fp(_fp: usize) -> usize { - panic!() -} - -pub const NEXT_OLDER_FP_FROM_FP_OFFSET: usize = 0; - -pub fn assert_fp_is_aligned(_fp: usize) { - panic!() -} diff --git a/crates/wasmtime/src/runtime/vm/interpreter.rs b/crates/wasmtime/src/runtime/vm/interpreter.rs index 6ec0b188b260..b3e98ee0bd9b 100644 --- a/crates/wasmtime/src/runtime/vm/interpreter.rs +++ b/crates/wasmtime/src/runtime/vm/interpreter.rs @@ -8,6 +8,7 @@ use core::ptr::NonNull; use pulley_interpreter::interp::{DoneReason, RegType, TrapKind, Val, Vm, XRegVal}; use pulley_interpreter::{FReg, Reg, XReg}; use wasmtime_environ::{BuiltinFunctionIndex, HostCall, Trap}; +use wasmtime_unwinder::Unwind; /// Interpreter state stored within a `Store`. #[repr(transparent)] @@ -70,6 +71,11 @@ impl Interpreter { pub fn pulley(&self) -> &Vm { unsafe { self.pulley.get().as_ref() } } + + /// Get an implementation of `Unwind` used to walk the Pulley stack. + pub fn unwinder(&self) -> &'static dyn Unwind { + &UnwindPulley + } } /// Wrapper around `&mut pulley_interpreter::Vm` to enable compiling this to a @@ -80,6 +86,36 @@ pub struct InterpreterRef<'a> { _phantom: marker::PhantomData<&'a mut Vm>, } +/// An implementation of stack-walking details specifically designed +/// for unwinding Pulley's runtime stack. +pub struct UnwindPulley; + +unsafe impl Unwind for UnwindPulley { + fn next_older_fp_from_fp_offset(&self) -> usize { + 0 + } + fn next_older_sp_from_fp_offset(&self) -> usize { + if cfg!(target_pointer_width = "32") { + 8 + } else { + 16 + } + } + unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize { + // The calling convention always pushes the return pointer (aka the PC + // of the next older frame) just before this frame. + *(fp as *mut usize).offset(1) + } + fn assert_fp_is_aligned(&self, fp: usize) { + let expected = if cfg!(target_pointer_width = "32") { + 8 + } else { + 16 + }; + assert_eq!(fp % expected, 0, "stack should always be aligned"); + } +} + /// Equivalent of a native platform's `jmp_buf` (sort of). /// /// This structure ensures that all callee-save state in Pulley is saved at wasm diff --git a/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs b/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs index c884f6b37a8a..a22c54fff725 100644 --- a/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs +++ b/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs @@ -10,6 +10,7 @@ use crate::{Engine, ValRaw}; use core::marker; use core::mem; use core::ptr::NonNull; +use wasmtime_unwinder::Unwind; pub struct Interpreter { empty: Uninhabited, @@ -26,6 +27,10 @@ impl Interpreter { pub fn as_interpreter_ref(&mut self) -> InterpreterRef<'_> { match self.empty {} } + + pub fn unwinder(&self) -> &'static dyn Unwind { + match self.empty {} + } } pub struct InterpreterRef<'a> { diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index d267904e8f15..f925394528d3 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -28,7 +28,7 @@ use core::ptr::{self, NonNull}; pub use self::backtrace::Backtrace; #[cfg(feature = "gc")] -pub use self::backtrace::Frame; +pub use wasmtime_unwinder::Frame; pub use self::coredump::CoreDumpStack; pub use self::tls::tls_eager_initialize; diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs index 84acb28ae2a2..97008c10d7c2 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/backtrace.rs @@ -31,35 +31,12 @@ use crate::runtime::vm::{ #[cfg(all(feature = "gc", feature = "stack-switching"))] use crate::vm::stack_switching::{VMContRef, VMStackState}; use core::ops::ControlFlow; +use wasmtime_unwinder::Frame; /// A WebAssembly stack trace. #[derive(Debug)] pub struct Backtrace(Vec); -/// A stack frame within a Wasm stack trace. -#[derive(Debug)] -pub struct Frame { - pc: usize, - #[cfg_attr( - not(feature = "gc"), - expect(dead_code, reason = "not worth #[cfg] annotations to remove") - )] - fp: usize, -} - -impl Frame { - /// Get this frame's program counter. - pub fn pc(&self) -> usize { - self.pc - } - - /// Get this frame's frame pointer. - #[cfg(feature = "gc")] - pub fn fp(&self) -> usize { - self.fp - } -} - impl Backtrace { /// Returns an empty backtrace pub fn empty() -> Backtrace { @@ -257,7 +234,7 @@ impl Backtrace { // Handle the stack that is currently running (which may be a // continuation or the initial stack). - Self::trace_through_wasm(unwind, pc, fp, trampoline_fp, &mut f)?; + wasmtime_unwinder::visit_frames(unwind, pc, fp, trampoline_fp, &mut f)?; // Note that the rest of this function has no effect if `chain` is // `Some(VMStackChain::InitialStack(_))` (i.e., there is only one stack to @@ -318,7 +295,7 @@ impl Backtrace { debug_assert!(parent_stack_range.contains(&parent_limits.stack_limit)); }); - Self::trace_through_wasm( + wasmtime_unwinder::visit_frames( unwind, resume_pc, resume_fp, @@ -329,108 +306,6 @@ impl Backtrace { ControlFlow::Continue(()) } - /// Walk through a contiguous sequence of Wasm frames starting with the - /// frame at the given PC and FP and ending at `trampoline_sp`. - unsafe fn trace_through_wasm( - unwind: &dyn Unwind, - mut pc: usize, - mut fp: usize, - trampoline_fp: usize, - mut f: impl FnMut(Frame) -> ControlFlow<()>, - ) -> ControlFlow<()> { - log::trace!("=== Tracing through contiguous sequence of Wasm frames ==="); - log::trace!("trampoline_fp = 0x{:016x}", trampoline_fp); - log::trace!(" initial pc = 0x{:016x}", pc); - log::trace!(" initial fp = 0x{:016x}", fp); - - // We already checked for this case in the `trace_with_trap_state` - // caller. - assert_ne!(pc, 0); - assert_ne!(fp, 0); - assert_ne!(trampoline_fp, 0); - - // This loop will walk the linked list of frame pointers starting at - // `fp` and going up until `trampoline_fp`. We know that both `fp` and - // `trampoline_fp` are "trusted values" aka generated and maintained by - // Cranelift. This means that it should be safe to walk the linked list - // of pointers and inspect wasm frames. - // - // Note, though, that any frames outside of this range are not - // guaranteed to have valid frame pointers. For example native code - // might be using the frame pointer as a general purpose register. Thus - // we need to be careful to only walk frame pointers in this one - // contiguous linked list. - // - // To know when to stop iteration all architectures' stacks currently - // look something like this: - // - // | ... | - // | Native Frames | - // | ... | - // |-------------------| - // | ... | <-- Trampoline FP | - // | Trampoline Frame | | - // | ... | <-- Trampoline SP | - // |-------------------| Stack - // | Return Address | Grows - // | Previous FP | <-- Wasm FP Down - // | ... | | - // | Wasm Frames | | - // | ... | V - // - // The trampoline records its own frame pointer (`trampoline_fp`), - // which is guaranteed to be above all Wasm. To check when we've - // reached the trampoline frame, it is therefore sufficient to - // check when the next frame pointer is equal to `trampoline_fp`. Once - // that's hit then we know that the entire linked list has been - // traversed. - // - // Note that it might be possible that this loop doesn't execute at all. - // For example if the entry trampoline called wasm which `return_call`'d - // an imported function which is an exit trampoline, then - // `fp == trampoline_fp` on the entry of this function, meaning the loop - // won't actually execute anything. - while fp != trampoline_fp { - // At the start of each iteration of the loop, we know that `fp` is - // a frame pointer from Wasm code. Therefore, we know it is not - // being used as an extra general-purpose register, and it is safe - // dereference to get the PC and the next older frame pointer. - // - // The stack also grows down, and therefore any frame pointer we are - // dealing with should be less than the frame pointer on entry to - // Wasm. Finally also assert that it's aligned correctly as an - // additional sanity check. - assert!(trampoline_fp > fp, "{trampoline_fp:#x} > {fp:#x}"); - unwind.assert_fp_is_aligned(fp); - - log::trace!("--- Tracing through one Wasm frame ---"); - log::trace!("pc = {:p}", pc as *const ()); - log::trace!("fp = {:p}", fp as *const ()); - - f(Frame { pc, fp })?; - - pc = unwind.get_next_older_pc_from_fp(fp); - - // We rely on this offset being zero for all supported architectures - // in `crates/cranelift/src/component/compiler.rs` when we set the - // Wasm exit FP. If this ever changes, we will need to update that - // code as well! - assert_eq!(unwind.next_older_fp_from_fp_offset(), 0); - - // Get the next older frame pointer from the current Wasm frame - // pointer. - let next_older_fp = *(fp as *mut usize).add(unwind.next_older_fp_from_fp_offset()); - - // Because the stack always grows down, the older FP must be greater - // than the current FP. - assert!(next_older_fp > fp, "{next_older_fp:#x} > {fp:#x}"); - fp = next_older_fp; - } - - log::trace!("=== Done tracing contiguous sequence of Wasm frames ==="); - ControlFlow::Continue(()) - } - /// Iterate over the frames inside this backtrace. pub fn frames<'a>( &'a self, diff --git a/crates/wasmtime/src/runtime/vm/unwind.rs b/crates/wasmtime/src/runtime/vm/unwind.rs deleted file mode 100644 index 6deee517cb6d..000000000000 --- a/crates/wasmtime/src/runtime/vm/unwind.rs +++ /dev/null @@ -1,59 +0,0 @@ -//! Support for low-level primitives of unwinding the stack. - -#[cfg(has_host_compiler_backend)] -use crate::runtime::vm::arch; - -/// Implementation necessary to unwind the stack, used by `Backtrace`. -pub unsafe trait Unwind { - /// Returns the offset, from the current frame pointer, of where to get to - /// the previous frame pointer on the stack. - fn next_older_fp_from_fp_offset(&self) -> usize; - - /// Load the return address of a frame given the frame pointer for that - /// frame. - unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize; - - /// Debug assertion that the frame pointer is aligned. - fn assert_fp_is_aligned(&self, fp: usize); -} - -/// A host-backed implementation of unwinding, using the native platform ABI -/// that Cranelift has. -#[cfg(has_host_compiler_backend)] -pub struct UnwindHost; - -#[cfg(has_host_compiler_backend)] -unsafe impl Unwind for UnwindHost { - fn next_older_fp_from_fp_offset(&self) -> usize { - arch::NEXT_OLDER_FP_FROM_FP_OFFSET - } - unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize { - arch::get_next_older_pc_from_fp(fp) - } - fn assert_fp_is_aligned(&self, fp: usize) { - arch::assert_fp_is_aligned(fp) - } -} - -/// An implementation specifically designed for unwinding Pulley's runtime stack -/// (which might not match the native host). -pub struct UnwindPulley; - -unsafe impl Unwind for UnwindPulley { - fn next_older_fp_from_fp_offset(&self) -> usize { - 0 - } - unsafe fn get_next_older_pc_from_fp(&self, fp: usize) -> usize { - // The calling convention always pushes the return pointer (aka the PC - // of the next older frame) just before this frame. - *(fp as *mut usize).offset(1) - } - fn assert_fp_is_aligned(&self, fp: usize) { - let expected = if cfg!(target_pointer_width = "32") { - 8 - } else { - 16 - }; - assert_eq!(fp % expected, 0, "stack should always be aligned"); - } -} diff --git a/fuzz/fuzz_targets/cranelift-fuzzgen.rs b/fuzz/fuzz_targets/cranelift-fuzzgen.rs index a5541ee23680..2b7c18b5f33d 100644 --- a/fuzz/fuzz_targets/cranelift-fuzzgen.rs +++ b/fuzz/fuzz_targets/cranelift-fuzzgen.rs @@ -19,7 +19,7 @@ use std::sync::atomic::Ordering; use cranelift_codegen::data_value::DataValue; use cranelift_codegen::ir::{LibCall, TrapCode}; use cranelift_codegen::isa; -use cranelift_filetests::function_runner::{TestFileCompiler, Trampoline}; +use cranelift_filetests::function_runner::{CompiledTestFile, TestFileCompiler, Trampoline}; use cranelift_fuzzgen::*; use cranelift_interpreter::environment::FuncIndex; use cranelift_interpreter::environment::FunctionStore; @@ -286,8 +286,12 @@ fn run_in_interpreter(interpreter: &mut Interpreter, args: &[DataValue]) -> RunR } } -fn run_in_host(trampoline: &Trampoline, args: &[DataValue]) -> RunResult { - let res = trampoline.call(args); +fn run_in_host( + compiled: &CompiledTestFile, + trampoline: &Trampoline, + args: &[DataValue], +) -> RunResult { + let res = trampoline.call(compiled, args); RunResult::Success(res) } @@ -413,6 +417,6 @@ fuzz_target!(|testcase: TestCase| { let compiled = compiler.compile().unwrap(); let trampoline = compiled.get_trampoline(testcase.main()).unwrap(); - run_test_inputs(&testcase, |args| run_in_host(&trampoline, args)); + run_test_inputs(&testcase, |args| run_in_host(&compiled, &trampoline, args)); } }); diff --git a/scripts/publish.rs b/scripts/publish.rs index cc76639d260f..112ae8e956fa 100644 --- a/scripts/publish.rs +++ b/scripts/publish.rs @@ -42,6 +42,9 @@ const CRATES_TO_PUBLISH: &[&str] = &[ "cranelift-object", "cranelift-interpreter", "wasmtime-jit-icache-coherence", + // Wasmtime unwinder, used by both `cranelift-jit` (optionally) and filetests, and by Wasmtime. + "wasmtime-unwinder", + // Cranelift crates that use Wasmtime unwinder. "cranelift-jit", "cranelift", // wiggle