Refactor exception handling to do a separate unwind, like traps. Support host-to-Wasm, Wasm-to-host, and host-to-host exceptions.

Author: cfallin

crates/cranelift/src/compiler.rs

@@ -1374,10 +1374,21 @@ fn save_last_wasm_exit_fp_and_pc(
     ptr: &impl PtrSize,
     limits: Value,
 ) {
+    // Save the trampoline FP to the limits. Exception unwind needs
+    // this so that it can know the SP (bottom of frame) for the very
+    // last Wasm frame.
+    let trampoline_fp = builder.ins().get_frame_pointer(pointer_type);
+    builder.ins().store(
+        MemFlags::trusted(),
+        trampoline_fp,
+        limits,
+        ptr.vmstore_context_last_wasm_exit_trampoline_fp(),
+    );
+
     // Save the exit Wasm FP to the limits. We dereference the current FP to get
     // the previous FP because the current FP is the trampoline's FP, and we
     // want the Wasm function's FP, which is the caller of this trampoline.
-    let trampoline_fp = builder.ins().get_frame_pointer(pointer_type);
+
     let wasm_fp = builder.ins().load(
         pointer_type,
         MemFlags::trusted(),

crates/environ/src/trap_encoding.rs

@@ -107,10 +107,6 @@ pub enum Trap {
     /// that all host tasks have completed and any/all host-owned stream/future
     /// handles have been dropped.
     AsyncDeadlock,
-
-    /// An exception was thrown out of Wasm into the host and this is
-    /// not supported in the current configuration.
-    ExceptionToHost,
     // if adding a variant here be sure to update the `check!` macro below
 }
 
@@ -152,7 +148,6 @@ impl Trap {
             ContinuationAlreadyConsumed
             DisabledOpcode
             AsyncDeadlock
-            ExceptionToHost
         }
 
         None
@@ -188,7 +183,6 @@ impl fmt::Display for Trap {
             ContinuationAlreadyConsumed => "continuation already consumed",
             DisabledOpcode => "pulley opcode disabled at compile time was executed",
             AsyncDeadlock => "deadlock detected: event loop cannot make further progress",
-            ExceptionToHost => "Wasm exception thrown across host boundary",
         };
         write!(f, "wasm trap: {desc}")
     }

crates/environ/src/vmoffsets.rs

@@ -190,20 +190,26 @@ pub trait PtrSize {
     /// Return the offset of the `gc_heap.base` field within a `VMStoreContext`.
     fn vmstore_context_gc_heap_base(&self) -> u8 {
         let offset = self.vmstore_context_gc_heap() + self.vmmemory_definition_base();
-        debug_assert!(offset < self.vmstore_context_last_wasm_exit_fp());
+        debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());
         offset
     }
 
     /// Return the offset of the `gc_heap.current_length` field within a `VMStoreContext`.
     fn vmstore_context_gc_heap_current_length(&self) -> u8 {
         let offset = self.vmstore_context_gc_heap() + self.vmmemory_definition_current_length();
-        debug_assert!(offset < self.vmstore_context_last_wasm_exit_fp());
+        debug_assert!(offset < self.vmstore_context_last_wasm_exit_trampoline_fp());
         offset
     }
 
+    /// Return the offset of the `last_wasm_exit_trampoline_fp` field
+    /// of `VMStoreContext`.
+    fn vmstore_context_last_wasm_exit_trampoline_fp(&self) -> u8 {
+        self.vmstore_context_gc_heap() + self.size_of_vmmemory_definition()
+    }
+
     /// Return the offset of the `last_wasm_exit_fp` field of `VMStoreContext`.
     fn vmstore_context_last_wasm_exit_fp(&self) -> u8 {
-        self.vmstore_context_gc_heap() + self.size_of_vmmemory_definition()
+        self.vmstore_context_last_wasm_exit_trampoline_fp() + self.size()
     }
 
     /// Return the offset of the `last_wasm_exit_pc` field of `VMStoreContext`.

crates/unwinder/src/throw.rs

@@ -50,32 +50,39 @@ pub unsafe fn compute_throw_action<F: Fn(&Frame) -> Option<usize>>(
     unwind: &dyn Unwind,
     frame_handler: F,
     exit_pc: usize,
-    exit_frame: usize,
+    exit_trampoline_frame: usize,
     entry_frame: usize,
 ) -> ThrowAction {
     // SAFETY: the safety of `visit_frames` relies on the correctness of the
     // parameters passed in which is forwarded as a contract to this function
     // tiself.
     let result = unsafe {
-        crate::stackwalk::visit_frames(unwind, exit_pc, exit_frame, entry_frame, |frame| {
-            let Some(sp) = frame.sp() else {
-                // Cannot possibly unwind to this frame if SP is not
-                // known. This is only the case for the first
-                // (trampoline) frame; after that, we know SP at the
-                // callsite because we know the offset from the lower
-                // FP to the next frame's SP.
-                return ControlFlow::Continue(());
-            };
+        crate::stackwalk::visit_frames(
+            unwind,
+            exit_pc,
+            exit_trampoline_frame,
+            entry_frame,
+            |frame| {
+                log::trace!("visit_frame: frame {frame:?}");
+                let Some(sp) = frame.sp() else {
+                    // Cannot possibly unwind to this frame if SP is not
+                    // known. This is only the case for the first
+                    // (trampoline) frame; after that, we know SP at the
+                    // callsite because we know the offset from the lower
+                    // FP to the next frame's SP.
+                    return ControlFlow::Continue(());
+                };
 
-            if let Some(handler_pc) = frame_handler(&frame) {
-                return ControlFlow::Break(ThrowAction::Handler {
-                    pc: handler_pc,
-                    sp,
-                    fp: frame.fp(),
-                });
-            }
-            ControlFlow::Continue(())
-        })
+                if let Some(handler_pc) = frame_handler(&frame) {
+                    return ControlFlow::Break(ThrowAction::Handler {
+                        pc: handler_pc,
+                        sp,
+                        fp: frame.fp(),
+                    });
+                }
+                ControlFlow::Continue(())
+            },
+        )
     };
     match result {
         ControlFlow::Break(action) => action,

crates/wasmtime/src/runtime/func.rs

@@ -1,15 +1,16 @@
-use crate::prelude::*;
 use crate::runtime::Uninhabited;
 use crate::runtime::vm::{
     InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, VMCommonStackInformation,
     VMContext, VMFuncRef, VMFunctionImport, VMOpaqueContext, VMStoreContext,
 };
 use crate::store::{AutoAssertNoGc, StoreId, StoreOpaque};
 use crate::type_registry::RegisteredType;
+use crate::vm::{ExceptionTombstone, VMGcRef};
 use crate::{
     AsContext, AsContextMut, CallHook, Engine, Extern, FuncType, Instance, ModuleExport, Ref,
     StoreContext, StoreContextMut, Val, ValRaw, ValType,
 };
+use crate::{ExnRef, Rooted, prelude::*};
 use alloc::sync::Arc;
 use core::ffi::c_void;
 #[cfg(feature = "async")]
@@ -1265,7 +1266,20 @@ impl Func {
 
         val_vec.extend((0..ty.results().len()).map(|_| Val::null_func_ref()));
         let (params, results) = val_vec.split_at_mut(nparams);
-        func(caller.sub_caller(), params, results)?;
+        match func(caller.sub_caller(), params, results) {
+            Ok(()) => {}
+            Err(e) if e.is::<Rooted<ExnRef>>() => {
+                let exnref = e.downcast::<Rooted<ExnRef>>().unwrap();
+                let exnref = VMGcRef::from_raw_u32(exnref.to_raw(&mut caller.store.0).unwrap())
+                    .unwrap()
+                    .into_exnref_unchecked();
+                caller.store.0.set_pending_exception(exnref);
+                return Err(ExceptionTombstone.into());
+            }
+            Err(e) => {
+                return Err(e);
+            }
+        }
 
         // Unlike our arguments we need to dynamically check that the return
         // values produced are correct. There could be a bug in `func` that
@@ -1513,7 +1527,7 @@ pub(crate) fn invoke_wasm_and_catch_traps<T>(
         let result = crate::runtime::vm::catch_traps(store, &mut previous_runtime_state, closure);
         core::mem::drop(previous_runtime_state);
         store.0.call_hook(CallHook::ReturningFromWasm)?;
-        result.map_err(|t| crate::trap::from_runtime_box(store.0, t))
+        result
     }
 }
 
@@ -1532,6 +1546,9 @@ pub(crate) struct EntryStoreContext {
     /// Contains value of `last_wasm_exit_fp` field to restore in
     /// `VMStoreContext` when exiting Wasm.
     pub last_wasm_exit_fp: usize,
+    /// Contains value of `last_wasm_exit_trampoline_fp` field to restore in
+    /// `VMStoreContext` when exiting Wasm.
+    pub last_wasm_exit_trampoline_fp: usize,
     /// Contains value of `last_wasm_entry_fp` field to restore in
     /// `VMStoreContext` when exiting Wasm.
     pub last_wasm_entry_fp: usize,
@@ -1625,6 +1642,11 @@ impl EntryStoreContext {
         unsafe {
             let last_wasm_exit_pc = *store.0.vm_store_context().last_wasm_exit_pc.get();
             let last_wasm_exit_fp = *store.0.vm_store_context().last_wasm_exit_fp.get();
+            let last_wasm_exit_trampoline_fp = *store
+                .0
+                .vm_store_context()
+                .last_wasm_exit_trampoline_fp
+                .get();
             let last_wasm_entry_fp = *store.0.vm_store_context().last_wasm_entry_fp.get();
 
             let stack_chain = (*store.0.vm_store_context().stack_chain.get()).clone();
@@ -1638,6 +1660,7 @@ impl EntryStoreContext {
                 stack_limit,
                 last_wasm_exit_pc,
                 last_wasm_exit_fp,
+                last_wasm_exit_trampoline_fp,
                 last_wasm_entry_fp,
                 stack_chain,
                 vm_store_context,
@@ -1657,6 +1680,8 @@ impl EntryStoreContext {
             }
 
             *(*self.vm_store_context).last_wasm_exit_fp.get() = self.last_wasm_exit_fp;
+            *(*self.vm_store_context).last_wasm_exit_trampoline_fp.get() =
+                self.last_wasm_exit_trampoline_fp;
             *(*self.vm_store_context).last_wasm_exit_pc.get() = self.last_wasm_exit_pc;
             *(*self.vm_store_context).last_wasm_entry_fp.get() = self.last_wasm_entry_fp;
             *(*self.vm_store_context).stack_chain.get() = self.stack_chain.clone();

crates/wasmtime/src/runtime/gc/enabled/exnref.rs

@@ -630,6 +630,15 @@ impl ExnRef {
     }
 }
 
+/// Rooted<ExnRef> implements Error so that it can be boxed and
+/// returned in an `anyhow::Result` from host-to-Wasm function calls.
+impl core::error::Error for Rooted<ExnRef> {}
+impl core::fmt::Display for Rooted<ExnRef> {
+    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+        write!(f, "Wasm exception")
+    }
+}
+
 unsafe impl WasmTy for Rooted<ExnRef> {
     #[inline]
     fn valtype() -> ValType {

crates/wasmtime/src/runtime/store.rs

@@ -89,8 +89,6 @@ use crate::prelude::*;
 use crate::runtime::vm::GcRootsList;
 #[cfg(feature = "stack-switching")]
 use crate::runtime::vm::VMContRef;
-#[cfg(feature = "gc")]
-use crate::runtime::vm::VMExnRef;
 use crate::runtime::vm::mpk::ProtectionKey;
 use crate::runtime::vm::{
     self, GcStore, Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle,
@@ -124,6 +122,8 @@ pub(crate) use token::StoreToken;
 mod async_;
 #[cfg(all(feature = "async", feature = "call-hook"))]
 pub use self::async_::CallHookHandler;
+
+use super::vm::VMExnRef;
 #[cfg(feature = "gc")]
 mod gc;
 
@@ -357,6 +357,12 @@ pub struct StoreOpaque {
     // Types for which the embedder has created an allocator for.
     #[cfg(feature = "gc")]
     gc_host_alloc_types: crate::hash_set::HashSet<crate::type_registry::RegisteredType>,
+    /// Pending exception, if any. This is also a GC root, because it
+    /// needs to be rooted somewhere between the time that a pending
+    /// exception is set and the time that the Wasm-to-host entry
+    /// trampoline starts the unwind (right before it would return).
+    #[cfg(feature = "gc")]
+    pending_exception: Option<VMGcRef>,
 
     // Numbers of resources instantiated in this store, and their limits
     instance_count: usize,
@@ -575,6 +581,8 @@ impl<T> Store<T> {
             gc_roots_list: GcRootsList::default(),
             #[cfg(feature = "gc")]
             gc_host_alloc_types: Default::default(),
+            #[cfg(feature = "gc")]
+            pending_exception: None,
             modules: ModuleRegistry::default(),
             func_refs: FuncRefs::default(),
             host_globals: PrimaryMap::new(),
@@ -1659,6 +1667,7 @@ impl StoreOpaque {
         self.trace_wasm_continuation_roots(gc_roots_list);
         self.trace_vmctx_roots(gc_roots_list);
         self.trace_user_roots(gc_roots_list);
+        self.trace_pending_exception_roots(gc_roots_list);
 
         log::trace!("End trace GC roots")
     }
@@ -1780,6 +1789,17 @@ impl StoreOpaque {
         log::trace!("End trace GC roots :: user");
     }
 
+    #[cfg(feature = "gc")]
+    fn trace_pending_exception_roots(&mut self, gc_roots_list: &mut GcRootsList) {
+        log::trace!("Begin trace GC roots :: pending exception");
+        if let Some(pending_exception) = self.pending_exception.as_mut() {
+            unsafe {
+                gc_roots_list.add_root(pending_exception.into(), "Pending exception");
+            }
+        }
+        log::trace!("End trace GC roots :: pending exception");
+    }
+
     /// Insert a host-allocated GC type into this store.
     ///
     /// This makes it suitable for the embedder to allocate instances of this
@@ -2144,18 +2164,6 @@ at https://bytecodealliance.org/security.
         Ok(ptr)
     }
 
-    /// Throws an exception.
-    ///
-    /// # Safety
-    ///
-    /// Must be invoked when Wasm code is on the stack.
-    #[cfg(feature = "gc")]
-    pub(crate) unsafe fn throw_ref(&mut self, exnref: VMExnRef) -> ! {
-        let mut nogc = AutoAssertNoGc::new(self);
-        // SAFETY: this is invoked when Wasm is on the stack.
-        unsafe { vm::throw(&mut nogc, exnref) }
-    }
-
     /// Constructs and executes an `InstanceAllocationRequest` and pushes the
     /// returned instance into the store.
     ///
@@ -2231,6 +2239,24 @@ at https://bytecodealliance.org/security.
 
         Ok(id)
     }
+
+    /// Set a pending exception. The `exnref` is taken and held on
+    /// this store to be fetched later by an unwind. This method does
+    /// *not* set up an unwind request on the TLS call state; that
+    /// must be done separately.
+    #[cfg(feature = "gc")]
+    pub(crate) fn set_pending_exception(&mut self, exnref: VMExnRef) {
+        self.pending_exception = Some(exnref.into());
+    }
+
+    /// Take a pending exception.
+    #[cfg(feature = "gc")]
+    pub(crate) fn take_pending_exception(&mut self) -> VMExnRef {
+        self.pending_exception
+            .take()
+            .expect("No pending exception set")
+            .into_exnref_unchecked()
+    }
 }
 
 /// Helper parameter to [`StoreOpaque::allocate_instance`].

crates/wasmtime/src/runtime/store/async_.rs

@@ -230,6 +230,8 @@ impl StoreOpaque {
         self.trace_vmctx_roots(gc_roots_list);
         Yield::new().await;
         self.trace_user_roots(gc_roots_list);
+        Yield::new().await;
+        self.trace_pending_exception_roots(gc_roots_list);
 
         log::trace!("End trace GC roots")
     }

crates/wasmtime/src/runtime/trap.rs

@@ -1,5 +1,8 @@
+use super::ExnRef;
 #[cfg(feature = "coredump")]
 use super::coredump::WasmCoreDump;
+use super::store::AutoAssertNoGc;
+use super::vm::ExceptionTombstone;
 use crate::prelude::*;
 use crate::store::StoreOpaque;
 use crate::{AsContext, Module};
@@ -79,6 +82,13 @@ pub(crate) fn from_runtime_box(
         coredumpstack,
     } = *runtime_trap;
     let (mut error, pc) = match reason {
+        crate::runtime::vm::TrapReason::User(error) if error.is::<ExceptionTombstone>() => {
+            let mut nogc = AutoAssertNoGc::new(store);
+            let exnref = nogc.take_pending_exception();
+            let exnref = ExnRef::_from_raw(&mut nogc, exnref.as_gc_ref().as_raw_u32())
+                .expect("Exception should be non-null");
+            (exnref.into(), None)
+        }
         // For user-defined errors they're already an `anyhow::Error` so no
         // conversion is really necessary here, but a `backtrace` may have
         // been captured so it's attempted to get inserted here.