[Feature] Improved Panic Handling

Cargo.toml

@@ -124,7 +124,7 @@ package = [
   "dep:dotenvy"
 ]
 
-async = [ "snarkvm-ledger/async", "snarkvm-synthesizer/async" ]
+async = [ "snarkvm-ledger?/async", "snarkvm-synthesizer?/async", "snarkvm-utilities?/async" ]
 cuda = [ "snarkvm-algorithms/cuda" ]
 history = [ "snarkvm-synthesizer/history" ]
 parameters_no_std_out = [ "snarkvm-parameters/no_std_out" ]
@@ -515,6 +515,9 @@ default-features = false
 [workspace.dependencies.smallvec]
 version = "1.14"
 
+[workspace.dependencies.tokio]
+version = "1"
+
 [workspace.dependencies.tempfile]
 version = "3.15"
 

ledger/narwhal/data/Cargo.toml

@@ -39,7 +39,7 @@ features = [ "preserve_order" ]
 
 [dependencies.tokio]
 optional = true
-version = "1"
+workspace = true
 features = [ "rt" ]
 
 [dev-dependencies.snarkvm-ledger-block]

synthesizer/src/vm/helpers/sequential_op.rs

@@ -16,6 +16,8 @@
 use crate::vm::*;
 use console::network::prelude::Network;
 
+use snarkvm_utilities::catch_unwind;
+
 use std::{fmt, thread};
 use tokio::sync::oneshot;
 
@@ -29,25 +31,32 @@ impl<N: Network, C: ConsensusStorage<N>> VM<N, C> {
         // Spawn a dedicated thread.
         let vm = self.clone();
         thread::spawn(move || {
-            // Sequentially process incoming operations.
-            while let Ok(request) = request_rx.recv() {
-                let SequentialOperationRequest { op, response_tx } = request;
-                debug!("Sequentially processing operation '{op}'");
-
-                // Perform the queued operation.
-                let ret = match op {
-                    SequentialOperation::AddNextBlock(block) => {
-                        let ret = vm.add_next_block_inner(block);
-                        SequentialOperationResult::AddNextBlock(ret)
-                    }
-                    SequentialOperation::AtomicSpeculate(a, b, c, d, e, f) => {
-                        let ret = vm.atomic_speculate_inner(a, b, c, d, e, f);
-                        SequentialOperationResult::AtomicSpeculate(ret)
-                    }
-                };
-
-                // Relay the results of the operation to the caller.
-                let _ = response_tx.send(ret);
+            let result = catch_unwind(move || {
+                // Sequentially process incoming operations.
+                while let Ok(request) = request_rx.recv() {
+                    let SequentialOperationRequest { op, response_tx } = request;
+                    debug!("Sequentially processing operation '{op}'");
+
+                    // Perform the queued operation.
+                    let ret = match op {
+                        SequentialOperation::AddNextBlock(block) => {
+                            let ret = vm.add_next_block_inner(block);
+                            SequentialOperationResult::AddNextBlock(ret)
+                        }
+                        SequentialOperation::AtomicSpeculate(a, b, c, d, e, f) => {
+                            let ret = vm.atomic_speculate_inner(a, b, c, d, e, f);
+                            SequentialOperationResult::AtomicSpeculate(ret)
+                        }
+                    };
+
+                    // Relay the results of the operation to the caller.
+                    let _ = response_tx.send(ret);
+                }
+            });
+
+            if let Err((msg, backtrace)) = result {
+                error!("Sequential ops thread encountered a fatal error: {msg}");
+                error!("Backtrace: {backtrace:?}");
             }
         })
     }

utilities/Cargo.toml

@@ -40,6 +40,11 @@ workspace = true
 [dependencies.colored]
 workspace = true
 
+[dependencies.tokio]
+workspace = true
+optional = true
+features = ["rt"]
+
 [dependencies.num_cpus]
 version = "1"
 
@@ -83,6 +88,7 @@ workspace = true
 
 [features]
 default = [ "derive" ]
+async = [ "tokio" ]
 derive = [ "snarkvm-utilities-derives" ]
 dev-print = [ ]
 serial = [ "derive" ]

utilities/src/errors.rs

@@ -14,7 +14,26 @@
 // limitations under the License.
 
 use colored::Colorize;
-use std::borrow::Borrow;
+
+use std::{
+    any::Any,
+    backtrace::Backtrace,
+    borrow::Borrow,
+    cell::Cell,
+    panic,
+    sync::atomic::{AtomicBool, Ordering},
+};
+
+thread_local! {
+/// The message backtrace of the last panic on this thread (if any).
+///
+/// We store this information here instead of directly processing it in a panic hook, because panic hooks are global whereas this can be processed on a per-thread basis.
+/// For example, one thread may execute a program where panics should *not* cause the entire process to terminate, while in another thread there is a panic due to a bug.
+static PANIC_INFO: Cell<Option<(String, Backtrace)>> = const { Cell::new(None) };
+}
+
+/// Keeps track of whether a panic hook was installed already.
+static PANIC_HOOK_INSTALLED: AtomicBool = const { AtomicBool::new(false) };
 
 /// Generates an `io::Error` from the given string.
 #[inline]
@@ -80,7 +99,30 @@ pub trait PrettyUnwrap {
     fn pretty_expect<S: ToString>(self, context: S) -> Self::Inner;
 }
 
-/// Helper for `PrettyUnwrap`, which creates a panic with the `anyhow::Error` nicely formatted and also logs the panic.
+/// Set the global panic hook for the process.
+///
+/// This function should be called once at startup. Subsequent calls to it have no effect.
+pub fn set_panic_hook() {
+    // Check if the hook was already installed.
+    // Note, that this allows for a small race condition, where the hook is installed by another thread after the check, but before the load.
+    // However, that is safe as the installed hook will be indentical, and this check merely exists for performance reasons.
+    if PANIC_HOOK_INSTALLED.load(Ordering::Acquire) {
+        return;
+    }
+
+    // Install the hook.
+    std::panic::set_hook(Box::new(|err| {
+        let msg = err.to_string();
+        let trace = Backtrace::force_capture();
+        PANIC_INFO.with(move |info| info.set(Some((msg, trace))));
+    }));
+
+    // Mark the hook as installed.
+    PANIC_HOOK_INSTALLED.store(true, Ordering::Release);
+}
+
+/// Helper for `PrettyUnwrap`:
+/// Creates a panic with the `anyhow::Error` nicely formatted.
 #[track_caller]
 #[inline]
 fn pretty_panic(error: &anyhow::Error) -> ! {
@@ -97,6 +139,7 @@ impl<T> PrettyUnwrap for anyhow::Result<T> {
     type Inner = T;
 
     #[track_caller]
+    #[inline]
     fn pretty_unwrap(self) -> Self::Inner {
         match self {
             Ok(result) => result,
@@ -117,9 +160,59 @@ impl<T> PrettyUnwrap for anyhow::Result<T> {
     }
 }
 
+/// `try_vm_runtime` executes the given closure in an environment which will safely halt
+/// without producing logs that look like unexpected behavior.
+/// In debug mode, it prints to stderr using the format: "VM safely halted at {location}: {halt message}".
+///
+/// Note: For this to work as expected, panics must be set to `unwind` during compilation (default), and the closure cannot invoke any async code that may potentially execute in a different OS thread.
+#[track_caller]
+#[inline]
+pub fn try_vm_runtime<R, F: FnMut() -> R>(f: F) -> Result<R, Box<dyn Any + Send>> {
+    // Perform the operation that may panic.
+    let result = std::panic::catch_unwind(panic::AssertUnwindSafe(f));
+
+    if result.is_err() {
+        // Get the stored panic and backtrace from the thread-local variable.
+        let (msg, _) = PANIC_INFO.with(|info| info.take()).expect("No panic information stored?");
+
+        #[cfg(debug_assertions)]
+        {
+            // Remove all words up to "panicked".
+            // And prepend with "VM Safely halted"
+            let msg = msg
+                .to_string()
+                .split_ascii_whitespace()
+                .skip_while(|&word| word != "panicked")
+                .collect::<Vec<&str>>()
+                .join(" ")
+                .replacen("panicked", "VM safely halted", 1);
+
+            eprintln!("{msg}");
+        }
+        #[cfg(not(debug_assertions))]
+        {
+            // Discard message
+            let _ = msg;
+        }
+    }
+
+    // Return the result, allowing regular error-handling.
+    result
+}
+
+/// `catch_unwind` calls the given closure `f` and, if `f` panics, returns the panic message and backtrace.
+#[inline]
+pub fn catch_unwind<R, F: FnMut() -> R>(f: F) -> Result<R, (String, Backtrace)> {
+    // Perform the operation that may panic.
+    std::panic::catch_unwind(panic::AssertUnwindSafe(f)).map_err(|_| {
+        // Get the stored panic and backtrace from the thread-local variable.
+        PANIC_INFO.with(|info| info.take()).expect("No panic information stored?")
+    })
+}
+
 #[cfg(test)]
 mod tests {
-    use super::{PrettyUnwrap, flatten_error, pretty_panic};
+    use super::{PrettyUnwrap, catch_unwind, flatten_error, pretty_panic, set_panic_hook, try_vm_runtime};
 
     use anyhow::{Context, Result, anyhow, bail};
     use colored::Colorize;
@@ -177,14 +270,31 @@ mod tests {
         assert_eq!(*result.downcast::<String>().expect("Error was not a string"), expected);
     }
 
+    // Ensure catch_unwind stores the panic message as expected.
+    #[test]
+    fn test_catch_unwind() {
+        set_panic_hook();
+        let result = catch_unwind(move || {
+            panic!("This is my message");
+        });
+        // Remove hook so test asserts work normally again.
+        let _ = std::panic::take_hook();
+
+        let (msg, bt) = result.expect_err("No panic caught");
+        assert!(msg.ends_with("This is my message"));
+
+        // This function should be in the panics backtrace
+        assert!(bt.to_string().contains("test_catch_unwind"));
+    }
+
     /// Ensure catch_unwind does not break `try_vm_runtime`.
     #[test]
     fn test_nested_with_try_vm_runtime() {
-        use crate::try_vm_runtime;
+        set_panic_hook();
 
         let result = std::panic::catch_unwind(|| {
             // try_vm_runtime uses catch_unwind internally
-            let vm_result = try_vm_runtime!(|| {
+            let vm_result = try_vm_runtime(|| {
                 panic!("VM operation failed!");
             });
 

utilities/src/lib.rs

@@ -34,9 +34,6 @@ pub use bytes::*;
 pub mod defer;
 pub use defer::*;
 
-mod vm_error;
-pub use vm_error::*;
-
 pub mod iterator;
 pub use iterator::*;
 
@@ -58,5 +55,24 @@ pub use serialize::*;
 pub mod errors;
 pub use errors::*;
 
+#[cfg(feature = "async")]
+/// Helpers to spawn async tasks.
+pub mod task;
+#[cfg(feature = "async")]
+pub use task::*;
+
 /// Use old name for backward-compatibility.
 pub use errors::io_error as error;
+
+/// This macro provides a VM runtime environment which will safely halt
+/// without producing logs that look like unexpected behavior.
+/// In debug mode, it prints to stderr using the format: "VM safely halted at {location}: {halt message}".
+///
+/// It is more efficient to set the panic hook once and directly use `errors::try_vm_runtime`.
+#[macro_export]
+macro_rules! try_vm_runtime {
+    ($e:expr) => {{
+        $crate::errors::set_panic_hook();
+        $crate::errors::try_vm_runtime($e)
+    }};
+}

utilities/src/task.rs

@@ -0,0 +1,71 @@
+// Copyright (c) 2019-2025 Provable Inc.
+// This file is part of the snarkVM library.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::{
+    future::Future,
+    pin::Pin,
+    task::{Context, Poll},
+};
+
+/// Wrapper around `tokio::JoinHandle` that propagates panics.
+pub struct JoinHandle<R: Send + 'static> {
+    inner: tokio::task::JoinHandle<R>,
+}
+
+/// Wrapper around `tokio::spawn_blocking` that propagates panics.
+pub fn spawn_blocking<F, R>(f: F) -> JoinHandle<R>
+where
+    F: FnOnce() -> R + Send + 'static,
+    R: Send + 'static,
+{
+    JoinHandle { inner: tokio::task::spawn_blocking(f) }
+}
+
+/// Wrapper around `tokio::spawn` that propagates panics.
+pub fn spawn<F>(f: F) -> JoinHandle<F::Output>
+where
+    F: Future + Send + 'static,
+    F::Output: Send + 'static,
+{
+    JoinHandle { inner: tokio::task::spawn(f) }
+}
+
+impl<R: Send + 'static> JoinHandle<R> {
+    pub fn abort(&self) {
+        self.inner.abort();
+    }
+}
+
+impl<R: Send + 'static> Future for JoinHandle<R> {
+    type Output = R;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let Poll::Ready(result) = std::pin::pin!(&mut self.inner).poll(cx) else {
+            return Poll::Pending;
+        };
+
+        match result {
+            Ok(value) => Poll::Ready(value),
+            Err(err) => {
+                if err.is_panic() {
+                    // Resume the panic on the main task
+                    std::panic::resume_unwind(err.into_panic());
+                } else {
+                    panic!("Got unexpected tokio error: {err}");
+                }
+            }
+        }
+    }
+}