op-reth: print a backtrace when an integration test dies on a signal

`p2p_version::peers_negotiate_eth_69` intermittently segfaults during teardown,
after its body has already passed (#20973). A binary killed by a signal produces
no Rust backtrace — `RUST_BACKTRACE` only covers panics — so five recurrences
over two months have yielded nothing beyond the signal number, and the crash has
never reproduced locally across 570+ attempts. Investigation is currently stalled
for want of a faulting stack.

Core dumps are not an option here: on the CircleCI Docker executor `/proc/sys` is
read-only so `core_pattern` cannot be set, and the host's pipe pattern makes the
kernel write the core outside the container. An in-process handler needs no
privileges, no artifacts and no CI configuration, and works locally too.

Install a handler for SIGSEGV/SIGBUS/SIGILL/SIGFPE that reports the faulting
thread's name and address, then its stack. The thread name alone discriminates
between the candidate causes: a database `atexit` destructor racing its own live
threads, background threads outliving process exit, or a runtime teardown race.

The stack is emitted in two stages, which matters:

  - `backtrace_symbols_fd` first. It writes straight to the fd and does not call
    malloc, so it is safe here. This is the guaranteed output.
  - `std::backtrace::Backtrace` second, for names and line numbers. Its
    symbolization allocates heavily and faults a second time when called from a
    handler — killing the process before anything is printed, since SIGSEGV is
    blocked inside its own handler. Attempting it only after the reliable frames
    are already on the wire means that failure costs nothing.

The handler re-raises with the default disposition, so the process still dies
with the original signal and nextest still records a crash. That composes with
the retry added in the parent commit: nextest prints a failed attempt's stderr
even when a later attempt passes, so a recurrence yields a stack while the job
stays green.

`libc` is already a workspace dependency; this adds it as a dev-dependency only.

Instrumentation only — no behavior change to any test, and no effect unless a
fatal signal fires.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: sebastianst

rust/Cargo.lock

@@ -105,6 +105,8 @@ alloy-op-hardforks.workspace = true
 alloy-rpc-types-admin.workspace = true
 alloy-signer-local.workspace = true
 futures.workspace = true
+# Fatal-signal handler in tests/it/crash_backtrace.rs.
+libc.workspace = true
 op-alloy-network.workspace = true
 
 [features]

rust/op-reth/crates/node/Cargo.toml

@@ -0,0 +1,149 @@
+//! Prints a backtrace when the test process dies on a fatal signal.
+//!
+//! A binary killed by a signal produces no Rust backtrace — `RUST_BACKTRACE` only
+//! covers panics — so a crash that happens *after* a test body has passed leaves
+//! nothing to diagnose beyond the signal number. Installing a handler recovers the
+//! faulting thread's name and stack, which is the evidence needed to tell apart the
+//! candidate causes of a teardown crash (a database's `atexit` destructor racing its
+//! own live threads, background threads outliving process exit, and so on).
+//!
+//! The handler re-raises with the default disposition, so the process still dies with
+//! the original signal and the test runner still records a crash rather than a pass.
+
+use std::sync::Once;
+
+static INIT: Once = Once::new();
+
+/// Installs the fatal-signal handler once per process.
+///
+/// Call at the start of a test that launches a full node. The handler stays installed
+/// for the lifetime of the process, so it also covers teardown after the test body has
+/// returned — which is when such crashes typically occur.
+pub(crate) fn install() {
+    INIT.call_once(install_inner);
+}
+
+/// Seconds allowed for backtrace capture before the watchdog kills the process.
+const CAPTURE_WATCHDOG_SECS: libc::c_uint = 10;
+
+/// Frames captured on a fault. Deep enough for a teardown stack through a runtime.
+const MAX_FRAMES: usize = 64;
+
+fn install_inner() {
+    unsafe {
+        // The first `backtrace()` call loads libgcc's unwinder and may allocate. Doing it
+        // here means the call inside the handler does not have to.
+        let mut warmup = [std::ptr::null_mut::<libc::c_void>(); 1];
+        libc::backtrace(warmup.as_mut_ptr(), warmup.len() as libc::c_int);
+
+        let mut action: libc::sigaction = std::mem::zeroed();
+        action.sa_sigaction = handler as *const () as usize;
+        action.sa_flags = libc::SA_SIGINFO | libc::SA_ONSTACK;
+        libc::sigemptyset(&raw mut action.sa_mask);
+
+        for sig in [libc::SIGSEGV, libc::SIGBUS, libc::SIGILL, libc::SIGFPE] {
+            libc::sigaction(sig, &raw const action, std::ptr::null_mut());
+        }
+    }
+}
+
+/// Writes directly to fd 2. `println!`/`eprintln!` take a lock and allocate, neither of
+/// which is legal in a signal handler; `write(2)` is async-signal-safe.
+fn write_stderr(bytes: &[u8]) {
+    unsafe {
+        libc::write(libc::STDERR_FILENO, bytes.as_ptr().cast(), bytes.len());
+    }
+}
+
+fn write_u64(mut n: u64) {
+    let mut buf = [0u8; 20];
+    let mut i = buf.len();
+    loop {
+        i -= 1;
+        buf[i] = b'0' + (n % 10) as u8;
+        n /= 10;
+        if n == 0 {
+            break;
+        }
+    }
+    write_stderr(&buf[i..]);
+}
+
+extern "C" fn handler(sig: libc::c_int, info: *mut libc::siginfo_t, _ctx: *mut libc::c_void) {
+    // Everything up to the watchdog is async-signal-safe, so this much is emitted even
+    // if the richer capture below wedges.
+    write_stderr(b"\n=== fatal signal ");
+    write_u64(sig as u64);
+    write_stderr(b" in thread '");
+    write_thread_name();
+    write_stderr(b"'");
+
+    if !info.is_null() {
+        write_stderr(b" faulting address 0x");
+        let addr = unsafe { (*info).si_addr() } as usize;
+        write_hex(addr as u64);
+    }
+    write_stderr(b" ===\n");
+
+    // Belt and braces: `backtrace()` is warmed up at install time so it should not
+    // allocate here, but a wedge would otherwise hang the job until the CI timeout.
+    unsafe {
+        libc::signal(libc::SIGALRM, libc::SIG_DFL);
+        libc::alarm(CAPTURE_WATCHDOG_SECS);
+    }
+
+    // `backtrace_symbols_fd` writes straight to the fd and is documented not to call
+    // malloc, unlike `std::backtrace::Backtrace`, whose symbolization allocates heavily
+    // and faults a second time when called from here — killing the process before it can
+    // print anything, since SIGSEGV is blocked inside its own handler.
+    //
+    // The tradeoff is resolution: this yields function names and addresses but no line
+    // numbers or inlined frames. Addresses can be resolved offline with addr2line against
+    // the same binary when more detail is needed.
+    unsafe {
+        let mut frames = [std::ptr::null_mut::<libc::c_void>(); MAX_FRAMES];
+        let n = libc::backtrace(frames.as_mut_ptr(), frames.len() as libc::c_int);
+        libc::backtrace_symbols_fd(frames.as_ptr(), n, libc::STDERR_FILENO);
+    }
+
+    // Now that the reliable frames are already on the wire, try for names and line
+    // numbers as a bonus. This is the step that can fault; if it does, the process dies
+    // with the original signal anyway and nothing above is lost.
+    write_stderr(b"--- symbolized (best effort; empty here means symbolization faulted) ---\n");
+    write_stderr(std::backtrace::Backtrace::force_capture().to_string().as_bytes());
+    write_stderr(b"\n");
+
+    // Die with the original signal so the runner still sees a crash. Restoring the
+    // default disposition first prevents recursing back into this handler.
+    unsafe {
+        libc::alarm(0);
+        libc::signal(sig, libc::SIG_DFL);
+        libc::raise(sig);
+    }
+}
+
+fn write_hex(n: u64) {
+    const DIGITS: &[u8; 16] = b"0123456789abcdef";
+    let mut buf = [0u8; 16];
+    for (i, slot) in buf.iter_mut().enumerate() {
+        *slot = DIGITS[((n >> (60 - i * 4)) & 0xf) as usize];
+    }
+    write_stderr(&buf);
+}
+
+/// The faulting thread's name is the single most useful field here: it distinguishes a
+/// crash in, say, a database transaction-manager thread from one in a runtime worker.
+///
+/// Writes straight from a stack buffer — building a `String` would allocate, which is
+/// not legal before the watchdog below is armed.
+fn write_thread_name() {
+    let mut buf = [0i8; 32];
+    let rc = unsafe { libc::pthread_getname_np(libc::pthread_self(), buf.as_mut_ptr(), buf.len()) };
+    if rc != 0 {
+        write_stderr(b"<unknown>");
+        return;
+    }
+    let len = buf.iter().position(|&c| c == 0).unwrap_or(buf.len());
+    let bytes = unsafe { std::slice::from_raw_parts(buf.as_ptr().cast::<u8>(), len) };
+    write_stderr(bytes);
+}

rust/op-reth/crates/node/tests/it/crash_backtrace.rs

@@ -1,5 +1,7 @@
 #![allow(missing_docs)]
 
+mod crash_backtrace;
+
 mod builder;
 
 mod custom_pool;

rust/op-reth/crates/node/tests/it/main.rs

@@ -15,6 +15,10 @@ const EXPECTED_ETH_VERSION: u64 = 69;
 
 #[tokio::test]
 async fn peers_negotiate_eth_69() -> eyre::Result<()> {
+    // This test intermittently segfaults during teardown, after the body below has
+    // passed (ethereum-optimism/optimism#20973). The handler outlives the body, so it
+    // reports the faulting thread and stack when that happens.
+    crate::crash_backtrace::install();
     reth_tracing::init_test_tracing();
 
     let (nodes, _wallet) = setup(2).await?;