feat: add failsafe to transaction replay

hstove · hstove · commit 983f9cecb804 · 2025-06-19T15:19:21.000-07:00
diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs
@@ -21,6 +21,7 @@ use std::time::{Duration, UNIX_EPOCH};
 use blockstack_lib::chainstate::burn::ConsensusHashExtensions;
 use blockstack_lib::chainstate::nakamoto::{NakamotoBlock, NakamotoBlockHeader};
 use blockstack_lib::chainstate::stacks::{StacksTransaction, TransactionPayload};
+use blockstack_lib::net::api::get_tenures_fork_info::TenureForkingInfo;
 use blockstack_lib::net::api::postblock_proposal::NakamotoBlockProposal;
 use clarity::types::chainstate::StacksAddress;
 #[cfg(any(test, feature = "testing"))]
@@ -620,7 +621,7 @@ impl LocalStateMachine {
                 client,
                 &expected_burn_block,
                 &prior_state_machine,
-                replay_state,
+                &replay_state,
             )? {
                 match new_replay_state {
                     ReplayState::Unset => {
@@ -632,6 +633,18 @@ impl LocalStateMachine {
                         *tx_replay_scope = Some(new_scope);
                     }
                 }
+            } else {
+                if Self::handle_possible_replay_failsafe(
+                    &replay_state,
+                    &expected_burn_block,
+                    client,
+                )? {
+                    info!(
+                    "Signer state: replay set is stalled after 2 tenures. Clearing the replay set."
+                );
+                    tx_replay_set = ReplayTransactionSet::none();
+                    *tx_replay_scope = None;
+                }
             }
         }
 
@@ -981,7 +994,7 @@ impl LocalStateMachine {
         client: &StacksClient,
         expected_burn_block: &NewBurnBlock,
         prior_state_machine: &SignerStateMachine,
-        replay_state: ReplayState,
+        replay_state: &ReplayState,
     ) -> Result<Option<ReplayState>, SignerChainstateError> {
         if expected_burn_block.burn_block_height > prior_state_machine.burn_block_height {
             // no bitcoin fork, because we're advancing the burn block height
@@ -1088,7 +1101,7 @@ impl LocalStateMachine {
         client: &StacksClient,
         expected_burn_block: &NewBurnBlock,
         prior_state_machine: &SignerStateMachine,
-        scope: ReplayScope,
+        scope: &ReplayScope,
     ) -> Result<Option<ReplayState>, SignerChainstateError> {
         info!("Tx Replay: detected bitcoin fork while in replay mode. Tryng to handle the fork";
             "expected_burn_block.height" => expected_burn_block.burn_block_height,
@@ -1182,6 +1195,10 @@ impl LocalStateMachine {
             return Ok(None);
         }
 
+        Ok(Some(Self::get_forked_txs_from_fork_info(&fork_info)))
+    }
+
+    fn get_forked_txs_from_fork_info(fork_info: &Vec<TenureForkingInfo>) -> Vec<StacksTransaction> {
         // Collect transactions to be replayed across the forked blocks
         let mut forked_blocks = fork_info
             .iter()
@@ -1201,6 +1218,70 @@ impl LocalStateMachine {
                 ))
             .cloned()
             .collect::<Vec<_>>();
-        Ok(Some(forked_txs))
+        forked_txs
+    }
+
+    /// If it has been 2 burn blocks since the origin of our replay set, and
+    /// we haven't produced any replay blocks since then, we should reset our replay set
+    ///
+    /// Returns a `bool` indicating whether the replay set should be reset.
+    fn handle_possible_replay_failsafe(
+        replay_state: &ReplayState,
+        new_burn_block: &NewBurnBlock,
+        client: &StacksClient,
+    ) -> Result<bool, SignerChainstateError> {
+        let ReplayState::InProgress(_, replay_scope) = replay_state else {
+            // Not in replay - skip
+            return Ok(false);
+        };
+
+        // if replay_scope.fork_origin.burn_block_height + 2 >= new_burn_block.burn_block_height {
+        if new_burn_block.burn_block_height < replay_scope.fork_origin.burn_block_height + 2 {
+            // We havent' had two burn blocks yet - skip
+            return Ok(false);
+        }
+
+        info!("Signer state: checking for replay set failsafe";
+            "replay_scope.fork_origin.burn_block_height" => replay_scope.fork_origin.burn_block_height,
+            "new_burn_block.burn_block_height" => new_burn_block.burn_block_height,
+        );
+        let Ok(fork_info) = client.get_tenure_forking_info(
+            &replay_scope.fork_origin.consensus_hash,
+            &new_burn_block.consensus_hash,
+        ) else {
+            warn!("Signer state: failed to get fork info");
+            return Ok(false);
+        };
+
+        let tenures_with_sortition = fork_info
+            .iter()
+            .filter(|fork_info| {
+                fork_info.was_sortition
+                    && fork_info
+                        .nakamoto_blocks
+                        .as_ref()
+                        .map(|b| b.len())
+                        .unwrap_or(0)
+                        > 0
+            })
+            .count();
+
+        info!("Signer state: fork info in failsafe check";
+            "tenures_with_sortition" => tenures_with_sortition,
+            "fork_info" => ?fork_info,
+        );
+
+        if tenures_with_sortition < 2 {
+            // We might have had 2 burn blocks, but not 2 tenures.
+            return Ok(false);
+        }
+
+        let forked_txs = Self::get_forked_txs_from_fork_info(&fork_info);
+
+        info!("Signer state: forked txs in failsafe check";
+            "forked_txs_len" => forked_txs.len(),
+        );
+
+        Ok(forked_txs.is_empty())
     }
 }
diff --git a/stackslib/src/net/api/postblock_proposal.rs b/stackslib/src/net/api/postblock_proposal.rs
@@ -68,6 +68,10 @@ pub static TEST_REPLAY_TRANSACTIONS: LazyLock<
     TestFlag<std::collections::VecDeque<StacksTransaction>>,
 > = LazyLock::new(TestFlag::default);
 
+#[cfg(any(test, feature = "testing"))]
+/// Whether to reject any transaction while we're in a replay set.
+pub static TEST_REJECT_REPLAY_TXS: LazyLock<TestFlag<bool>> = LazyLock::new(TestFlag::default);
+
 // This enum is used to supply a `reason_code` for validation
 //  rejection responses. This is serialized as an enum with string
 //  type (in jsonschema terminology).
@@ -200,6 +204,24 @@ fn fault_injection_validation_delay() {
 #[cfg(not(any(test, feature = "testing")))]
 fn fault_injection_validation_delay() {}
 
+#[cfg(any(test, feature = "testing"))]
+fn fault_injection_reject_replay_txs() -> Result<(), BlockValidateRejectReason> {
+    let reject = TEST_REJECT_REPLAY_TXS.get();
+    if reject {
+        Err(BlockValidateRejectReason {
+            reason_code: ValidateRejectCode::InvalidTransactionReplay,
+            reason: "Rejected by test flag".into(),
+        })
+    } else {
+        Ok(())
+    }
+}
+
+#[cfg(not(any(test, feature = "testing")))]
+fn fault_injection_reject_replay_txs() -> Result<(), BlockValidateRejectReason> {
+    Ok(())
+}
+
 /// Represents a block proposed to the `v3/block_proposal` endpoint for validation
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct NakamotoBlockProposal {
@@ -723,6 +745,7 @@ impl NakamotoBlockProposal {
                     // Allow this to happen, tenure extend checks happen elsewhere.
                     break;
                 }
+                fault_injection_reject_replay_txs()?;
                 let Some(replay_tx) = replay_txs.pop_front() else {
                     // During transaction replay, we expect that the block only
                     // contains transactions from the replay set. Thus, if we're here,
diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs
@@ -49,7 +49,6 @@ use stacks::chainstate::stacks::boot::MINERS_NAME;
 use stacks::chainstate::stacks::db::{StacksBlockHeaderTypes, StacksChainState, StacksHeaderInfo};
 use stacks::chainstate::stacks::miner::{
     TransactionEvent, TransactionSuccessEvent, TEST_EXCLUDE_REPLAY_TXS,
-    TEST_MINE_ALLOWED_REPLAY_TXS,
 };
 use stacks::chainstate::stacks::{
     StacksTransaction, TenureChangeCause, TenureChangePayload, TransactionPayload,
@@ -65,8 +64,8 @@ use stacks::core::{StacksEpochId, CHAIN_ID_TESTNET, HELIUM_BLOCK_LIMIT_20};
 use stacks::libstackerdb::StackerDBChunkData;
 use stacks::net::api::getsigner::GetSignerResponse;
 use stacks::net::api::postblock_proposal::{
-    BlockValidateResponse, ValidateRejectCode, TEST_VALIDATE_DELAY_DURATION_SECS,
-    TEST_VALIDATE_STALL,
+    BlockValidateResponse, ValidateRejectCode, TEST_REJECT_REPLAY_TXS,
+    TEST_VALIDATE_DELAY_DURATION_SECS, TEST_VALIDATE_STALL,
 };
 use stacks::net::relay::fault_injection::{clear_ignore_block, set_ignore_block};
 use stacks::types::chainstate::{
@@ -3802,6 +3801,195 @@ fn tx_replay_btc_on_stx_invalidation() {
     signer_test.shutdown();
 }
 
+/// Test scenario to ensure that the replay set is cleared
+/// if there have been multiple tenures with a stalled replay set.
+///
+/// This test is executed by triggering a fork, and then using
+/// a test flag to reject any transaction replay blocks.
+///
+/// The test mines a number of burn blocks during replay before
+/// validating that the replay set is eventually cleared.
+#[ignore]
+#[test]
+fn tx_replay_failsafe() {
+    if env::var("BITCOIND_TEST") != Ok("1".into()) {
+        return;
+    }
+
+    let num_signers = 5;
+    let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes());
+    let sender_addr = tests::to_addr(&sender_sk);
+    let send_amt = 100;
+    let send_fee = 180;
+    let signer_test: SignerTest<SpawnedSigner> =
+        SignerTest::new_with_config_modifications_and_snapshot(
+            num_signers,
+            vec![(sender_addr, (send_amt + send_fee) * 10)],
+            |c| {
+                c.validate_with_replay_tx = true;
+            },
+            |node_config| {
+                node_config.miner.block_commit_delay = Duration::from_secs(1);
+                node_config.miner.replay_transactions = true;
+                node_config.miner.activated_vrf_key_path =
+                    Some(format!("{}/vrf_key", node_config.node.working_dir));
+            },
+            None,
+            None,
+            Some(function_name!()),
+        );
+
+    let conf = &signer_test.running_nodes.conf;
+    let _http_origin = format!("http://{}", &conf.node.rpc_bind);
+    let btc_controller = &signer_test.running_nodes.btc_regtest_controller;
+
+    if signer_test.bootstrap_snapshot() {
+        signer_test.shutdown_and_snapshot();
+        return;
+    }
+
+    info!("------------------------- Beginning test -------------------------");
+
+    let burnchain = conf.get_burnchain();
+
+    let tip = signer_test.get_peer_info();
+    let pox_info = signer_test.get_pox_data();
+
+    info!("---- Burnchain ----";
+        // "burnchain" => ?conf.burnchain,
+        "pox_constants" => ?burnchain.pox_constants,
+        "cycle" => burnchain.pox_constants.reward_cycle_index(0, tip.burn_block_height),
+        "pox_info" => ?pox_info,
+    );
+
+    let pre_fork_tenures = 11;
+    for i in 0..pre_fork_tenures {
+        info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1);
+        signer_test.mine_nakamoto_block(Duration::from_secs(30), true);
+    }
+
+    info!("---- Submitting STX transfer ----");
+
+    let tip = get_chain_info(&conf);
+    // Make a transfer tx (this will get forked)
+    let (txid, nonce) = signer_test
+        .submit_transfer_tx(&sender_sk, send_fee, send_amt)
+        .unwrap();
+
+    // Ensure we got a new block with this tx
+    signer_test
+        .wait_for_nonce_increase(&sender_addr, nonce)
+        .expect("Timed out waiting for transfer tx to be mined");
+
+    wait_for(30, || {
+        let new_tip = get_chain_info(&conf);
+        Ok(new_tip.stacks_tip_height > tip.stacks_tip_height)
+    })
+    .expect("Timed out waiting for transfer tx to be mined");
+
+    let tip = get_chain_info(&conf);
+
+    info!("---- Triggering Bitcoin fork ----";
+        "tip.stacks_tip_height" => tip.stacks_tip_height,
+        "tip.burn_block_height" => tip.burn_block_height,
+    );
+
+    let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2);
+    btc_controller.invalidate_block(&burn_header_hash_to_fork);
+    btc_controller.build_next_block(3);
+
+    TEST_MINE_STALL.set(true);
+
+    let submitted_commits = signer_test
+        .running_nodes
+        .counters
+        .naka_submitted_commits
+        .clone();
+
+    // we need to mine some blocks to get back to being considered a frequent miner
+    for i in 0..3 {
+        let current_burn_height = get_chain_info(&conf).burn_block_height;
+        info!(
+            "Mining block #{i} to be considered a frequent miner";
+            "current_burn_height" => current_burn_height,
+        );
+        let commits_count = submitted_commits.load(Ordering::SeqCst);
+        next_block_and(&btc_controller, 60, || {
+            Ok(submitted_commits.load(Ordering::SeqCst) > commits_count)
+        })
+        .unwrap();
+    }
+
+    info!("---- Wait for tx replay set to be updated ----");
+
+    signer_test
+        .wait_for_signer_state_check(30, |state| {
+            let Some(tx_replay_set) = state.get_tx_replay_set() else {
+                return Ok(false);
+            };
+            let len_ok = tx_replay_set.len() == 1;
+            let txid_ok = tx_replay_set[0].txid().to_hex() == txid;
+            info!("---- Signer state check ----";
+                "tx_replay_set" => ?tx_replay_set,
+                "len_ok" => len_ok,
+                "txid_ok" => txid_ok,
+            );
+            Ok(len_ok && txid_ok)
+        })
+        .expect("Timed out waiting for tx replay set to be updated");
+
+    let tip_after_fork = get_chain_info(&conf);
+
+    info!("---- Waiting for two tenures, without replay set cleared ----";
+        "tip_after_fork.stacks_tip_height" => tip_after_fork.stacks_tip_height,
+        "tip_after_fork.burn_block_height" => tip_after_fork.burn_block_height
+    );
+
+    TEST_REJECT_REPLAY_TXS.set(true);
+    TEST_MINE_STALL.set(false);
+
+    wait_for(30, || {
+        let tip = get_chain_info(&conf);
+        Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height)
+    })
+    .expect("Timed out waiting for one TenureChange block to be mined");
+
+    signer_test
+        .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_some()))
+        .expect("Expected replay set to still be set");
+
+    info!("---- Mining a second tenure ----");
+
+    signer_test.mine_nakamoto_block(Duration::from_secs(30), true);
+
+    signer_test
+        .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_some()))
+        .expect("Expected replay set to still be set");
+
+    wait_for(30, || {
+        let tip = get_chain_info(&conf);
+        Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height + 1)
+    })
+    .expect("Timed out waiting for a TenureChange block to be mined");
+
+    info!("---- Mining a third tenure ----");
+    signer_test.mine_nakamoto_block(Duration::from_secs(30), true);
+
+    wait_for(30, || {
+        let tip = get_chain_info(&conf);
+        Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height + 1)
+    })
+    .expect("Timed out waiting for a TenureChange block to be mined");
+
+    info!("---- Waiting for tx replay set to be cleared ----");
+
+    signer_test
+        .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_none()))
+        .expect("Expected replay set to be cleared");
+
+    signer_test.shutdown();
+}
+
 /// Test scenario where two signers disagree on the tx replay set,
 /// which means there is no consensus on the tx replay set.
 #[test]
