diff --git a/stacks-node/src/burnchains/bitcoin_regtest_controller.rs b/stacks-node/src/burnchains/bitcoin_regtest_controller.rs index c14376399e..b663162d90 100644 --- a/stacks-node/src/burnchains/bitcoin_regtest_controller.rs +++ b/stacks-node/src/burnchains/bitcoin_regtest_controller.rs @@ -2816,6 +2816,17 @@ impl BitcoinRPCRequest { BitcoinRPCRequest::send(config, payload) } + pub fn get_chain_tips(config: &Config) -> RPCResult { + let payload = BitcoinRPCRequest { + method: "getchaintips".to_string(), + params: vec![], + id: "stacks".to_string(), + jsonrpc: "2.0".to_string(), + }; + + BitcoinRPCRequest::send(config, payload) + } + pub fn send(config: &Config, payload: BitcoinRPCRequest) -> RPCResult { let request = BitcoinRPCRequest::build_rpc_request(config, &payload); let timeout = Duration::from_secs(u64::from(config.burnchain.timeout)); diff --git a/stacks-node/src/tests/nakamoto_integrations.rs b/stacks-node/src/tests/nakamoto_integrations.rs index ba73bc7f83..ac9b3d21c8 100644 --- a/stacks-node/src/tests/nakamoto_integrations.rs +++ b/stacks-node/src/tests/nakamoto_integrations.rs @@ -111,6 +111,7 @@ use stacks_common::util::secp256k1::{MessageSignature, Secp256k1PrivateKey, Secp use stacks_common::util::{get_epoch_time_secs, sleep_ms}; use stacks_signer::chainstate::v1::SortitionsView; use stacks_signer::chainstate::ProposalEvalConfig; +use stacks_signer::config::DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS; use stacks_signer::signerdb::{BlockInfo, BlockState, ExtraBlockInfo, SignerDb}; use stacks_signer::v0::SpawnedSigner; @@ -6663,6 +6664,7 @@ fn signer_chainstate() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut sortitions_view = SortitionsView::fetch_view(proposal_conf, &signer_client).unwrap(); @@ -6773,6 +6775,7 @@ fn signer_chainstate() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let burn_block_height = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) .unwrap() @@ -6845,6 +6848,7 @@ fn signer_chainstate() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut sortitions_view = SortitionsView::fetch_view(proposal_conf, &signer_client).unwrap(); sortitions_view diff --git a/stacks-node/src/tests/signer/mod.rs b/stacks-node/src/tests/signer/mod.rs index 35652042ed..361712d55f 100644 --- a/stacks-node/src/tests/signer/mod.rs +++ b/stacks-node/src/tests/signer/mod.rs @@ -35,6 +35,7 @@ use libsigner::v0::messages::{ use libsigner::v0::signer_state::MinerState; use libsigner::{BlockProposal, SignerEntries, SignerEventTrait}; use serde::{Deserialize, Serialize}; +use stacks::burnchains::Txid; use stacks::chainstate::coordinator::comm::CoordinatorChannels; use stacks::chainstate::nakamoto::signer_set::NakamotoSigners; use stacks::chainstate::nakamoto::NakamotoBlock; @@ -239,6 +240,9 @@ impl SignerTest { let (mut naka_conf, _miner_account) = naka_neon_integration_conf(snapshot_name.map(|n| n.as_bytes())); + naka_conf.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", naka_conf.node.working_dir)); + node_config_modifier(&mut naka_conf); // Add initial balances to the config @@ -364,7 +368,11 @@ impl SignerTest { let metadata_path = snapshot_path.join("metadata.json"); if !metadata_path.clone().exists() { warn!("Snapshot metadata file does not exist, not restoring snapshot"); - return SetupSnapshotResult::NoSnapshot; + std::fs::remove_dir_all(snapshot_path.clone()).unwrap(); + return SetupSnapshotResult::WithSnapshot(SnapshotSetupInfo { + snapshot_path: snapshot_path.clone(), + snapshot_exists: false, + }); } let Ok(metadata) = serde_json::from_reader::<_, SnapshotMetadata>( File::open(metadata_path.clone()).unwrap(), @@ -1066,6 +1074,20 @@ impl SignerTest { }) } + pub fn wait_for_replay_set_eq(&self, timeout: u64, expected_txids: Vec) { + self.wait_for_signer_state_check(timeout, |state| { + let Some(replay_set) = state.get_tx_replay_set() else { + return Ok(false); + }; + let txids = replay_set + .iter() + .map(|tx| tx.txid().to_hex()) + .collect::>(); + Ok(txids == expected_txids) + }) + .expect("Timed out waiting for replay set to be equal to expected txids"); + } + /// Replace the test's configured signer st pub fn replace_signers( &mut self, @@ -1584,6 +1606,31 @@ impl SignerTest { .send_message_with_retry::(accepted.into()) .expect("Failed to send accept signature"); } + + /// Get the txid of the parent block commit transaction for the given miner + pub fn get_parent_block_commit_txid(&self, miner_pk: &StacksPublicKey) -> Option { + let Some(confirmed_utxo) = self + .running_nodes + .btc_regtest_controller + .get_all_utxos(&miner_pk) + .into_iter() + .find(|utxo| utxo.confirmations == 0) + else { + return None; + }; + let unconfirmed_txid = Txid::from_bitcoin_tx_hash(&confirmed_utxo.txid); + let unconfirmed_tx = self + .running_nodes + .btc_regtest_controller + .get_raw_transaction(&unconfirmed_txid); + let parent_txid = unconfirmed_tx + .input + .get(0) + .expect("First input should exist") + .previous_output + .txid; + Some(Txid::from_bitcoin_tx_hash(&parent_txid)) + } } fn setup_stx_btc_node( diff --git a/stacks-node/src/tests/signer/v0.rs b/stacks-node/src/tests/signer/v0.rs index f66e9cc192..1544b73019 100644 --- a/stacks-node/src/tests/signer/v0.rs +++ b/stacks-node/src/tests/signer/v0.rs @@ -63,8 +63,8 @@ use stacks::core::{StacksEpochId, CHAIN_ID_TESTNET, HELIUM_BLOCK_LIMIT_20}; use stacks::libstackerdb::StackerDBChunkData; use stacks::net::api::getsigner::GetSignerResponse; use stacks::net::api::postblock_proposal::{ - BlockValidateResponse, ValidateRejectCode, TEST_VALIDATE_DELAY_DURATION_SECS, - TEST_VALIDATE_STALL, + BlockValidateResponse, ValidateRejectCode, TEST_REJECT_REPLAY_TXS, + TEST_VALIDATE_DELAY_DURATION_SECS, TEST_VALIDATE_STALL, }; use stacks::net::relay::fault_injection::{clear_ignore_block, set_ignore_block}; use stacks::types::chainstate::{ @@ -85,7 +85,10 @@ use stacks_common::util::sleep_ms; use stacks_signer::chainstate::v1::SortitionsView; use stacks_signer::chainstate::ProposalEvalConfig; use stacks_signer::client::StackerDB; -use stacks_signer::config::{build_signer_config_tomls, GlobalConfig as SignerConfig, Network}; +use stacks_signer::config::{ + build_signer_config_tomls, GlobalConfig as SignerConfig, Network, + DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, +}; use stacks_signer::signerdb::SignerDb; use stacks_signer::v0::signer::TEST_REPEAT_PROPOSAL_RESPONSE; use stacks_signer::v0::signer_state::{ @@ -101,6 +104,7 @@ use tracing_subscriber::prelude::*; use tracing_subscriber::{fmt, EnvFilter}; use super::SignerTest; +use crate::burnchains::bitcoin_regtest_controller::BitcoinRPCRequest; use crate::event_dispatcher::{ EventObserver, MinedNakamotoBlockEvent, TEST_SKIP_BLOCK_ANNOUNCEMENT, }; @@ -1723,6 +1727,7 @@ fn block_proposal_rejection() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -3259,37 +3264,46 @@ fn tx_replay_forking_test() { } let num_signers = 5; - let sender_sk = Secp256k1PrivateKey::random(); + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender_addr = tests::to_addr(&sender_sk); let send_amt = 100; let send_fee = 180; let deploy_fee = 1000000; let call_fee = 1000; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![( - sender_addr, - (send_amt + send_fee) * 10 + deploy_fee + call_fee, - )], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![( + sender_addr, + (send_amt + send_fee) * 10 + deploy_fee + call_fee, + )], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let http_origin = format!("http://{}", &conf.node.rpc_bind); let stacks_miner_pk = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); - let pre_fork_tenures = 10; + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + + info!("------------------------- Beginning test -------------------------"); + + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); @@ -3315,49 +3329,26 @@ fn tx_replay_forking_test() { info!("------------------------- Triggering Bitcoin Fork -------------------------"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let tip_before = signer_test.get_peer_info(); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); + fault_injection_stall_miner(); + btc_controller.build_next_block(2); info!("Wait for block off of shallow fork"); - fault_injection_stall_miner(); - - let submitted_commits = signer_test - .running_nodes - .counters - .naka_submitted_commits - .clone(); + signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + btc_controller.build_next_block(1); + wait_for(30, || { + let tip = signer_test.get_peer_info(); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for stacks tip to decrease"); let post_fork_1_nonce = get_account(&http_origin, &sender_addr).nonce; - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == txid; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); // We should have forked 1 tx assert_eq!(post_fork_1_nonce, pre_fork_1_nonce - 1); @@ -3417,39 +3408,23 @@ fn tx_replay_forking_test() { fault_injection_stall_miner(); + info!("---- Triggering deeper fork ----"); + + let tip_before = signer_test.get_peer_info(); + let burn_header_hash_to_fork = btc_controller.get_block_hash(pre_fork_2_tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(4); - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and( - &signer_test.running_nodes.btc_regtest_controller, - 60, - || Ok(submitted_commits.load(Ordering::SeqCst) > commits_count), - ) - .unwrap(); - } + wait_for(30, || { + let tip = signer_test.get_peer_info(); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for stacks tip to decrease"); let expected_tx_replay_txids = vec![transfer_txid, contract_deploy_txid, contract_call_txid]; - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let tx_replay_set_txids = tx_replay_set - .iter() - .map(|tx| tx.txid().to_hex()) - .collect::>(); - Ok(tx_replay_set_txids == expected_tx_replay_txids) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, expected_tx_replay_txids.clone()); info!("---- Mining post-fork block to clear tx replay set ----"); let tip_after_fork = get_chain_info(&conf); @@ -3525,37 +3500,46 @@ fn tx_replay_reject_invalid_proposals_during_replay() { } let num_signers = 5; - let sender_sk = Secp256k1PrivateKey::random(); + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender_addr = tests::to_addr(&sender_sk); - let sender_sk2 = Secp256k1PrivateKey::random(); + let sender_sk2 = Secp256k1PrivateKey::from_seed("sender_2".as_bytes()); let sender_addr2 = tests::to_addr(&sender_sk2); let send_amt = 100; let send_fee = 180; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![ - (sender_addr, send_amt + send_fee), - (sender_addr2, send_amt + send_fee), - ], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![ + (sender_addr, send_amt + send_fee), + (sender_addr2, send_amt + send_fee), + ], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; let stacks_miner_pk = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); - let pre_fork_tenures = 10; + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + + info!("------------------------- Beginning test -------------------------"); + + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); @@ -3579,50 +3563,17 @@ fn tx_replay_reject_invalid_proposals_during_replay() { info!("------------------------- Triggering Bitcoin Fork -------------------------"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); - fault_injection_stall_miner(); + btc_controller.build_next_block(2); - let submitted_commits = signer_test - .running_nodes - .counters - .naka_submitted_commits - .clone(); + info!("Wait for block off of shallow fork"); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); let post_fork_1_nonce = get_account(&http_origin, &sender_addr).nonce; - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == txid; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); - // We should have forked 1 tx assert_eq!(post_fork_1_nonce, pre_fork_1_nonce - 1); @@ -3748,6 +3699,7 @@ fn tx_replay_btc_on_stx_invalidation() { vec![(sender_addr, (send_amt + send_fee) * 10)], |c| { c.validate_with_replay_tx = true; + c.reset_replay_set_after_fork_blocks = 5; }, |node_config| { node_config.miner.block_commit_delay = Duration::from_secs(1); @@ -3811,7 +3763,7 @@ fn tx_replay_btc_on_stx_invalidation() { "Pre-stx operation should submit successfully" ); - let pre_fork_tenures = 9; + let pre_fork_tenures = 10; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -3918,6 +3870,9 @@ fn tx_replay_btc_on_stx_invalidation() { }) .expect("Timed out waiting for block to advance by 1"); + let account = get_account(&_http_origin, &recipient_addr); + assert_eq!(account.nonce, 0, "Expected recipient nonce to be 0"); + let blocks = test_observer::get_blocks(); let block: StacksBlockEvent = serde_json::from_value(blocks.last().unwrap().clone()).expect("Failed to parse block"); @@ -3937,62 +3892,361 @@ fn tx_replay_btc_on_stx_invalidation() { signer_test.shutdown(); } -/// Test scenario where two signers disagree on the tx replay set, -/// which means there is no consensus on the tx replay set. -#[test] +/// Test scenario to ensure that the replay set is cleared +/// if there have been multiple tenures with a stalled replay set. +/// +/// This test is executed by triggering a fork, and then using +/// a test flag to reject any transaction replay blocks. +/// +/// The test mines a number of burn blocks during replay before +/// validating that the replay set is eventually cleared. #[ignore] -fn tx_replay_disagreement() { +#[test] +fn tx_replay_failsafe() { if env::var("BITCOIND_TEST") != Ok("1".into()) { return; } let num_signers = 5; - let mut miners = MultipleMinerTest::new_with_config_modifications( - num_signers, - 10, - |config| { - config.validate_with_replay_tx = true; - }, - |config| { - config.burnchain.pox_reward_length = Some(30); - config.miner.block_commit_delay = Duration::from_secs(0); - config.miner.tenure_cost_limit_per_block_percentage = None; - config.miner.replay_transactions = true; - }, - |config| { - config.burnchain.pox_reward_length = Some(30); - config.miner.block_commit_delay = Duration::from_secs(0); - config.miner.tenure_cost_limit_per_block_percentage = None; - config.miner.replay_transactions = true; - }, - ); - - let (conf_1, _conf_2) = miners.get_node_configs(); - let _skip_commit_op_rl1 = miners - .signer_test - .running_nodes - .counters - .naka_skip_commit_op - .clone(); - let skip_commit_op_rl2 = miners.rl2_counters.naka_skip_commit_op.clone(); + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); + let sender_addr = tests::to_addr(&sender_sk); + let send_amt = 100; + let send_fee = 180; + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender_addr, (send_amt + send_fee) * 10)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); - // Make sure that the first miner wins the first sortition. - info!("Pausing miner 2's block commit submissions"); - skip_commit_op_rl2.set(true); - miners.boot_to_epoch_3(); - let btc_controller = &miners.signer_test.running_nodes.btc_regtest_controller; + let conf = &signer_test.running_nodes.conf; + let _http_origin = format!("http://{}", &conf.node.rpc_bind); + let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let pre_fork_tenures = 10; + let miner_pk = btc_controller + .get_mining_pubkey() + .as_deref() + .map(Secp256k1PublicKey::from_hex) + .unwrap() + .unwrap(); - for i in 0..pre_fork_tenures { - info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); - miners - .signer_test - .mine_nakamoto_block(Duration::from_secs(30), false); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; } - let ignore_bitcoin_fork_keys = miners - .signer_test + info!("------------------------- Beginning test -------------------------"); + + let burnchain = conf.get_burnchain(); + + let tip = signer_test.get_peer_info(); + let pox_info = signer_test.get_pox_data(); + + info!("---- Burnchain ----"; + // "burnchain" => ?conf.burnchain, + "pox_constants" => ?burnchain.pox_constants, + "cycle" => burnchain.pox_constants.reward_cycle_index(0, tip.burn_block_height), + "pox_info" => ?pox_info, + ); + + let pre_fork_tenures = 3; + for i in 0..pre_fork_tenures { + info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); + } + + info!("---- Submitting STX transfer ----"); + + let tip = get_chain_info(&conf); + // Make a transfer tx (this will get forked) + let (txid, nonce) = signer_test + .submit_transfer_tx(&sender_sk, send_fee, send_amt) + .unwrap(); + + // Ensure we got a new block with this tx + signer_test + .wait_for_nonce_increase(&sender_addr, nonce) + .expect("Timed out waiting for transfer tx to be mined"); + + wait_for(30, || { + let new_tip = get_chain_info(&conf); + Ok(new_tip.stacks_tip_height > tip.stacks_tip_height) + }) + .expect("Timed out waiting for transfer tx to be mined"); + + let tip_before = get_chain_info(&conf); + + info!("---- Triggering Bitcoin fork ----"; + "tip.stacks_tip_height" => tip_before.stacks_tip_height, + "tip.burn_block_height" => tip_before.burn_block_height, + ); + + let mut commit_txid: Option = None; + wait_for(30, || { + let Some(txid) = signer_test.get_parent_block_commit_txid(&miner_pk) else { + return Ok(false); + }; + commit_txid = Some(txid); + Ok(true) + }) + .expect("Failed to get unconfirmed tx"); + + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_before.burn_block_height); + btc_controller.invalidate_block(&burn_header_hash_to_fork); + btc_controller.build_next_block(1); + + fault_injection_stall_miner(); + + // Wait for the block commit re-broadcast to be confirmed + wait_for(10, || { + let is_confirmed = + BitcoinRPCRequest::check_transaction_confirmed(&conf, &commit_txid.unwrap()).unwrap(); + Ok(is_confirmed) + }) + .expect("Timed out waiting for transaction to be confirmed"); + + let tip_before = get_chain_info(&conf); + + info!("---- Building next block ----"; + "tip_before.stacks_tip_height" => tip_before.stacks_tip_height, + "tip_before.burn_block_height" => tip_before.burn_block_height, + ); + + btc_controller.build_next_block(1); + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for next block to be mined"); + + info!("---- Wait for tx replay set to be updated ----"); + + signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); + + let tip_after_fork = get_chain_info(&conf); + + info!("---- Waiting for two tenures, without replay set cleared ----"; + "tip_after_fork.stacks_tip_height" => tip_after_fork.stacks_tip_height, + "tip_after_fork.burn_block_height" => tip_after_fork.burn_block_height + ); + + TEST_REJECT_REPLAY_TXS.set(true); + fault_injection_unstall_miner(); + + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height) + }) + .expect("Timed out waiting for one TenureChange block to be mined"); + + signer_test + .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_some())) + .expect("Expected replay set to still be set"); + + info!("---- Mining a second tenure ----"); + + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); + + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height + 1) + }) + .expect("Timed out waiting for a TenureChange block to be mined"); + + signer_test + .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_some())) + .expect("Expected replay set to still be set"); + + info!("---- Mining a third tenure ----"); + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); + + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height + 2) + }) + .expect("Timed out waiting for a TenureChange block to be mined"); + + info!("---- Waiting for tx replay set to be cleared ----"); + + signer_test + .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_none())) + .expect("Expected replay set to be cleared"); + + signer_test.shutdown(); +} + +/// Simple/fast test scenario for transaction replay. +/// +/// We fork one tenure, which has a STX transfer. The test +/// verifies that the replay set is updated correctly, and then +/// exits. +#[ignore] +#[test] +fn tx_replay_starts_correctly() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let num_signers = 5; + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); + let sender_addr = tests::to_addr(&sender_sk); + let send_amt = 100; + let send_fee = 180; + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender_addr, (send_amt + send_fee) * 10)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); + + let conf = &signer_test.running_nodes.conf; + let _http_origin = format!("http://{}", &conf.node.rpc_bind); + let btc_controller = &signer_test.running_nodes.btc_regtest_controller; + + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + + info!("------------------------- Beginning test -------------------------"); + + let tip = signer_test.get_peer_info(); + + info!("---- Tip ----"; + "tip.stacks_tip_height" => tip.stacks_tip_height, + "tip.burn_block_height" => tip.burn_block_height, + ); + + let pre_fork_tenures = 1; + for i in 0..pre_fork_tenures { + info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); + } + + info!("---- Submitting STX transfer ----"); + + // let tip = get_chain_info(&conf); + // Make a transfer tx (this will get forked) + let (txid, nonce) = signer_test + .submit_transfer_tx(&sender_sk, send_fee, send_amt) + .unwrap(); + + // Ensure we got a new block with this tx + signer_test + .wait_for_nonce_increase(&sender_addr, nonce) + .expect("Timed out waiting for transfer tx to be mined"); + + let tip_before = get_chain_info(&conf); + + info!("---- Triggering Bitcoin fork ----"; + "tip.stacks_tip_height" => tip_before.stacks_tip_height, + "tip.burn_block_height" => tip_before.burn_block_height, + "tip.consensus_hash" => %tip_before.pox_consensus, + ); + + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_before.burn_block_height); + btc_controller.invalidate_block(&burn_header_hash_to_fork); + fault_injection_stall_miner(); + btc_controller.build_next_block(2); + + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for next block to be mined"); + + let tip = get_chain_info(&conf); + + info!("---- Tip after fork ----"; + "tip.stacks_tip_height" => tip.stacks_tip_height, + "tip.burn_block_height" => tip.burn_block_height, + ); + + info!("---- Wait for tx replay set to be updated ----"); + + signer_test.wait_for_replay_set_eq(5, vec![txid.clone()]); + + signer_test.shutdown(); +} + +/// Test scenario where two signers disagree on the tx replay set, +/// which means there is no consensus on the tx replay set. +#[test] +#[ignore] +fn tx_replay_disagreement() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let num_signers = 5; + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); + let sender_addr = tests::to_addr(&sender_sk); + let send_amt = 100; + let send_fee = 180; + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender_addr, (send_amt + send_fee) * 10)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); + + let conf = &signer_test.running_nodes.conf; + let _http_origin = format!("http://{}", &conf.node.rpc_bind); + let btc_controller = &signer_test.running_nodes.btc_regtest_controller; + + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + + info!("------------------------- Beginning test -------------------------"); + + let miner_pk = btc_controller + .get_mining_pubkey() + .as_deref() + .map(Secp256k1PublicKey::from_hex) + .unwrap() + .unwrap(); + + let pre_fork_tenures = 2; + + for i in 0..pre_fork_tenures { + info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); + } + + let ignore_bitcoin_fork_keys = signer_test .signer_stacks_private_keys .iter() .enumerate() @@ -4007,58 +4261,68 @@ fn tx_replay_disagreement() { TEST_IGNORE_BITCOIN_FORK_PUBKEYS.set(ignore_bitcoin_fork_keys); info!("------------------------- Triggering Bitcoin Fork -------------------------"); - let tip = get_chain_info(&conf_1); + let tip = get_chain_info(&conf); wait_for_state_machine_update_by_miner_tenure_id( 30, &tip.pox_consensus, - &miners.signer_test.signer_addresses_versions(), + &signer_test.signer_addresses_versions(), ) .expect("Failed to update signers state machines"); // Make a transfer tx (this will get forked) - let (txid, _) = miners.send_transfer_tx(); + let (txid, _) = signer_test + .submit_transfer_tx(&sender_sk, send_fee, send_amt) + .unwrap(); wait_for(30, || { - let new_tip = get_chain_info(&conf_1); + let new_tip = get_chain_info(&conf); Ok(new_tip.stacks_tip_height > tip.stacks_tip_height) }) .expect("Timed out waiting for transfer tx to be mined"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let mut commit_txid: Option = None; + wait_for(30, || { + let Some(txid) = signer_test.get_parent_block_commit_txid(&miner_pk) else { + return Ok(false); + }; + commit_txid = Some(txid); + Ok(true) + }) + .expect("Failed to get unconfirmed tx"); + + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(1); - // note, we should still have normal signer states! - miners.signer_test.check_signer_states_normal(); + // Wait for the block commit re-broadcast to be confirmed + wait_for(10, || { + let is_confirmed = + BitcoinRPCRequest::check_transaction_confirmed(&conf, &commit_txid.unwrap()).unwrap(); + Ok(is_confirmed) + }) + .expect("Timed out waiting for transaction to be confirmed"); - info!("Wait for block off of shallow fork"); + let tip_before = get_chain_info(&conf); - fault_injection_stall_miner(); + info!("---- Building next block ----"; + "tip_before.stacks_tip_height" => tip_before.stacks_tip_height, + "tip_before.burn_block_height" => tip_before.burn_block_height, + ); - let submitted_commits = miners - .signer_test - .running_nodes - .counters - .naka_submitted_commits - .clone(); + btc_controller.build_next_block(1); + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for next block to be mined"); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf_1).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + fault_injection_stall_miner(); + + btc_controller.build_next_block(1); // Wait for the signer states to be updated. Odd indexed signers // should not have a replay set. wait_for(30, || { - let (signer_states, _) = miners.signer_test.get_burn_updated_states(); + let (signer_states, _) = signer_test.get_burn_updated_states(); let all_pass = signer_states.iter().enumerate().all(|(i, state)| { if i % 2 == 0 { let Some(tx_replay_set) = state.get_tx_replay_set() else { @@ -4073,27 +4337,26 @@ fn tx_replay_disagreement() { }) .expect("Timed out waiting for signer states to be updated"); - let tip = get_chain_info(&conf_1); + let tip = get_chain_info(&conf); fault_injection_unstall_miner(); // Now, wait for the tx replay set to be cleared wait_for(30, || { - let new_tip = get_chain_info(&conf_1); + let new_tip = get_chain_info(&conf); Ok(new_tip.stacks_tip_height >= tip.stacks_tip_height + 2) }) .expect("Timed out waiting for transfer tx to be mined"); - miners - .signer_test + signer_test .wait_for_signer_state_check(30, |state| { let tx_replay_set = state.get_tx_replay_set(); Ok(tx_replay_set.is_none()) }) .expect("Timed out waiting for tx replay set to be cleared"); - miners.shutdown(); + signer_test.shutdown(); } #[test] @@ -4105,7 +4368,6 @@ fn tx_replay_disagreement() { /// The test flow is: /// /// - Boot to Epoch 3 -/// - Mine 3 tenures /// - Submit 2 STX Transfer txs (Tx1, Tx2) in the last tenure /// - Trigger a Bitcoin fork (3 blocks) /// - Verify that signers move into tx replay state [Tx1, Tx2] @@ -4116,33 +4378,39 @@ fn tx_replay_solved_by_mempool_txs() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 2; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); - let pre_fork_tenures = 3; + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -4169,41 +4437,15 @@ fn tx_replay_solved_by_mempool_txs() { info!("------------------------- Triggering Bitcoin Fork -------------------------"); let tip = get_chain_info(&conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); + fault_injection_stall_miner(); + btc_controller.build_next_block(2); info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1 - && tx_replay_set[1].txid().to_hex() == sender1_tx2; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone(), sender1_tx2.clone()]); // We should have forked 2 txs let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; @@ -4376,31 +4618,37 @@ fn tx_replay_with_fork_occured_before_starting_replaying_txs() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 1; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); let pre_fork_tenures = 12; //go to 2nd tenure of 12th cycle for i in 0..pre_fork_tenures { @@ -4419,43 +4667,17 @@ fn tx_replay_with_fork_occured_before_starting_replaying_txs() { let sender1_nonce = get_account(&http_origin, &sender1_addr).nonce; assert_eq!(1, sender1_nonce); - info!("------------------------- Triggering Bitcoin Fork #1 -------------------------"); - let tip = get_chain_info(&conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); - btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); + info!("------------------------- Triggering Bitcoin Fork #1 -------------------------"); + let tip = get_chain_info(&conf); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); + btc_controller.invalidate_block(&burn_header_hash_to_fork); fault_injection_stall_miner(); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + info!("Wait for block off of shallow fork"); // Signers move in Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); // We should have forked 1 tx let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; @@ -4465,37 +4687,11 @@ fn tx_replay_with_fork_occured_before_starting_replaying_txs() { let tip = get_chain_info(&conf); let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); - - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + btc_controller.build_next_block(2); //Signers still are in the initial state of Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); info!("----------- Solve TX Replay ------------"); fault_injection_unstall_miner(); @@ -4533,31 +4729,36 @@ fn tx_replay_with_fork_after_empty_tenures_before_starting_replaying_txs() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 1; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + c.reset_replay_set_after_fork_blocks = 5; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); let pre_fork_tenures = 10; //go to Tenure #4 in Cycle #12 for i in 0..pre_fork_tenures { @@ -4579,41 +4780,15 @@ fn tx_replay_with_fork_after_empty_tenures_before_starting_replaying_txs() { info!("------------------------- Triggering Bitcoin Fork #1 -------------------------"); let tip = get_chain_info(&conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + info!("Wait for block off of shallow fork"); // Signers moved in Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); // We should have forked tx1 let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; @@ -4625,17 +4800,6 @@ fn tx_replay_with_fork_after_empty_tenures_before_starting_replaying_txs() { _ = wait_for_tenure_change_tx(30, TenureChangeCause::BlockFound, tip.stacks_tip_height + 1); fault_injection_stall_miner(); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - - fault_injection_unstall_miner(); - let tip = get_chain_info(&conf); - _ = wait_for_tenure_change_tx(30, TenureChangeCause::BlockFound, tip.stacks_tip_height + 1); - fault_injection_stall_miner(); - signer_test .wait_for_signer_state_check(30, |state| { let Some(tx_replay_set) = state.get_tx_replay_set() else { @@ -4653,36 +4817,13 @@ fn tx_replay_with_fork_after_empty_tenures_before_starting_replaying_txs() { let tip = get_chain_info(&conf); let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + fault_injection_stall_miner(); + btc_controller.build_next_block(2); info!("Wait for block off of shallow fork"); - fault_injection_stall_miner(); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } // Signers still are in Tx Replay mode (as the initial replay state) - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); info!("------------------------- Mine Tx Replay Set -------------------------"); fault_injection_unstall_miner(); @@ -4716,37 +4857,40 @@ fn tx_replay_with_fork_causing_replay_set_to_be_updated() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 2; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); let pre_fork_tenures = 10; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.check_signer_states_normal(); } // Make 2 transfer txs, each in its own tenure so that can be forked in different forks @@ -4760,11 +4904,9 @@ fn tx_replay_with_fork_causing_replay_set_to_be_updated() { .expect("Expect sender1 nonce increased"); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.mine_nakamoto_block(Duration::from_secs(30), true); let tip_at_tx2 = get_chain_info(&conf); - assert_eq!(244, tip_at_tx2.burn_block_height); + assert_eq!(242, tip_at_tx2.burn_block_height); let (sender1_tx2, sender1_nonce) = signer_test .submit_transfer_tx(&sender1_sk, send_fee, send_amt) .unwrap(); @@ -4776,82 +4918,46 @@ fn tx_replay_with_fork_causing_replay_set_to_be_updated() { assert_eq!(2, sender1_nonce); info!("------------------------- Triggering Bitcoin Fork #1 -------------------------"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_tx2.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_tx2.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); + btc_controller.build_next_block(1); info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - assert_eq!(247, get_chain_info(&conf).burn_block_height); + btc_controller.build_next_block(1); + + wait_for(10, || { + let tip = get_chain_info(&conf); + Ok(tip.burn_block_height == 243) + }) + .expect("Timed out waiting for burn block height to be 243"); // Signers move in Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx2; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx2.clone()]); // We should have forked one tx (Tx2) let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; assert_eq!(1, sender1_nonce_post_fork); - info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); + info!( + "------------------------- Triggering Bitcoin Fork #2 from {} -------------------------", + tip_at_tx1.burn_block_height + ); let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_tx1.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(7); + btc_controller.build_next_block(4); + wait_for(10, || { + let tip = get_chain_info(&conf); + info!("Burn block height: {}", tip.burn_block_height); + Ok(tip.burn_block_height == 244) + }) + .expect("Timed out waiting for burn block height to be 244"); info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - assert_eq!(250, get_chain_info(&conf).burn_block_height); - //Signers should update the Tx Replay Set - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1 - && tx_replay_set[1].txid().to_hex() == sender1_tx2; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone(), sender1_tx2.clone()]); info!("----------- Solve TX Replay ------------"); fault_injection_unstall_miner(); @@ -4888,31 +4994,35 @@ fn tx_replay_with_fork_causing_replay_to_be_cleared_due_to_cycle() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 2; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); let pre_fork_tenures = 8; for i in 0..pre_fork_tenures { @@ -4945,40 +5055,13 @@ fn tx_replay_with_fork_causing_replay_to_be_cleared_due_to_cycle() { assert_eq!(1, sender1_nonce); info!("------------------------- Triggering Bitcoin Fork #1 -------------------------"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_rc12.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_rc12.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + btc_controller.build_next_block(2); // Signers move in Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); // We should have forked one tx (Tx2) let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; @@ -4987,25 +5070,10 @@ fn tx_replay_with_fork_causing_replay_to_be_cleared_due_to_cycle() { info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_rc11.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(7); - - info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); + btc_controller.build_next_block(6); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + info!("Wait for block off of shallow fork"); //Signers should clear the Tx Replay Set signer_test @@ -5025,7 +5093,6 @@ fn tx_replay_with_fork_causing_replay_to_be_cleared_due_to_cycle() { /// The test flow is: /// /// - Boot to Epoch 3 -/// - Mine 10 tenures (to handle multiple fork in Cycle 12) /// - Deploy 1 Big Contract and mine 2 tenures (to escape fork) /// - Submit 2 Contract Call txs (Tx1, Tx2) in the last tenure, /// requiring Tenure Extend due to Tenure Budget exceeded @@ -5041,35 +5108,40 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { } let num_signers = 5; - let sender_sk = Secp256k1PrivateKey::random(); + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender_addr = tests::to_addr(&sender_sk); let deploy_fee = 1000000; let call_fee = 1000; let call_num = 2; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender_addr, deploy_fee + call_fee * call_num)], - |c| { - c.validate_with_replay_tx = true; - c.tenure_idle_timeout = Duration::from_secs(10); - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender_addr, deploy_fee + call_fee * call_num)], + |c| { + c.validate_with_replay_tx = true; + c.tenure_idle_timeout = Duration::from_secs(10); + c.reset_replay_set_after_fork_blocks = 5; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let stacks_miner_pk = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); - let pre_fork_tenures = 10; + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -5092,7 +5164,6 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { .expect("Timed out waiting for nonce to increase"); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.mine_nakamoto_block(Duration::from_secs(30), true); // Then, sumbmit 2 Contract Calls that require Tenure Extension to be addressed. info!("---- Submit big tx1 to be mined ----"); @@ -5104,12 +5175,13 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { .expect("Timed out waiting for nonce to increase"); info!("---- Submit big tx2 to be mined ----"); + let tip = get_chain_info(conf); + let (txid2, txid2_nonce) = signer_test .submit_contract_call(&sender_sk, call_fee, "big-contract", "big-tx", &vec![]) .unwrap(); // Tenure Extend happen because of tenure budget exceeded - let tip = get_chain_info(conf); _ = wait_for_tenure_change_tx(30, TenureChangeCause::Extended, tip.stacks_tip_height + 1); signer_test @@ -5121,39 +5193,12 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { info!("------------------------- Triggering Bitcoin Fork -------------------------"); let tip = get_chain_info(conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 1); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); let post_fork_nonce = get_account(&http_origin, &sender_addr).nonce; assert_eq!(1, post_fork_nonce); //due to contract deploy tx @@ -5167,54 +5212,19 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { fault_injection_stall_miner(); // Signers still waiting for the Tx Replay set to be completed - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); //Fork in the middle of Tx Replay let tip = get_chain_info(&conf); let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 1); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(2); info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); let post_fork_nonce = get_account(&http_origin, &sender_addr).nonce; assert_eq!(1, post_fork_nonce); //due to contract deploy tx @@ -5246,7 +5256,6 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { /// The test flow is: /// /// - Boot to Epoch 3 -/// - Mine 10 tenures (to handle multiple fork in Cycle 12) /// - Deploy 1 Big Contract and mine 2 tenures (to escape fork) /// - Submit 2 Contract Call txs (Tx1, Tx2) in the last tenure, /// requiring Tenure Extend due to Tenure Budget exceeded @@ -5265,45 +5274,49 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send1_deploy_fee = 1000000; let send1_call_fee = 1000; let send1_call_num = 2; - let sender2_sk = Secp256k1PrivateKey::random(); + let sender2_sk = Secp256k1PrivateKey::from_seed("sender_2".as_bytes()); let sender2_addr = tests::to_addr(&sender2_sk); let send2_amt = 100; let send2_fee = 180; let send2_txs = 1; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![ - ( - sender1_addr, - send1_deploy_fee + send1_call_fee * send1_call_num, - ), - (sender2_addr, (send2_amt + send2_fee) * send2_txs), - ], - |c| { - c.validate_with_replay_tx = true; - c.tenure_idle_timeout = Duration::from_secs(10); - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![ + ( + sender1_addr, + send1_deploy_fee + send1_call_fee * send1_call_num, + ), + (sender2_addr, (send2_amt + send2_fee) * send2_txs), + ], + |c| { + c.validate_with_replay_tx = true; + c.tenure_idle_timeout = Duration::from_secs(10); + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let stacks_miner_pk = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); - let pre_fork_tenures = 10; + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -5326,7 +5339,6 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted .expect("Timed out waiting for nonce to increase"); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.mine_nakamoto_block(Duration::from_secs(30), true); // Then, sumbmit 2 Contract Calls that require Tenure Extension to be addressed. info!("---- Waiting for first big tx to be mined ----"); @@ -5368,39 +5380,12 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted info!("------------------------- Triggering Bitcoin Fork -------------------------"); let tip = get_chain_info(conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 1); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); let post_fork_nonce = get_account(&http_origin, &sender1_addr).nonce; assert_eq!(1, post_fork_nonce); //due to contract deploy tx @@ -5414,17 +5399,7 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted fault_injection_stall_miner(); // Signers still waiting for the Tx Replay set to be completed - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); info!("---- New Transaction is Submitted ----"); // Tx3 reach the mempool, meanwhile mining is stalled @@ -5435,39 +5410,14 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); //Fork in the middle of Tx Replay let tip = get_chain_info(&conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 1); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(2); info!("Wait for block off of shallow fork"); fault_injection_stall_miner(); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; assert_eq!(1, sender1_nonce_post_fork); //due to contract deploy tx @@ -10613,6 +10563,7 @@ fn block_validation_response_timeout() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -10902,6 +10853,7 @@ fn block_validation_pending_table() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -12257,6 +12209,7 @@ fn incoming_signers_ignore_block_proposals() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -12432,6 +12385,7 @@ fn outgoing_signers_ignore_block_proposals() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), diff --git a/stacks-signer/CHANGELOG.md b/stacks-signer/CHANGELOG.md index b02a15af4f..32d0ea908d 100644 --- a/stacks-signer/CHANGELOG.md +++ b/stacks-signer/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE - Added `info` logs to the signer to provide more visibility into the block approval/rejection status - Introduced `capitulate_miner_view_timeout_secs`: the duration (in seconds) for the signer to wait between updating the local state machine viewpoint and capitulating to other signers' miner views. - Added codepath to enable signers to evaluate block proposals and miner activity against global signer state for improved consistency and correctness. Currently feature gated behind the `SUPPORTED_SIGNER_PROTOCOL_VERSION` +- When a transaction replay set has been active for a configurable number of burn blocks (which defaults to `2`), and the replay set still hasn't been cleared, the replay set is automatically cleared. This is provided as a "failsafe" to ensure chain liveness as transaction replay is rolled out. ### Changed diff --git a/stacks-signer/src/chainstate/mod.rs b/stacks-signer/src/chainstate/mod.rs index ff39ca9d28..8f68400d67 100644 --- a/stacks-signer/src/chainstate/mod.rs +++ b/stacks-signer/src/chainstate/mod.rs @@ -86,6 +86,9 @@ pub struct ProposalEvalConfig { pub reorg_attempts_activity_timeout: Duration, /// Time to wait before submitting a block proposal to the stacks-node pub proposal_wait_for_parent_time: Duration, + /// How many blocks after a fork should we reset the replay set, + /// as a failsafe mechanism + pub reset_replay_set_after_fork_blocks: u64, } impl From<&SignerConfig> for ProposalEvalConfig { @@ -98,6 +101,7 @@ impl From<&SignerConfig> for ProposalEvalConfig { reorg_attempts_activity_timeout: value.reorg_attempts_activity_timeout, tenure_idle_timeout_buffer: value.tenure_idle_timeout_buffer, proposal_wait_for_parent_time: value.proposal_wait_for_parent_time, + reset_replay_set_after_fork_blocks: value.reset_replay_set_after_fork_blocks, } } } diff --git a/stacks-signer/src/chainstate/tests/v1.rs b/stacks-signer/src/chainstate/tests/v1.rs index 1337b6967a..beba40bc54 100644 --- a/stacks-signer/src/chainstate/tests/v1.rs +++ b/stacks-signer/src/chainstate/tests/v1.rs @@ -45,6 +45,7 @@ use crate::chainstate::v1::{SortitionMinerStatus, SortitionState, SortitionsView use crate::chainstate::{ProposalEvalConfig, SortitionData}; use crate::client::tests::MockServerClient; use crate::client::StacksClient; +use crate::config::DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS; use crate::signerdb::{BlockInfo, SignerDb}; fn setup_test_environment( @@ -99,6 +100,7 @@ fn setup_test_environment( tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(3), proposal_wait_for_parent_time: Duration::from_secs(0), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }, }; diff --git a/stacks-signer/src/chainstate/tests/v2.rs b/stacks-signer/src/chainstate/tests/v2.rs index a5aee9c153..8198f1f9a0 100644 --- a/stacks-signer/src/chainstate/tests/v2.rs +++ b/stacks-signer/src/chainstate/tests/v2.rs @@ -47,6 +47,7 @@ use crate::chainstate::v2::{GlobalStateView, SortitionState}; use crate::chainstate::{ProposalEvalConfig, SignerChainstateError, SortitionData}; use crate::client::tests::MockServerClient; use crate::client::StacksClient; +use crate::config::DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS; use crate::signerdb::tests::tmp_db_path; use crate::signerdb::{BlockInfo, SignerDb}; @@ -94,6 +95,7 @@ fn setup_test_environment( tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(3), proposal_wait_for_parent_time: Duration::from_secs(0), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let stacks_client = StacksClient::new( diff --git a/stacks-signer/src/client/mod.rs b/stacks-signer/src/client/mod.rs index 974297cb9e..9431bf6365 100644 --- a/stacks-signer/src/client/mod.rs +++ b/stacks-signer/src/client/mod.rs @@ -434,6 +434,7 @@ pub(crate) mod tests { reorg_attempts_activity_timeout: config.reorg_attempts_activity_timeout, proposal_wait_for_parent_time: config.proposal_wait_for_parent_time, validate_with_replay_tx: config.validate_with_replay_tx, + reset_replay_set_after_fork_blocks: config.reset_replay_set_after_fork_blocks, capitulate_miner_view_timeout: config.capitulate_miner_view_timeout, #[cfg(any(test, feature = "testing"))] supported_signer_protocol_version: SUPPORTED_SIGNER_PROTOCOL_VERSION, diff --git a/stacks-signer/src/config.rs b/stacks-signer/src/config.rs index f1eb4cbb95..2bac62ca47 100644 --- a/stacks-signer/src/config.rs +++ b/stacks-signer/src/config.rs @@ -51,6 +51,9 @@ const DEFAULT_TENURE_IDLE_TIMEOUT_BUFFER_SECS: u64 = 2; /// cannot determine that our stacks-node has processed the parent /// block const DEFAULT_PROPOSAL_WAIT_TIME_FOR_PARENT_SECS: u64 = 15; +/// Default number of blocks after a fork to reset the replay set, +/// as a failsafe mechanism +pub const DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS: u64 = 2; /// Default time (in secs) to wait between updating our local state /// machine view point and capitulating to other signers tenure view const DEFAULT_CAPITULATE_MINER_VIEW_SECS: u64 = 20; @@ -189,6 +192,9 @@ pub struct SignerConfig { pub proposal_wait_for_parent_time: Duration, /// Whether or not to validate blocks with replay transactions pub validate_with_replay_tx: bool, + /// How many blocks after a fork should we reset the replay set, + /// as a failsafe mechanism + pub reset_replay_set_after_fork_blocks: u64, /// Time to wait between updating our local state machine view point and capitulating to other signers miner view pub capitulate_miner_view_timeout: Duration, #[cfg(any(test, feature = "testing"))] @@ -247,6 +253,9 @@ pub struct GlobalConfig { pub dry_run: bool, /// Whether or not to validate blocks with replay transactions pub validate_with_replay_tx: bool, + /// How many blocks after a fork should we reset the replay set, + /// as a failsafe mechanism + pub reset_replay_set_after_fork_blocks: u64, /// Time to wait between updating our local state machine view point and capitulating to other signers miner view pub capitulate_miner_view_timeout: Duration, #[cfg(any(test, feature = "testing"))] @@ -303,6 +312,9 @@ struct RawConfigFile { pub dry_run: Option, /// Whether or not to validate blocks with replay transactions pub validate_with_replay_tx: Option, + /// How many blocks after a fork should we reset the replay set, + /// as a failsafe mechanism + pub reset_replay_set_after_fork_blocks: Option, /// Time to wait (in secs) between updating our local state machine view point and capitulating to other signers miner view pub capitulate_miner_view_timeout_secs: Option, #[cfg(any(test, feature = "testing"))] @@ -433,6 +445,10 @@ impl TryFrom for GlobalConfig { // https://github.com/stacks-network/stacks-core/issues/6087 let validate_with_replay_tx = raw_data.validate_with_replay_tx.unwrap_or(false); + let reset_replay_set_after_fork_blocks = raw_data + .reset_replay_set_after_fork_blocks + .unwrap_or(DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS); + let capitulate_miner_view_timeout = Duration::from_secs( raw_data .capitulate_miner_view_timeout_secs @@ -466,6 +482,7 @@ impl TryFrom for GlobalConfig { tenure_idle_timeout_buffer, proposal_wait_for_parent_time, validate_with_replay_tx, + reset_replay_set_after_fork_blocks, capitulate_miner_view_timeout, #[cfg(any(test, feature = "testing"))] supported_signer_protocol_version, @@ -751,6 +768,7 @@ network = "mainnet" auth_password = "abcd" db_path = ":memory:" validate_with_replay_tx = true +reset_replay_set_after_fork_blocks = 100 capitulate_miner_view_timeout_secs = 1000 "# ); @@ -758,6 +776,7 @@ capitulate_miner_view_timeout_secs = 1000 assert_eq!(config.stacks_address.to_string(), expected_addr); assert_eq!(config.to_chain_id(), CHAIN_ID_MAINNET); assert!(config.validate_with_replay_tx); + assert_eq!(config.reset_replay_set_after_fork_blocks, 100); assert_eq!( config.capitulate_miner_view_timeout, Duration::from_secs(1000) diff --git a/stacks-signer/src/runloop.rs b/stacks-signer/src/runloop.rs index 2ac46c93ac..5a5103747a 100644 --- a/stacks-signer/src/runloop.rs +++ b/stacks-signer/src/runloop.rs @@ -329,6 +329,7 @@ impl, T: StacksMessageCodec + Clone + Send + Debug> RunLo reorg_attempts_activity_timeout: self.config.reorg_attempts_activity_timeout, proposal_wait_for_parent_time: self.config.proposal_wait_for_parent_time, validate_with_replay_tx: self.config.validate_with_replay_tx, + reset_replay_set_after_fork_blocks: self.config.reset_replay_set_after_fork_blocks, capitulate_miner_view_timeout: self.config.capitulate_miner_view_timeout, #[cfg(any(test, feature = "testing"))] supported_signer_protocol_version: self.config.supported_signer_protocol_version, diff --git a/stacks-signer/src/tests/signer_state.rs b/stacks-signer/src/tests/signer_state.rs index d338b97d1b..12dcafa5aa 100644 --- a/stacks-signer/src/tests/signer_state.rs +++ b/stacks-signer/src/tests/signer_state.rs @@ -40,7 +40,7 @@ use stacks_common::function_name; use crate::chainstate::{ProposalEvalConfig, SortitionData}; use crate::client::tests::{build_get_tenure_tip_response, MockServerClient}; use crate::client::StacksClient; -use crate::config::GlobalConfig; +use crate::config::{GlobalConfig, DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS}; use crate::signerdb::tests::{create_block_override, tmp_db_path}; use crate::signerdb::SignerDb; use crate::v0::signer_state::{LocalStateMachine, NewBurnBlock, StateMachineUpdate}; @@ -284,6 +284,7 @@ fn check_miner_inactivity_timeout() { tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(3), proposal_wait_for_parent_time: Duration::from_secs(0), + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let block_sk = StacksPrivateKey::from_seed(&[0, 1]); diff --git a/stacks-signer/src/v0/signer.rs b/stacks-signer/src/v0/signer.rs index 9646a7f46b..154db33b3a 100644 --- a/stacks-signer/src/v0/signer.rs +++ b/stacks-signer/src/v0/signer.rs @@ -130,6 +130,8 @@ pub struct Signer { pub validate_with_replay_tx: bool, /// Scope of Tx Replay in terms of Burn block boundaries pub tx_replay_scope: ReplayScopeOpt, + /// The number of blocks after the past tip to reset the replay set + pub reset_replay_set_after_fork_blocks: u64, /// Time to wait between updating our local state machine view point and capitulating to other signers miner view pub capitulate_miner_view_timeout: Duration, /// The last time we capitulated our miner viewpoint @@ -310,6 +312,7 @@ impl SignerTrait for Signer { global_state_evaluator, validate_with_replay_tx: signer_config.validate_with_replay_tx, tx_replay_scope: None, + reset_replay_set_after_fork_blocks: signer_config.reset_replay_set_after_fork_blocks, capitulate_miner_view_timeout: signer_config.capitulate_miner_view_timeout, last_capitulate_miner_view: SystemTime::now(), #[cfg(any(test, feature = "testing"))] diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index d0183e7529..ec680182b0 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -20,6 +20,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use blockstack_lib::chainstate::burn::ConsensusHashExtensions; use blockstack_lib::chainstate::stacks::{StacksTransaction, TransactionPayload}; +use blockstack_lib::net::api::get_tenures_fork_info::TenureForkingInfo; use blockstack_lib::net::api::postblock_proposal::NakamotoBlockProposal; use blockstack_lib::util_lib::db::Error as DBError; #[cfg(any(test, feature = "testing"))] @@ -603,9 +604,11 @@ impl LocalStateMachine { && next_burn_block_hash != expected_burn_block.consensus_hash; if node_behind_expected || node_on_equal_fork { let err_msg = format!( - "Node has not processed the next burn block yet. Expected height = {}, Expected consensus hash = {}", + "Node has not processed the next burn block yet. Expected height = {}, Expected consensus hash = {}, Node height = {}, Node consensus hash = {}", expected_burn_block.burn_block_height, expected_burn_block.consensus_hash, + next_burn_block_height, + next_burn_block_hash, ); *self = Self::Pending { update: StateMachineUpdate::BurnBlock(expected_burn_block), @@ -629,7 +632,7 @@ impl LocalStateMachine { client, &expected_burn_block, &prior_state_machine, - replay_state, + &replay_state, )? { match new_replay_state { ReplayState::Unset => { @@ -641,6 +644,17 @@ impl LocalStateMachine { *tx_replay_scope = Some(new_scope); } } + } else if Self::handle_possible_replay_failsafe( + &replay_state, + &expected_burn_block, + proposal_config.reset_replay_set_after_fork_blocks, + ) { + info!( + "Signer state: replay set is stalled after {} tenures. Clearing the replay set.", + proposal_config.reset_replay_set_after_fork_blocks + ); + tx_replay_set = ReplayTransactionSet::none(); + *tx_replay_scope = None; } } @@ -1031,11 +1045,24 @@ impl LocalStateMachine { client: &StacksClient, expected_burn_block: &NewBurnBlock, prior_state_machine: &SignerStateMachine, - replay_state: ReplayState, + replay_state: &ReplayState, ) -> Result, SignerChainstateError> { if expected_burn_block.burn_block_height > prior_state_machine.burn_block_height { - // no bitcoin fork, because we're advancing the burn block height - return Ok(None); + if Self::new_burn_block_fork_descendency_check( + db, + expected_burn_block, + prior_state_machine.burn_block_height, + prior_state_machine.burn_block, + ) { + info!("Detected bitcoin fork - prior tip is not parent of new tip."; + "new_tip.burn_block_height" => expected_burn_block.burn_block_height, + "new_tip.consensus_hash" => %expected_burn_block.consensus_hash, + "prior_tip.burn_block_height" => prior_state_machine.burn_block_height, + "prior_tip.consensus_hash" => %prior_state_machine.burn_block, + ); + } else { + return Ok(None); + } } if expected_burn_block.consensus_hash == prior_state_machine.burn_block { // no bitcoin fork, because we're at the same burn block hash as before @@ -1140,7 +1167,7 @@ impl LocalStateMachine { client: &StacksClient, expected_burn_block: &NewBurnBlock, prior_state_machine: &SignerStateMachine, - scope: ReplayScope, + scope: &ReplayScope, ) -> Result, SignerChainstateError> { info!("Tx Replay: detected bitcoin fork while in replay mode. Tryng to handle the fork"; "expected_burn_block.height" => expected_burn_block.burn_block_height, @@ -1235,6 +1262,10 @@ impl LocalStateMachine { return Ok(None); } + Ok(Some(Self::get_forked_txs_from_fork_info(&fork_info))) + } + + fn get_forked_txs_from_fork_info(fork_info: &[TenureForkingInfo]) -> Vec { // Collect transactions to be replayed across the forked blocks let mut forked_blocks = fork_info .iter() @@ -1254,6 +1285,83 @@ impl LocalStateMachine { )) .cloned() .collect::>(); - Ok(Some(forked_txs)) + forked_txs + } + + /// If it has been `reset_replay_set_after_fork_blocks` burn blocks since the origin of our replay set, and + /// we haven't produced any replay blocks since then, we should reset our replay set + /// + /// Returns a `bool` indicating whether the replay set should be reset. + fn handle_possible_replay_failsafe( + replay_state: &ReplayState, + new_burn_block: &NewBurnBlock, + reset_replay_set_after_fork_blocks: u64, + ) -> bool { + match replay_state { + ReplayState::Unset => { + // not in replay - skip + false + } + ReplayState::InProgress(_, replay_scope) => { + let failsafe_height = + replay_scope.past_tip.burn_block_height + reset_replay_set_after_fork_blocks; + new_burn_block.burn_block_height > failsafe_height + } + } + } + + /// Check if the new burn block is a fork, by checking if the new burn block + /// is a descendant of the prior burn block + fn new_burn_block_fork_descendency_check( + db: &SignerDb, + new_burn_block: &NewBurnBlock, + prior_burn_block_height: u64, + prior_burn_block_ch: ConsensusHash, + ) -> bool { + let max_height_delta = 10; + let height_delta = match new_burn_block + .burn_block_height + .checked_sub(prior_burn_block_height) + { + None | Some(0) => return false, // same height or older + Some(d) if d > max_height_delta => return false, // too far apart + Some(d) => d, + }; + + let mut parent_burn_block_info = match db + .get_burn_block_by_ch(&new_burn_block.consensus_hash) + .and_then(|burn_block_info| { + db.get_burn_block_by_hash(&burn_block_info.parent_burn_block_hash) + }) { + Ok(info) => info, + Err(e) => { + warn!( + "Failed to get parent burn block info for {}", + new_burn_block.consensus_hash; + "error" => ?e, + ); + return false; + } + }; + + for _ in 0..height_delta { + if parent_burn_block_info.block_height == prior_burn_block_height { + return parent_burn_block_info.consensus_hash != prior_burn_block_ch; + } + + parent_burn_block_info = + match db.get_burn_block_by_hash(&parent_burn_block_info.parent_burn_block_hash) { + Ok(bi) => bi, + Err(e) => { + warn!( + "Failed to get parent burn block info for {}. Error: {e}", + parent_burn_block_info.parent_burn_block_hash + ); + return false; + } + }; + } + + false } } diff --git a/stackslib/src/net/api/postblock_proposal.rs b/stackslib/src/net/api/postblock_proposal.rs index d7b5abfcf3..8c637a66ea 100644 --- a/stackslib/src/net/api/postblock_proposal.rs +++ b/stackslib/src/net/api/postblock_proposal.rs @@ -68,6 +68,10 @@ pub static TEST_REPLAY_TRANSACTIONS: LazyLock< TestFlag>, > = LazyLock::new(TestFlag::default); +#[cfg(any(test, feature = "testing"))] +/// Whether to reject any transaction while we're in a replay set. +pub static TEST_REJECT_REPLAY_TXS: LazyLock> = LazyLock::new(TestFlag::default); + // This enum is used to supply a `reason_code` for validation // rejection responses. This is serialized as an enum with string // type (in jsonschema terminology). @@ -200,6 +204,24 @@ fn fault_injection_validation_delay() { #[cfg(not(any(test, feature = "testing")))] fn fault_injection_validation_delay() {} +#[cfg(any(test, feature = "testing"))] +fn fault_injection_reject_replay_txs() -> Result<(), BlockValidateRejectReason> { + let reject = TEST_REJECT_REPLAY_TXS.get(); + if reject { + Err(BlockValidateRejectReason { + reason_code: ValidateRejectCode::InvalidTransactionReplay, + reason: "Rejected by test flag".into(), + }) + } else { + Ok(()) + } +} + +#[cfg(not(any(test, feature = "testing")))] +fn fault_injection_reject_replay_txs() -> Result<(), BlockValidateRejectReason> { + Ok(()) +} + /// Represents a block proposed to the `v3/block_proposal` endpoint for validation #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct NakamotoBlockProposal { @@ -724,6 +746,7 @@ impl NakamotoBlockProposal { // Allow this to happen, tenure extend checks happen elsewhere. break; } + fault_injection_reject_replay_txs()?; let Some(replay_tx) = replay_txs.pop_front() else { // During transaction replay, we expect that the block only // contains transactions from the replay set. Thus, if we're here,