Merge pull request #5075 from stacks-network/test/partial-tenure-fork

Test partial tenure fork

Author: jferrant

.github/workflows/bitcoin-tests.yml

@@ -112,6 +112,8 @@ jobs:
           - tests::nakamoto_integrations::multiple_miners
           - tests::nakamoto_integrations::utxo_check_on_startup_panic
           - tests::nakamoto_integrations::utxo_check_on_startup_recover
+          - tests::signer::v0::multiple_miners_with_nakamoto_blocks
+          - tests::signer::v0::partial_tenure_fork
           # Do not run this one until we figure out why it fails in CI
           # - tests::neon_integrations::bitcoin_reorg_flap
           # - tests::neon_integrations::bitcoin_reorg_flap_with_follower

stackslib/src/chainstate/nakamoto/coordinator/tests.rs

@@ -41,10 +41,10 @@ use crate::chainstate::burn::db::sortdb::{SortitionDB, SortitionHandle};
 use crate::chainstate::burn::operations::{BlockstackOperationType, LeaderBlockCommitOp};
 use crate::chainstate::coordinator::tests::{p2pkh_from, pox_addr_from};
 use crate::chainstate::nakamoto::coordinator::load_nakamoto_reward_set;
+use crate::chainstate::nakamoto::fault_injection::*;
 use crate::chainstate::nakamoto::miner::NakamotoBlockBuilder;
 use crate::chainstate::nakamoto::signer_set::NakamotoSigners;
 use crate::chainstate::nakamoto::test_signers::TestSigners;
-use crate::chainstate::nakamoto::test_stall::*;
 use crate::chainstate::nakamoto::tests::get_account;
 use crate::chainstate::nakamoto::tests::node::TestStacker;
 use crate::chainstate::nakamoto::{

stackslib/src/chainstate/nakamoto/mod.rs

@@ -271,27 +271,26 @@ lazy_static! {
 }
 
 #[cfg(test)]
-mod test_stall {
-    pub static TEST_PROCESS_BLOCK_STALL: std::sync::Mutex<Option<bool>> =
-        std::sync::Mutex::new(None);
+mod fault_injection {
+    static PROCESS_BLOCK_STALL: std::sync::Mutex<bool> = std::sync::Mutex::new(false);
 
     pub fn stall_block_processing() {
-        if *TEST_PROCESS_BLOCK_STALL.lock().unwrap() == Some(true) {
+        if *PROCESS_BLOCK_STALL.lock().unwrap() {
             // Do an extra check just so we don't log EVERY time.
             warn!("Block processing is stalled due to testing directive.");
-            while *TEST_PROCESS_BLOCK_STALL.lock().unwrap() == Some(true) {
+            while *PROCESS_BLOCK_STALL.lock().unwrap() {
                 std::thread::sleep(std::time::Duration::from_millis(10));
             }
             info!("Block processing is no longer stalled due to testing directive.");
         }
     }
 
     pub fn enable_process_block_stall() {
-        TEST_PROCESS_BLOCK_STALL.lock().unwrap().replace(true);
+        *PROCESS_BLOCK_STALL.lock().unwrap() = true;
     }
 
     pub fn disable_process_block_stall() {
-        TEST_PROCESS_BLOCK_STALL.lock().unwrap().replace(false);
+        *PROCESS_BLOCK_STALL.lock().unwrap() = false;
     }
 }
 
@@ -1758,7 +1757,7 @@ impl NakamotoChainState {
         dispatcher_opt: Option<&'a T>,
     ) -> Result<Option<StacksEpochReceipt>, ChainstateError> {
         #[cfg(test)]
-        test_stall::stall_block_processing();
+        fault_injection::stall_block_processing();
 
         let nakamoto_blocks_db = stacks_chain_state.nakamoto_blocks_db();
         let Some((next_ready_block, block_size)) =

stackslib/src/net/relay.rs

@@ -70,6 +70,51 @@ pub const MAX_RECENT_MESSAGES: usize = 256;
 pub const MAX_RECENT_MESSAGE_AGE: usize = 600; // seconds; equal to the expected epoch length
 pub const RELAY_DUPLICATE_INFERENCE_WARMUP: usize = 128;
 
+#[cfg(any(test, feature = "testing"))]
+pub mod fault_injection {
+    use std::path::Path;
+
+    static IGNORE_BLOCK: std::sync::Mutex<Option<(u64, String)>> = std::sync::Mutex::new(None);
+
+    pub fn ignore_block(height: u64, working_dir: &str) -> bool {
+        if let Some((ignore_height, ignore_dir)) = &*IGNORE_BLOCK.lock().unwrap() {
+            let working_dir_path = Path::new(working_dir);
+            let ignore_dir_path = Path::new(ignore_dir);
+
+            let ignore = *ignore_height == height && working_dir_path.starts_with(ignore_dir_path);
+            if ignore {
+                warn!("Fault injection: ignore block at height {}", height);
+            }
+            return ignore;
+        }
+        false
+    }
+
+    pub fn set_ignore_block(height: u64, working_dir: &str) {
+        warn!(
+            "Fault injection: set ignore block at height {} for working directory {}",
+            height, working_dir
+        );
+        *IGNORE_BLOCK.lock().unwrap() = Some((height, working_dir.to_string()));
+    }
+
+    pub fn clear_ignore_block() {
+        warn!("Fault injection: clear ignore block");
+        *IGNORE_BLOCK.lock().unwrap() = None;
+    }
+}
+
+#[cfg(not(any(test, feature = "testing")))]
+pub mod fault_injection {
+    pub fn ignore_block(_height: u64, _working_dir: &str) -> bool {
+        false
+    }
+
+    pub fn set_ignore_block(_height: u64, _working_dir: &str) {}
+
+    pub fn clear_ignore_block() {}
+}
+
 pub struct Relayer {
     /// Connection to the p2p thread
     p2p: NetworkHandle,
@@ -845,6 +890,10 @@ impl Relayer {
             &obtained_method,
         );
 
+        if fault_injection::ignore_block(block.header.chain_length, &burnchain.working_dir) {
+            return Ok(false);
+        }
+
         // do we have this block?  don't lock the DB needlessly if so.
         if chainstate
             .nakamoto_blocks_db()

testnet/stacks-node/src/nakamoto_node/miner.rs

@@ -45,7 +45,6 @@ use stacks::chainstate::stacks::{
 use stacks::net::p2p::NetworkHandle;
 use stacks::net::stackerdb::StackerDBs;
 use stacks::net::{NakamotoBlocksData, StacksMessageType};
-use stacks::util::get_epoch_time_secs;
 use stacks::util::secp256k1::MessageSignature;
 use stacks_common::codec::read_next;
 use stacks_common::types::chainstate::{StacksAddress, StacksBlockId};
@@ -1068,9 +1067,12 @@ impl BlockMinerThread {
                     );
                     NakamotoNodeError::ParentNotFound
                 })?;
-        let current_timestamp = get_epoch_time_secs();
-        let time_since_parent_ms =
-            current_timestamp.saturating_sub(stacks_parent_header.burn_header_timestamp) * 1000;
+        let current_timestamp = x.header.timestamp;
+        let parent_timestamp = match stacks_parent_header.anchored_header.as_stacks_nakamoto() {
+            Some(naka_header) => naka_header.timestamp,
+            None => stacks_parent_header.burn_header_timestamp,
+        };
+        let time_since_parent_ms = current_timestamp.saturating_sub(parent_timestamp) * 1000;
         if time_since_parent_ms < self.config.miner.min_time_between_blocks_ms {
             debug!("Parent block mined {time_since_parent_ms} ms ago. Required minimum gap between blocks is {} ms", self.config.miner.min_time_between_blocks_ms;
                 "current_timestamp" => current_timestamp,

testnet/stacks-node/src/tests/nakamoto_integrations.rs

@@ -711,6 +711,7 @@ pub fn next_block_and_wait_for_commits(
         (0..commits_before.len()).map(|_| None).collect();
     let mut commit_sent_time: Vec<Option<Instant>> =
         (0..commits_before.len()).map(|_| None).collect();
+    sleep_ms(2000); // Make sure that the proposed stacks block has a different timestamp than its parent
     next_block_and(btc_controller, timeout_secs, || {
         for i in 0..commits_submitted.len() {
             let commits_sent = commits_submitted[i].load(Ordering::SeqCst);

testnet/stacks-node/src/tests/neon_integrations.rs

@@ -749,8 +749,8 @@ pub fn wait_for_microblocks(microblocks_processed: &Arc<AtomicU64>, timeout: u64
     return true;
 }
 
-/// returns Txid string
-pub fn submit_tx(http_origin: &str, tx: &Vec<u8>) -> String {
+/// returns Txid string upon success
+pub fn submit_tx_fallible(http_origin: &str, tx: &Vec<u8>) -> Result<String, String> {
     let client = reqwest::blocking::Client::new();
     let path = format!("{}/v2/transactions", http_origin);
     let res = client
@@ -768,13 +768,20 @@ pub fn submit_tx(http_origin: &str, tx: &Vec<u8>) -> String {
                 .txid()
                 .to_string()
         );
-        return res;
+        Ok(res)
     } else {
-        eprintln!("Submit tx error: {}", res.text().unwrap());
-        panic!("");
+        Err(res.text().unwrap())
     }
 }
 
+/// returns Txid string
+pub fn submit_tx(http_origin: &str, tx: &Vec<u8>) -> String {
+    submit_tx_fallible(http_origin, tx).unwrap_or_else(|e| {
+        eprintln!("Submit tx error: {}", e);
+        panic!("");
+    })
+}
+
 pub fn get_unconfirmed_tx(http_origin: &str, txid: &Txid) -> Option<String> {
     let client = reqwest::blocking::Client::new();
     let path = format!("{}/v2/transactions/unconfirmed/{}", http_origin, txid);