chore: halt block-processing while we process shadow blocks

Author: jcnelson

stackslib/src/chainstate/nakamoto/coordinator/mod.rs

@@ -58,6 +58,9 @@ use crate::monitoring::increment_stx_blocks_processed_counter;
 use crate::net::Error as NetError;
 use crate::util_lib::db::Error as DBError;
 
+#[cfg(any(test, feature = "testing"))]
+pub static TEST_COORDINATOR_STALL: std::sync::Mutex<Option<bool>> = std::sync::Mutex::new(None);
+
 #[cfg(test)]
 pub mod tests;
 
@@ -764,6 +767,21 @@ impl<
         true
     }
 
+    #[cfg(any(test, feature = "testing"))]
+    fn fault_injection_pause_nakamoto_block_processing() {
+        if *TEST_COORDINATOR_STALL.lock().unwrap() == Some(true) {
+            // Do an extra check just so we don't log EVERY time.
+            warn!("Coordinator is stalled due to testing directive");
+            while *TEST_COORDINATOR_STALL.lock().unwrap() == Some(true) {
+                std::thread::sleep(std::time::Duration::from_millis(10));
+            }
+            warn!("Coordinator is no longer stalled due to testing directive. Continuing...");
+        }
+    }
+
+    #[cfg(not(any(test, feature = "testing")))]
+    fn fault_injection_pause_nakamoto_block_processing() {}
+
     /// Handle one or more new Nakamoto Stacks blocks.
     /// If we process a PoX anchor block, then return its block hash.  This unblocks processing the
     /// next reward cycle's burnchain blocks.  Subsequent calls to this function will terminate
@@ -776,6 +794,8 @@ impl<
         );
 
         loop {
+            Self::fault_injection_pause_nakamoto_block_processing();
+
             // process at most one block per loop pass
             let mut processed_block_receipt = match NakamotoChainState::process_next_nakamoto_block(
                 &mut self.chain_state_db,

testnet/stacks-node/src/tests/nakamoto_integrations.rs

@@ -37,7 +37,7 @@ use stacks::chainstate::burn::operations::{
 };
 use stacks::chainstate::coordinator::comm::CoordinatorChannels;
 use stacks::chainstate::coordinator::OnChainRewardSetProvider;
-use stacks::chainstate::nakamoto::coordinator::load_nakamoto_reward_set;
+use stacks::chainstate::nakamoto::coordinator::{load_nakamoto_reward_set, TEST_COORDINATOR_STALL};
 use stacks::chainstate::nakamoto::miner::NakamotoBlockBuilder;
 use stacks::chainstate::nakamoto::shadow::shadow_chainstate_repair;
 use stacks::chainstate::nakamoto::test_signers::TestSigners;
@@ -9672,6 +9672,10 @@ fn test_shadow_recovery() {
 
     let stacks_height_before = get_chain_info(&naka_conf).stacks_tip_height;
 
+    // TODO: stall block processing; otherwise this test can flake
+    // stop block processing on the node
+    TEST_COORDINATOR_STALL.lock().unwrap().replace(true);
+
     // fix node
     let shadow_blocks = shadow_chainstate_repair(&mut chainstate, &mut sortdb).unwrap();
     assert!(shadow_blocks.len() > 0);
@@ -9685,6 +9689,7 @@ fn test_shadow_recovery() {
     })
     .unwrap();
 
+    TEST_COORDINATOR_STALL.lock().unwrap().replace(false);
     info!("Beginning post-shadow tenures");
 
     // revive ATC-C by waiting for commits