test: multi-miner & multi-signer scenario

* assert that both stacks-nodes have same chain height, and that they produced blocks in each bitcoin block of nakamoto
* signers are distributed as event observers across 2 stacks-nodes

Author: kantai

.github/workflows/bitcoin-tests.yml

@@ -90,6 +90,7 @@ jobs:
           - tests::signer::v0::forked_tenure_okay
           - tests::signer::v0::forked_tenure_invalid
           - tests::signer::v0::bitcoind_forking_test
+          - tests::signer::v0::multiple_miners
           - tests::nakamoto_integrations::stack_stx_burn_op_integration_test
           - tests::nakamoto_integrations::check_block_heights
           - tests::nakamoto_integrations::clarity_burn_state

testnet/stacks-node/src/tests/signer/mod.rs

@@ -44,7 +44,7 @@ use stacks::chainstate::stacks::{StacksPrivateKey, ThresholdSignature};
 use stacks::core::StacksEpoch;
 use stacks::net::api::postblock_proposal::BlockValidateResponse;
 use stacks::types::chainstate::StacksAddress;
-use stacks::util::secp256k1::MessageSignature;
+use stacks::util::secp256k1::{MessageSignature, Secp256k1PublicKey};
 use stacks_common::codec::StacksMessageCodec;
 use stacks_common::consts::SIGNER_SLOTS_PER_USER;
 use stacks_common::types::StacksEpochId;
@@ -105,14 +105,26 @@ impl<S: Signer<T> + Send + 'static, T: SignerEventTrait + 'static> SignerTest<Sp
         initial_balances: Vec<(StacksAddress, u64)>,
         wait_on_signers: Option<Duration>,
     ) -> Self {
-        Self::new_with_config_modifications(num_signers, initial_balances, wait_on_signers, |_| {})
+        Self::new_with_config_modifications(
+            num_signers,
+            initial_balances,
+            wait_on_signers,
+            |_| {},
+            |_| {},
+            &[],
+        )
     }
 
-    fn new_with_config_modifications<F: Fn(&mut SignerConfig) -> ()>(
+    fn new_with_config_modifications<
+        F: FnMut(&mut SignerConfig) -> (),
+        G: FnMut(&mut NeonConfig) -> (),
+    >(
         num_signers: usize,
         initial_balances: Vec<(StacksAddress, u64)>,
         wait_on_signers: Option<Duration>,
-        modifier: F,
+        mut signer_config_modifier: F,
+        node_config_modifier: G,
+        btc_miner_pubkeys: &[Secp256k1PublicKey],
     ) -> Self {
         // Generate Signer Data
         let signer_stacks_private_keys = (0..num_signers)
@@ -136,11 +148,10 @@ impl<S: Signer<T> + Send + 'static, T: SignerEventTrait + 'static> SignerTest<Sp
         } else {
             naka_conf.miner.wait_on_signers = Duration::from_secs(10);
         }
-
         let run_stamp = rand::random();
 
         // Setup the signer and coordinator configurations
-        let signer_configs = build_signer_config_tomls(
+        let signer_configs: Vec<_> = build_signer_config_tomls(
             &signer_stacks_private_keys,
             &naka_conf.node.rpc_bind,
             Some(Duration::from_millis(128)), // Timeout defaults to 5 seconds. Let's override it to 128 milliseconds.
@@ -151,23 +162,45 @@ impl<S: Signer<T> + Send + 'static, T: SignerEventTrait + 'static> SignerTest<Sp
             Some(100_000),
             None,
             Some(9000),
-        );
-
-        let spawned_signers: Vec<_> = (0..num_signers)
-            .into_iter()
-            .map(|i| {
-                info!("spawning signer");
-                let mut signer_config =
-                    SignerConfig::load_from_str(&signer_configs[i as usize]).unwrap();
-                modifier(&mut signer_config);
-                SpawnedSigner::new(signer_config)
-            })
+        )
+        .into_iter()
+        .map(|toml| {
+            let mut signer_config = SignerConfig::load_from_str(&toml).unwrap();
+            signer_config_modifier(&mut signer_config);
+            signer_config
+        })
+        .collect();
+        assert_eq!(signer_configs.len(), num_signers);
+
+        let spawned_signers = signer_configs
+            .iter()
+            .cloned()
+            .map(SpawnedSigner::new)
             .collect();
 
         // Setup the nodes and deploy the contract to it
-        let node = setup_stx_btc_node(naka_conf, &signer_stacks_private_keys, &signer_configs);
-        let config = SignerConfig::load_from_str(&signer_configs[0]).unwrap();
-        let stacks_client = StacksClient::from(&config);
+        let btc_miner_pubkeys = if btc_miner_pubkeys.is_empty() {
+            let pk = Secp256k1PublicKey::from_hex(
+                naka_conf
+                    .burnchain
+                    .local_mining_public_key
+                    .as_ref()
+                    .unwrap(),
+            )
+            .unwrap();
+            &[pk]
+        } else {
+            btc_miner_pubkeys
+        };
+        let node = setup_stx_btc_node(
+            naka_conf,
+            &signer_stacks_private_keys,
+            &signer_configs,
+            btc_miner_pubkeys,
+            node_config_modifier,
+        );
+        let config = signer_configs.first().unwrap();
+        let stacks_client = StacksClient::from(config);
 
         Self {
             running_nodes: node,
@@ -294,6 +327,33 @@ impl<S: Signer<T> + Send + 'static, T: SignerEventTrait + 'static> SignerTest<Sp
         test_observer::get_mined_nakamoto_blocks().pop().unwrap()
     }
 
+    fn mine_block_wait_on_processing(&mut self, timeout: Duration) {
+        let commits_submitted = self.running_nodes.commits_submitted.clone();
+        let blocks_len = test_observer::get_blocks().len();
+        let mined_block_time = Instant::now();
+        next_block_and_mine_commit(
+            &mut self.running_nodes.btc_regtest_controller,
+            timeout.as_secs(),
+            &self.running_nodes.coord_channel,
+            &commits_submitted,
+        )
+        .unwrap();
+
+        let t_start = Instant::now();
+        while test_observer::get_blocks().len() <= blocks_len {
+            assert!(
+                t_start.elapsed() < timeout,
+                "Timed out while waiting for nakamoto block to be processed"
+            );
+            thread::sleep(Duration::from_secs(1));
+        }
+        let mined_block_elapsed_time = mined_block_time.elapsed();
+        info!(
+            "Nakamoto block mine time elapsed: {:?}",
+            mined_block_elapsed_time
+        );
+    }
+
     fn wait_for_confirmed_block_v1(
         &mut self,
         block_signer_sighash: &Sha512Trunc256Sum,
@@ -537,17 +597,17 @@ impl<S: Signer<T> + Send + 'static, T: SignerEventTrait + 'static> SignerTest<Sp
     }
 }
 
-fn setup_stx_btc_node(
+fn setup_stx_btc_node<G: FnMut(&mut NeonConfig) -> ()>(
     mut naka_conf: NeonConfig,
     signer_stacks_private_keys: &[StacksPrivateKey],
-    signer_config_tomls: &[String],
+    signer_configs: &[SignerConfig],
+    btc_miner_pubkeys: &[Secp256k1PublicKey],
+    mut node_config_modifier: G,
 ) -> RunningNodes {
     // Spawn the endpoints for observing signers
-    for toml in signer_config_tomls {
-        let signer_config = SignerConfig::load_from_str(toml).unwrap();
-
+    for signer_config in signer_configs {
         naka_conf.events_observers.insert(EventObserverConfig {
-            endpoint: format!("{}", signer_config.endpoint),
+            endpoint: signer_config.endpoint.to_string(),
             events_keys: vec![
                 EventKeyType::StackerDBChunks,
                 EventKeyType::BlockProposal,
@@ -593,6 +653,8 @@ fn setup_stx_btc_node(
             }
         }
     }
+    node_config_modifier(&mut naka_conf);
+
     info!("Make new BitcoinCoreController");
     let mut btcd_controller = BitcoinCoreController::new(naka_conf.clone());
     btcd_controller
@@ -604,7 +666,7 @@ fn setup_stx_btc_node(
     let mut btc_regtest_controller = BitcoinRegtestController::new(naka_conf.clone(), None);
 
     info!("Bootstraping...");
-    btc_regtest_controller.bootstrap_chain(201);
+    btc_regtest_controller.bootstrap_chain_to_pks(201, btc_miner_pubkeys);
 
     info!("Chain bootstrapped...");
 

testnet/stacks-node/src/tests/signer/v0.rs

@@ -13,6 +13,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::str::FromStr;
 use std::sync::atomic::Ordering;
 use std::time::{Duration, Instant};
 use std::{env, thread};
@@ -28,7 +29,7 @@ use stacks::chainstate::stacks::db::{StacksChainState, StacksHeaderInfo};
 use stacks::codec::StacksMessageCodec;
 use stacks::libstackerdb::StackerDBChunkData;
 use stacks::net::api::postblock_proposal::TEST_VALIDATE_STALL;
-use stacks::types::chainstate::{StacksAddress, StacksPrivateKey};
+use stacks::types::chainstate::{StacksAddress, StacksPrivateKey, StacksPublicKey};
 use stacks::types::PublicKey;
 use stacks::util::secp256k1::{Secp256k1PrivateKey, Secp256k1PublicKey};
 use stacks::util_lib::boot::boot_code_id;
@@ -44,6 +45,7 @@ use super::SignerTest;
 use crate::event_dispatcher::MinedNakamotoBlockEvent;
 use crate::nakamoto_node::miner::TEST_BROADCAST_STALL;
 use crate::nakamoto_node::relayer::TEST_SKIP_COMMIT_OP;
+use crate::run_loop::boot_nakamoto;
 use crate::tests::nakamoto_integrations::{boot_to_epoch_3_reward_set, next_block_and};
 use crate::tests::neon_integrations::{
     get_account, get_chain_info, next_block_and_wait, submit_tx, test_observer,
@@ -611,6 +613,8 @@ fn forked_tenure_testing(
             // make the duration long enough that the reorg attempt will definitely be accepted
             config.first_proposal_burn_block_timing = proposal_limit;
         },
+        |_| {},
+        &[],
     );
     let http_origin = format!("http://{}", &signer_test.running_nodes.conf.node.rpc_bind);
 
@@ -799,11 +803,10 @@ fn bitcoind_forking_test() {
     let sender_addr = tests::to_addr(&sender_sk);
     let send_amt = 100;
     let send_fee = 180;
-    let mut signer_test: SignerTest<SpawnedSigner> = SignerTest::new_with_config_modifications(
+    let mut signer_test: SignerTest<SpawnedSigner> = SignerTest::new(
         num_signers,
         vec![(sender_addr.clone(), send_amt + send_fee)],
         Some(Duration::from_secs(15)),
-        |_config| {},
     );
     let conf = signer_test.running_nodes.conf.clone();
     let http_origin = format!("http://{}", &conf.node.rpc_bind);
@@ -936,6 +939,134 @@ fn bitcoind_forking_test() {
     signer_test.shutdown();
 }
 
+#[test]
+#[ignore]
+fn multiple_miners() {
+    if env::var("BITCOIND_TEST") != Ok("1".into()) {
+        return;
+    }
+
+    let num_signers = 5;
+    let sender_sk = Secp256k1PrivateKey::new();
+    let sender_addr = tests::to_addr(&sender_sk);
+    let send_amt = 100;
+    let send_fee = 180;
+
+    let btc_miner_1_seed = vec![1, 1, 1, 1];
+    let btc_miner_2_seed = vec![2, 2, 2, 2];
+    let btc_miner_1_pk = Keychain::default(btc_miner_1_seed.clone()).get_pub_key();
+    let btc_miner_2_pk = Keychain::default(btc_miner_2_seed.clone()).get_pub_key();
+
+    let node_1_rpc = 51024;
+    let node_1_p2p = 51023;
+    let node_2_rpc = 51026;
+    let node_2_p2p = 51025;
+
+    let node_1_rpc_bind = format!("127.0.0.1:{}", node_1_rpc);
+    let node_2_rpc_bind = format!("127.0.0.1:{}", node_2_rpc);
+    let mut node_2_listeners = Vec::new();
+
+    // partition the signer set so that ~half are listening and using node 1 for RPC and events,
+    //  and the rest are using node 2
+
+    let mut signer_test: SignerTest<SpawnedSigner> = SignerTest::new_with_config_modifications(
+        num_signers,
+        vec![(sender_addr.clone(), send_amt + send_fee)],
+        Some(Duration::from_secs(15)),
+        |signer_config| {
+            let node_host = if signer_config.endpoint.port() % 2 == 0 {
+                &node_1_rpc_bind
+            } else {
+                &node_2_rpc_bind
+            };
+            signer_config.node_host = node_host.to_string();
+        },
+        |config| {
+            let localhost = "127.0.0.1";
+            config.node.rpc_bind = format!("{}:{}", localhost, node_1_rpc);
+            config.node.p2p_bind = format!("{}:{}", localhost, node_1_p2p);
+            config.node.data_url = format!("http://{}:{}", localhost, node_1_rpc);
+            config.node.p2p_address = format!("{}:{}", localhost, node_1_p2p);
+
+            config.node.seed = btc_miner_1_seed.clone();
+            config.node.local_peer_seed = btc_miner_1_seed.clone();
+            config.burnchain.local_mining_public_key = Some(btc_miner_1_pk.to_hex());
+
+            config.events_observers.retain(|listener| {
+                let Ok(addr) = std::net::SocketAddr::from_str(&listener.endpoint) else {
+                    warn!(
+                        "Cannot parse {} to a socket, assuming it isn't a signer-listener binding",
+                        listener.endpoint
+                    );
+                    return true;
+                };
+                if addr.port() % 2 == 0 || addr.port() == test_observer::EVENT_OBSERVER_PORT {
+                    return true;
+                }
+                node_2_listeners.push(listener.clone());
+                false
+            })
+        },
+        &[btc_miner_1_pk.clone(), btc_miner_2_pk.clone()],
+    );
+    let conf = signer_test.running_nodes.conf.clone();
+    let mut conf_node_2 = conf.clone();
+    let localhost = "127.0.0.1";
+    conf_node_2.node.rpc_bind = format!("{}:{}", localhost, node_2_rpc);
+    conf_node_2.node.p2p_bind = format!("{}:{}", localhost, node_2_p2p);
+    conf_node_2.node.data_url = format!("http://{}:{}", localhost, node_2_rpc);
+    conf_node_2.node.p2p_address = format!("{}:{}", localhost, node_2_p2p);
+    conf_node_2.node.seed = btc_miner_2_seed.clone();
+    conf_node_2.burnchain.local_mining_public_key = Some(btc_miner_2_pk.to_hex());
+    conf_node_2.node.local_peer_seed = btc_miner_2_seed.clone();
+    conf_node_2.node.miner = true;
+    conf_node_2.events_observers.clear();
+    conf_node_2.events_observers.extend(node_2_listeners);
+    assert!(!conf_node_2.events_observers.is_empty());
+
+    let node_1_sk = Secp256k1PrivateKey::from_seed(&conf.node.local_peer_seed);
+    let node_1_pk = StacksPublicKey::from_private(&node_1_sk);
+
+    conf_node_2.node.working_dir = format!("{}-{}", conf_node_2.node.working_dir, "1");
+
+    conf_node_2.node.set_bootstrap_nodes(
+        format!("{}@{}", &node_1_pk.to_hex(), conf.node.p2p_bind),
+        conf.burnchain.chain_id,
+        conf.burnchain.peer_version,
+    );
+
+    let mut run_loop_2 = boot_nakamoto::BootRunLoop::new(conf_node_2.clone()).unwrap();
+    let _run_loop_2_thread = thread::Builder::new()
+        .name("run_loop_2".into())
+        .spawn(move || run_loop_2.start(None, 0))
+        .unwrap();
+
+    signer_test.boot_to_epoch_3();
+    let pre_nakamoto_peer_1_height = get_chain_info(&conf).stacks_tip_height;
+
+    info!("------------------------- Reached Epoch 3.0 -------------------------");
+
+    let nakamoto_tenures = 20;
+    for _i in 0..nakamoto_tenures {
+        let _mined_block = signer_test.mine_block_wait_on_processing(Duration::from_secs(30));
+    }
+
+    info!(
+        "New chain info: {:?}",
+        get_chain_info(&signer_test.running_nodes.conf)
+    );
+
+    info!("New chain info: {:?}", get_chain_info(&conf_node_2));
+
+    let peer_1_height = get_chain_info(&conf).stacks_tip_height;
+    let peer_2_height = get_chain_info(&conf_node_2).stacks_tip_height;
+    info!("Peer height information"; "peer_1" => peer_1_height, "peer_2" => peer_2_height, "pre_naka_height" => pre_nakamoto_peer_1_height);
+    assert_eq!(peer_1_height, peer_2_height);
+    assert_eq!(peer_1_height, pre_nakamoto_peer_1_height + nakamoto_tenures);
+
+    signer_test.shutdown();
+}
+
 #[test]
 #[ignore]
 /// This test checks the behavior at the end of a tenure. Specifically: