Merge branch 'develop' into test/fast-blocks-epoch3-change

Author: ASuciuX

.github/workflows/bitcoin-tests.yml

@@ -74,6 +74,7 @@ jobs:
           - tests::neon_integrations::min_txs
           - tests::neon_integrations::vote_for_aggregate_key_burn_op_test
           - tests::neon_integrations::mock_miner_replay
+          - tests::neon_integrations::listunspent_max_utxos
           - tests::epoch_25::microblocks_disabled
           - tests::should_succeed_handling_malformed_and_valid_txs
           - tests::nakamoto_integrations::simple_neon_integration

testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs

@@ -2314,6 +2314,10 @@ impl UTXOSet {
     pub fn total_available(&self) -> u64 {
         self.utxos.iter().map(|o| o.amount).sum()
     }
+
+    pub fn num_utxos(&self) -> usize {
+        self.utxos.len()
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -2607,7 +2611,7 @@ impl BitcoinRPCRequest {
                 max_conf.into(),
                 addresses.into(),
                 include_unsafe.into(),
-                json!({ "minimumAmount": minimum_amount }),
+                json!({ "minimumAmount": minimum_amount, "maximumCount": config.burnchain.max_unspent_utxos }),
             ],
             id: "stacks".to_string(),
             jsonrpc: "2.0".to_string(),

testnet/stacks-node/src/config.rs

@@ -1446,6 +1446,10 @@ pub struct BurnchainConfig {
     /// fault injection to simulate a slow burnchain peer.
     /// Delay burnchain block downloads by the given number of millseconds
     pub fault_injection_burnchain_block_delay: u64,
+    /// The maximum number of unspent UTXOs to request from the bitcoin node.
+    /// This value is passed as the `maximumCount` query option to the
+    /// `listunspent` RPC call.
+    pub max_unspent_utxos: Option<u64>,
 }
 
 impl BurnchainConfig {
@@ -1486,6 +1490,7 @@ impl BurnchainConfig {
             ast_precheck_size_height: None,
             affirmation_overrides: HashMap::new(),
             fault_injection_burnchain_block_delay: 0,
+            max_unspent_utxos: Some(1024),
         }
     }
     pub fn get_rpc_url(&self, wallet: Option<String>) -> String {
@@ -1582,6 +1587,7 @@ pub struct BurnchainConfigFile {
     pub ast_precheck_size_height: Option<u64>,
     pub affirmation_overrides: Option<Vec<AffirmationOverride>>,
     pub fault_injection_burnchain_block_delay: Option<u64>,
+    pub max_unspent_utxos: Option<u64>,
 }
 
 impl BurnchainConfigFile {
@@ -1797,6 +1803,13 @@ impl BurnchainConfigFile {
             fault_injection_burnchain_block_delay: self
                 .fault_injection_burnchain_block_delay
                 .unwrap_or(default_burnchain_config.fault_injection_burnchain_block_delay),
+            max_unspent_utxos: self
+                .max_unspent_utxos
+                .map(|val| {
+                    assert!(val <= 1024, "Value for max_unspent_utxos should be <= 1024");
+                    val
+                })
+                .or(default_burnchain_config.max_unspent_utxos),
         };
 
         if let BitcoinNetworkType::Mainnet = config.get_bitcoin_network().1 {

testnet/stacks-node/src/tests/nakamoto_integrations.rs

@@ -2674,8 +2674,6 @@ fn correct_burn_outs() {
 
     let mut last_block_time = None;
     for block in new_blocks_with_reward_set.iter() {
-        let cycle_number = block["cycle_number"].as_u64().unwrap();
-        let reward_set = block["reward_set"].as_object().unwrap();
         if let Some(block_time) = block["block_time"].as_u64() {
             if let Some(last) = last_block_time {
                 assert!(block_time > last, "Block times should be increasing");

testnet/stacks-node/src/tests/neon_integrations.rs

@@ -62,6 +62,7 @@ use stacks::net::atlas::{
     AtlasConfig, AtlasDB, GetAttachmentResponse, GetAttachmentsInvResponse,
     MAX_ATTACHMENT_INV_PAGES_PER_REQUEST,
 };
+use stacks::types::PublicKey;
 use stacks::util_lib::boot::{boot_code_addr, boot_code_id};
 use stacks::util_lib::db::{query_row_columns, query_rows, u64_to_sql};
 use stacks::util_lib::signed_structured_data::pox4::{
@@ -82,7 +83,7 @@ use super::{
     make_microblock, make_stacks_transfer, make_stacks_transfer_mblock_only, to_addr, ADDR_4, SK_1,
     SK_2, SK_3,
 };
-use crate::burnchains::bitcoin_regtest_controller::{self, BitcoinRPCRequest, UTXO};
+use crate::burnchains::bitcoin_regtest_controller::{self, addr2str, BitcoinRPCRequest, UTXO};
 use crate::config::{EventKeyType, EventObserverConfig, FeeEstimatorName, InitialBalance};
 use crate::neon_node::RelayerThread;
 use crate::operations::BurnchainOpSigner;
@@ -12794,3 +12795,49 @@ fn mock_miner_replay() {
     miner_channel.stop_chains_coordinator();
     follower_channel.stop_chains_coordinator();
 }
+
+#[test]
+#[ignore]
+/// Verify that the config option, `burnchain.max_unspent_utxos`, is respected.
+fn listunspent_max_utxos() {
+    if env::var("BITCOIND_TEST") != Ok("1".into()) {
+        return;
+    }
+
+    let (mut conf, _miner_account) = neon_integration_test_conf();
+    let prom_bind = format!("{}:{}", "127.0.0.1", 6000);
+    conf.node.prometheus_bind = Some(prom_bind.clone());
+
+    conf.burnchain.max_rbf = 1000000;
+    conf.burnchain.max_unspent_utxos = Some(10);
+
+    let mut btcd_controller = BitcoinCoreController::new(conf.clone());
+    btcd_controller
+        .start_bitcoind()
+        .map_err(|_e| ())
+        .expect("Failed starting bitcoind");
+
+    let mut btc_regtest_controller = BitcoinRegtestController::new(conf.clone(), None);
+
+    btc_regtest_controller.bootstrap_chain(201);
+
+    eprintln!("Chain bootstrapped...");
+
+    let keychain = Keychain::default(conf.node.seed.clone());
+    let mut op_signer = keychain.generate_op_signer();
+
+    let (_, network_id) = conf.burnchain.get_bitcoin_network();
+    let hash160 = Hash160::from_data(&op_signer.get_public_key().to_bytes());
+    let address = BitcoinAddress::from_bytes_legacy(
+        network_id,
+        LegacyBitcoinAddressType::PublicKeyHash,
+        &hash160.0,
+    )
+    .expect("Public key incorrect");
+
+    let filter_addresses = vec![addr2str(&address)];
+
+    let res = BitcoinRPCRequest::list_unspent(&conf, filter_addresses, false, 1, &None, 0);
+    let utxos = res.expect("Failed to get utxos");
+    assert_eq!(utxos.num_utxos(), 10);
+}

testnet/stacks-node/src/tests/signer/v0.rs

@@ -264,6 +264,11 @@ impl SignerTest<SpawnedSigner> {
         info!("Signers initialized");
 
         self.run_until_epoch_3_boundary();
+        std::thread::sleep(Duration::from_secs(1));
+        wait_for(60, || {
+            Ok(get_chain_info_opt(&self.running_nodes.conf).is_some())
+        })
+        .expect("Timed out waiting for network to restart after 3.0 boundary reached");
 
         // Wait until we see the first block of epoch 3.0.
         // Note, we don't use `nakamoto_blocks_mined` counter, because there
@@ -4128,25 +4133,24 @@ fn locally_rejected_blocks_overriden_by_global_acceptance() {
     signer_test.boot_to_epoch_3();
     info!("------------------------- Test Mine Nakamoto Block N -------------------------");
     let mined_blocks = signer_test.running_nodes.nakamoto_blocks_mined.clone();
-    let blocks_before = mined_blocks.load(Ordering::SeqCst);
     let info_before = signer_test
         .stacks_client
         .get_peer_info()
         .expect("Failed to get peer info");
-    let start_time = Instant::now();
     // submit a tx so that the miner will mine a stacks block
     let mut sender_nonce = 0;
     let transfer_tx =
         make_stacks_transfer(&sender_sk, sender_nonce, send_fee, &recipient, send_amt);
     let tx = submit_tx(&http_origin, &transfer_tx);
     info!("Submitted tx {tx} in to mine block N");
-    while mined_blocks.load(Ordering::SeqCst) <= blocks_before {
-        assert!(
-            start_time.elapsed() < short_timeout,
-            "FAIL: Test timed out while waiting for block production",
-        );
-        thread::sleep(Duration::from_secs(1));
-    }
+    wait_for(short_timeout.as_secs(), || {
+        let info_after = signer_test
+            .stacks_client
+            .get_peer_info()
+            .expect("Failed to get peer info");
+        Ok(info_after.stacks_tip_height > info_before.stacks_tip_height)
+    })
+    .expect("Timed out waiting for block to be mined and processed");
 
     sender_nonce += 1;
     let info_after = signer_test
@@ -4196,13 +4200,14 @@ fn locally_rejected_blocks_overriden_by_global_acceptance() {
         .stacks_client
         .get_peer_info()
         .expect("Failed to get peer info");
-    while mined_blocks.load(Ordering::SeqCst) <= blocks_before {
-        assert!(
-            start_time.elapsed() < short_timeout,
-            "FAIL: Test timed out while waiting for block production",
-        );
-        thread::sleep(Duration::from_secs(1));
-    }
+    wait_for(short_timeout.as_secs(), || {
+        let info_after = signer_test
+            .stacks_client
+            .get_peer_info()
+            .expect("Failed to get peer info");
+        Ok(info_after.stacks_tip_height > info_before.stacks_tip_height)
+    })
+    .expect("Timed out waiting for block to be mined and processed");
     loop {
         let stackerdb_events = test_observer::get_stackerdb_chunks();
         let block_rejections = stackerdb_events
@@ -4274,13 +4279,14 @@ fn locally_rejected_blocks_overriden_by_global_acceptance() {
         make_stacks_transfer(&sender_sk, sender_nonce, send_fee, &recipient, send_amt);
     let tx = submit_tx(&http_origin, &transfer_tx);
     info!("Submitted tx {tx} in to mine block N+2");
-    while mined_blocks.load(Ordering::SeqCst) <= blocks_before {
-        assert!(
-            start_time.elapsed() < short_timeout,
-            "FAIL: Test timed out while waiting for block production",
-        );
-        thread::sleep(Duration::from_secs(1));
-    }
+    wait_for(short_timeout.as_secs(), || {
+        let info_after = signer_test
+            .stacks_client
+            .get_peer_info()
+            .expect("Failed to get peer info");
+        Ok(info_after.stacks_tip_height > info_before.stacks_tip_height)
+    })
+    .expect("Timed out waiting for block to be mined and processed");
     let blocks_after = mined_blocks.load(Ordering::SeqCst);
     assert_eq!(blocks_after, blocks_before + 1);
 
@@ -4348,31 +4354,31 @@ fn reorg_locally_accepted_blocks_across_tenures_succeeds() {
     info!("------------------------- Starting Tenure A -------------------------");
     info!("------------------------- Test Mine Nakamoto Block N -------------------------");
     let mined_blocks = signer_test.running_nodes.nakamoto_blocks_mined.clone();
-    let blocks_before = mined_blocks.load(Ordering::SeqCst);
     let info_before = signer_test
         .stacks_client
         .get_peer_info()
         .expect("Failed to get peer info");
-    let start_time = Instant::now();
     // submit a tx so that the miner will mine a stacks block
     let mut sender_nonce = 0;
     let transfer_tx =
         make_stacks_transfer(&sender_sk, sender_nonce, send_fee, &recipient, send_amt);
     let tx = submit_tx(&http_origin, &transfer_tx);
     info!("Submitted tx {tx} in to mine block N");
-    while mined_blocks.load(Ordering::SeqCst) <= blocks_before {
-        assert!(
-            start_time.elapsed() < short_timeout,
-            "FAIL: Test timed out while waiting for block production",
-        );
-        thread::sleep(Duration::from_secs(1));
-    }
+    wait_for(short_timeout.as_secs(), || {
+        let info_after = signer_test
+            .stacks_client
+            .get_peer_info()
+            .expect("Failed to get peer info");
+        Ok(info_after.stacks_tip_height > info_before.stacks_tip_height)
+    })
+    .expect("Timed out waiting for block to be mined and processed");
 
     sender_nonce += 1;
     let info_after = signer_test
         .stacks_client
         .get_peer_info()
         .expect("Failed to get peer info");
+
     assert_eq!(
         info_before.stacks_tip_height + 1,
         info_after.stacks_tip_height
@@ -4400,13 +4406,12 @@ fn reorg_locally_accepted_blocks_across_tenures_succeeds() {
         make_stacks_transfer(&sender_sk, sender_nonce, send_fee, &recipient, send_amt);
     let tx = submit_tx(&http_origin, &transfer_tx);
     info!("Submitted tx {tx} in to attempt to mine block N+1");
-    let start_time = Instant::now();
     let blocks_before = mined_blocks.load(Ordering::SeqCst);
     let info_before = signer_test
         .stacks_client
         .get_peer_info()
         .expect("Failed to get peer info");
-    loop {
+    wait_for(short_timeout.as_secs(), || {
         let ignored_signers = test_observer::get_stackerdb_chunks()
             .into_iter()
             .flat_map(|chunk| chunk.modified_slots)
@@ -4423,15 +4428,9 @@ fn reorg_locally_accepted_blocks_across_tenures_succeeds() {
                 }
             })
             .collect::<Vec<_>>();
-        if ignored_signers.len() + ignoring_signers.len() == num_signers {
-            break;
-        }
-        assert!(
-            start_time.elapsed() < short_timeout,
-            "FAIL: Test timed out while waiting for block proposal acceptance",
-        );
-        sleep_ms(1000);
-    }
+        Ok(ignored_signers.len() + ignoring_signers.len() == num_signers)
+    })
+    .expect("FAIL: Timed out waiting for block proposal acceptance");
     let blocks_after = mined_blocks.load(Ordering::SeqCst);
     let info_after = signer_test
         .stacks_client
@@ -4464,25 +4463,23 @@ fn reorg_locally_accepted_blocks_across_tenures_succeeds() {
         .lock()
         .unwrap()
         .replace(Vec::new());
-    let mined_blocks = signer_test.running_nodes.nakamoto_blocks_mined.clone();
-    let blocks_before = mined_blocks.load(Ordering::SeqCst);
     let info_before = signer_test
         .stacks_client
         .get_peer_info()
         .expect("Failed to get peer info");
-    let start_time = Instant::now();
     // submit a tx so that the miner will mine a stacks block
     let transfer_tx =
         make_stacks_transfer(&sender_sk, sender_nonce, send_fee, &recipient, send_amt);
     let tx = submit_tx(&http_origin, &transfer_tx);
     info!("Submitted tx {tx} in to mine block N");
-    while mined_blocks.load(Ordering::SeqCst) <= blocks_before {
-        assert!(
-            start_time.elapsed() < short_timeout,
-            "FAIL: Test timed out while waiting for block production",
-        );
-        thread::sleep(Duration::from_secs(1));
-    }
+    wait_for(short_timeout.as_secs(), || {
+        let info_after = signer_test
+            .stacks_client
+            .get_peer_info()
+            .expect("Failed to get peer info");
+        Ok(info_after.stacks_tip_height > info_before.stacks_tip_height)
+    })
+    .expect("Timed out waiting for block to be mined and processed");
 
     let info_after = signer_test
         .stacks_client
@@ -4796,7 +4793,6 @@ fn miner_recovers_when_broadcast_block_delay_across_tenures_occurs() {
     // Induce block N+2 to get mined
     let transfer_tx =
         make_stacks_transfer(&sender_sk, sender_nonce, send_fee, &recipient, send_amt);
-    sender_nonce += 1;
 
     let tx = submit_tx(&http_origin, &transfer_tx);
     info!("Submitted tx {tx} in to attempt to mine block N+2");