Merge branch 'develop' into fix/5159

Author: jcnelson

.github/workflows/bitcoin-tests.yml

@@ -45,7 +45,8 @@ jobs:
           - tests::neon_integrations::liquid_ustx_integration
           - tests::neon_integrations::microblock_fork_poison_integration_test
           - tests::neon_integrations::microblock_integration_test
-          - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY
+          # Disable this flaky test. Microblocks are no longer supported anyways.
+          # - tests::neon_integrations::microblock_large_tx_integration_test_FLAKY
           - tests::neon_integrations::microblock_limit_hit_integration_test
           - tests::neon_integrations::miner_submit_twice
           - tests::neon_integrations::mining_events_integration_test
@@ -86,15 +87,14 @@ jobs:
           - tests::nakamoto_integrations::nakamoto_attempt_time
           - tests::signer::v0::block_proposal_rejection
           - tests::signer::v0::miner_gather_signatures
-          - tests::signer::v0::mine_2_nakamoto_reward_cycles
           - tests::signer::v0::end_of_tenure
           - tests::signer::v0::forked_tenure_okay
           - tests::signer::v0::forked_tenure_invalid
           - tests::signer::v0::empty_sortition
           - tests::signer::v0::bitcoind_forking_test
           - tests::signer::v0::multiple_miners
           - tests::signer::v0::mock_sign_epoch_25
-          - tests::signer::v0::signer_set_rollover
+          - tests::signer::v0::multiple_miners_mock_sign_epoch_25
           - tests::signer::v0::miner_forking
           - tests::signer::v0::reloads_signer_set_in
           - tests::signer::v0::signers_broadcast_signed_blocks
@@ -105,6 +105,10 @@ jobs:
           - tests::signer::v0::locally_rejected_blocks_overriden_by_global_acceptance
           - tests::signer::v0::reorg_locally_accepted_blocks_across_tenures_succeeds
           - tests::signer::v0::miner_recovers_when_broadcast_block_delay_across_tenures_occurs
+          - tests::signer::v0::multiple_miners_with_nakamoto_blocks
+          - tests::signer::v0::partial_tenure_fork
+          - tests::signer::v0::mine_2_nakamoto_reward_cycles
+          - tests::signer::v0::signer_set_rollover
           - tests::nakamoto_integrations::stack_stx_burn_op_integration_test
           - tests::nakamoto_integrations::check_block_heights
           - tests::nakamoto_integrations::clarity_burn_state

CHANGELOG.md

@@ -17,6 +17,28 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
     - `get-tenure-info?` added
     - `get-block-info?` removed
 
+## [2.5.0.0.7]
+
+### Added
+
+- Add warn logs for block validate rejections (#5079)
+- Neon mock miner replay (#5060)
+
+### Changed
+
+- Revert BurnchainHeaderHash serialization change (#5094)
+- boot_to_epoch_3 in SignerTest should wait for a new commit (#5087)
+- Fix block proposal rejection test (#5084)
+- Mock signing revamp (#5070)
+- Multi miner fixes jude (#5040)
+- Remove spurious deadlock condition whenever the sortition DB is opened
+
+## [2.5.0.0.6]
+
+### Changed
+
+- If there is a getchunk/putchunk that fails due to a stale (or future) version NACK, the StackerDB sync state machine should immediately retry sync (#5066)
+
 ## [2.5.0.0.5]
 
 ### Added

clarity/src/vm/database/sqlite.rs

@@ -21,7 +21,7 @@ use rusqlite::{
 };
 use stacks_common::types::chainstate::{BlockHeaderHash, StacksBlockId};
 use stacks_common::types::sqlite::NO_PARAMS;
-use stacks_common::util::db_common::tx_busy_handler;
+use stacks_common::util::db::tx_busy_handler;
 use stacks_common::util::hash::Sha512Trunc256Sum;
 
 use super::clarity_store::{make_contract_hash_key, ContractCommitment};

docs/rpc/openapi.yaml

@@ -568,7 +568,7 @@ paths:
             example:
               $ref: ./api/core-node/post-block-proposal-req.example.json
 
-  /v2/stacker_set/{cycle_number}:
+  /v3/stacker_set/{cycle_number}:
     get:
       summary: Fetch the stacker and signer set information for a given cycle.
       tags:

stacks-common/src/util/db.rs

@@ -0,0 +1,94 @@
+// Copyright (C) 2013-2020 Blockstack PBC, a public benefit corporation
+// Copyright (C) 2020 Stacks Open Internet Foundation
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+use std::backtrace::Backtrace;
+use std::sync::{LazyLock, Mutex};
+use std::thread;
+use std::time::{Duration, Instant};
+
+use hashbrown::HashMap;
+use rand::{thread_rng, Rng};
+use rusqlite::Connection;
+
+use crate::util::sleep_ms;
+
+/// Keep track of DB locks, for deadlock debugging
+///  - **key:** `rusqlite::Connection` debug print
+///  - **value:** Lock holder (thread name + timestamp)
+///
+/// This uses a `Mutex` inside of `LazyLock` because:
+///  - Using `Mutex` alone, it can't be statically initialized because `HashMap::new()` isn't `const`
+///  - Using `LazyLock` alone doesn't allow interior mutability
+static LOCK_TABLE: LazyLock<Mutex<HashMap<String, String>>> =
+    LazyLock::new(|| Mutex::new(HashMap::new()));
+/// Generate timestanps for use in `LOCK_TABLE`
+/// `Instant` is preferable to `SystemTime` because it uses `CLOCK_MONOTONIC` and is not affected by NTP adjustments
+static LOCK_TABLE_TIMER: LazyLock<Instant> = LazyLock::new(Instant::now);
+
+/// Call when using an operation which locks a database
+/// Updates `LOCK_TABLE`
+pub fn update_lock_table(conn: &Connection) {
+    let timestamp = LOCK_TABLE_TIMER.elapsed().as_millis();
+    // The debug format for `Connection` includes the path
+    let k = format!("{conn:?}");
+    let v = format!("{:?}@{timestamp}", thread::current().name());
+    LOCK_TABLE.lock().unwrap().insert(k, v);
+}
+
+/// Called by `rusqlite` if we are waiting too long on a database lock
+/// If called too many times, will assume a deadlock and panic
+pub fn tx_busy_handler(run_count: i32) -> bool {
+    const TIMEOUT: Duration = Duration::from_secs(300);
+    const AVG_SLEEP_TIME_MS: u64 = 100;
+
+    // First, check if this is taking unreasonably long. If so, it's probably a deadlock
+    let run_count = run_count.unsigned_abs();
+    let approx_time_elapsed =
+        Duration::from_millis(AVG_SLEEP_TIME_MS.saturating_mul(u64::from(run_count)));
+    if approx_time_elapsed > TIMEOUT {
+        error!("Deadlock detected. Waited {} seconds (estimated) for database lock. Giving up", approx_time_elapsed.as_secs();
+            "run_count" => run_count,
+            "backtrace" => ?Backtrace::capture()
+        );
+        for (k, v) in LOCK_TABLE.lock().unwrap().iter() {
+            error!("Database '{k}' last locked by {v}");
+        }
+        panic!("Deadlock in thread {:?}", thread::current().name());
+    }
+
+    let mut sleep_time_ms = 2u64.saturating_pow(run_count);
+
+    sleep_time_ms = sleep_time_ms.saturating_add(thread_rng().gen_range(0..sleep_time_ms));
+
+    if sleep_time_ms > AVG_SLEEP_TIME_MS {
+        let jitter = 10;
+        sleep_time_ms =
+            thread_rng().gen_range((AVG_SLEEP_TIME_MS - jitter)..(AVG_SLEEP_TIME_MS + jitter));
+    }
+
+    let msg = format!("Database is locked; sleeping {sleep_time_ms}ms and trying again");
+    if run_count > 10 && run_count % 10 == 0 {
+        warn!("{msg}";
+            "run_count" => run_count,
+            "backtrace" => ?Backtrace::capture()
+        );
+    } else {
+        debug!("{msg}");
+    }
+
+    sleep_ms(sleep_time_ms);
+    true
+}

stacks-common/src/util/mod.rs

@@ -19,6 +19,7 @@ pub mod log;
 #[macro_use]
 pub mod macros;
 pub mod chunked_encoding;
+pub mod db;
 pub mod hash;
 pub mod pair;
 pub mod pipe;
@@ -85,32 +86,6 @@ impl error::Error for HexError {
     }
 }
 
-pub mod db_common {
-    use std::{thread, time};
-
-    use rand::{thread_rng, Rng};
-
-    pub fn tx_busy_handler(run_count: i32) -> bool {
-        let mut sleep_count = 10;
-        if run_count > 0 {
-            sleep_count = 2u64.saturating_pow(run_count as u32);
-        }
-        sleep_count = sleep_count.saturating_add(thread_rng().gen::<u64>() % sleep_count);
-
-        if sleep_count > 5000 {
-            sleep_count = 5000;
-        }
-
-        debug!(
-            "Database is locked; sleeping {}ms and trying again",
-            &sleep_count
-        );
-
-        thread::sleep(time::Duration::from_millis(sleep_count));
-        true
-    }
-}
-
 /// Write any `serde_json` object directly to a file
 pub fn serialize_json_to_file<J, P>(json: &J, path: P) -> Result<(), std::io::Error>
 where

stacks-signer/CHANGELOG.md

@@ -7,6 +7,43 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 
 ## [Unreleased]
 
+### Added
+
+### Changed
+
+## [2.5.0.0.5.3]
+
+### Added
+
+### Changed
+
+- Update node endpoints to match stacks-core release 2.5.0.0.7
+  - `/v2/block_proposal` -> `/v3/block_proposal`
+  - `/v2/stacker_set` -> `/v3/stacker_set`
+
+## [2.5.0.0.5.2]
+
+### Added
+
+### Changed
+
+- Reuse BlockResponse slot for MockSignature message type (#5103)
+
+## [2.5.0.0.5.2-rc1]
+
+### Added
+
+- Signer set handoff integration test (#5037)
+- Add mock signing (#5020)
+- Add versioning info set at build-time (#5016)
+
+### Changed
+
+- Fix out of sync `RPCPeerInfo` with stacks-node (#5033, #5014, #4999)
+- Logging Improvements (#5025)
+- Timeout empty sortition (#5003)
+- Enum for version specific data (#4981)
+
 ## [2.5.0.0.5.1]
 
 ### Added

stacks-signer/src/client/stacks_client.rs

@@ -564,13 +564,13 @@ impl StacksClient {
                 warn!("Failed to parse the GetStackers error response: {e}");
                 backoff::Error::permanent(e.into())
             })?;
-            if &error_data.err_type == GetStackersErrors::NOT_AVAILABLE_ERR_TYPE {
-                return Err(backoff::Error::transient(ClientError::NoSortitionOnChain));
+            if error_data.err_type == GetStackersErrors::NOT_AVAILABLE_ERR_TYPE {
+                Err(backoff::Error::transient(ClientError::NoSortitionOnChain))
             } else {
                 warn!("Got error response ({status}): {}", error_data.err_msg);
-                return Err(backoff::Error::permanent(ClientError::RequestFailure(
+                Err(backoff::Error::permanent(ClientError::RequestFailure(
                     status,
-                )));
+                )))
             }
         };
         let stackers_response =

stacks-signer/src/runloop.rs

@@ -420,7 +420,9 @@ impl<Signer: SignerTrait<T>, T: StacksMessageCodec + Clone + Send + Debug> RunLo
             "reward_cycle_before_refresh" => reward_cycle_before_refresh,
             "current_reward_cycle" => current_reward_cycle,
             "configured_for_current" => Self::is_configured_for_cycle(&self.stacks_signers, current_reward_cycle),
+            "registered_for_current" => Self::is_registered_for_cycle(&self.stacks_signers, current_reward_cycle),
             "configured_for_next" => Self::is_configured_for_cycle(&self.stacks_signers, next_reward_cycle),
+            "registered_for_next" => Self::is_registered_for_cycle(&self.stacks_signers, next_reward_cycle),
             "is_in_next_prepare_phase" => is_in_next_prepare_phase,
         );
 
@@ -430,10 +432,10 @@ impl<Signer: SignerTrait<T>, T: StacksMessageCodec + Clone + Send + Debug> RunLo
         if !Self::is_configured_for_cycle(&self.stacks_signers, current_reward_cycle) {
             self.refresh_signer_config(current_reward_cycle);
         }
-        if is_in_next_prepare_phase {
-            if !Self::is_configured_for_cycle(&self.stacks_signers, next_reward_cycle) {
-                self.refresh_signer_config(next_reward_cycle);
-            }
+        if is_in_next_prepare_phase
+            && !Self::is_configured_for_cycle(&self.stacks_signers, next_reward_cycle)
+        {
+            self.refresh_signer_config(next_reward_cycle);
         }
 
         self.cleanup_stale_signers(current_reward_cycle);
@@ -455,6 +457,17 @@ impl<Signer: SignerTrait<T>, T: StacksMessageCodec + Clone + Send + Debug> RunLo
         signer.reward_cycle() == reward_cycle
     }
 
+    fn is_registered_for_cycle(
+        stacks_signers: &HashMap<u64, ConfiguredSigner<Signer, T>>,
+        reward_cycle: u64,
+    ) -> bool {
+        let Some(signer) = stacks_signers.get(&(reward_cycle % 2)) else {
+            return false;
+        };
+        signer.reward_cycle() == reward_cycle
+            && matches!(signer, ConfiguredSigner::RegisteredSigner(_))
+    }
+
     fn cleanup_stale_signers(&mut self, current_reward_cycle: u64) {
         let mut to_delete = Vec::new();
         for (idx, signer) in &mut self.stacks_signers {