Merge branch 'develop' into fix/relayer-drain-channel

Author: jcnelson

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 
 ## [Unreleased]
 
+### Changed
+- Add index for StacksBlockId to nakamoto block headers table (improves node performance)
+- Remove the panic for reporting DB deadlocks (just error and continue waiting)
+
 ## [3.0.0.0.0]
 
 ### Added

stacks-common/src/util/db.rs

@@ -51,26 +51,25 @@ pub fn update_lock_table(conn: &Connection) {
 /// Called by `rusqlite` if we are waiting too long on a database lock
 /// If called too many times, will assume a deadlock and panic
 pub fn tx_busy_handler(run_count: i32) -> bool {
-    const TIMEOUT: Duration = Duration::from_secs(300);
     const AVG_SLEEP_TIME_MS: u64 = 100;
 
+    // Every ~5min, report an error with a backtrace
+    //   5min * 60s/min * 1_000ms/s / 100ms
+    const ERROR_COUNT: u32 = 3_000;
+
     // First, check if this is taking unreasonably long. If so, it's probably a deadlock
     let run_count = run_count.unsigned_abs();
-    let approx_time_elapsed =
-        Duration::from_millis(AVG_SLEEP_TIME_MS.saturating_mul(u64::from(run_count)));
-    if approx_time_elapsed > TIMEOUT {
-        error!("Deadlock detected. Waited {} seconds (estimated) for database lock. Giving up", approx_time_elapsed.as_secs();
+    if run_count > 0 && run_count % ERROR_COUNT == 0 {
+        error!("Deadlock detected. Waited 5 minutes (estimated) for database lock.";
             "run_count" => run_count,
             "backtrace" => ?Backtrace::capture()
         );
         for (k, v) in LOCK_TABLE.lock().unwrap().iter() {
             error!("Database '{k}' last locked by {v}");
         }
-        panic!("Deadlock in thread {:?}", thread::current().name());
     }
 
     let mut sleep_time_ms = 2u64.saturating_pow(run_count);
-
     sleep_time_ms = sleep_time_ms.saturating_add(thread_rng().gen_range(0..sleep_time_ms));
 
     if sleep_time_ms > AVG_SLEEP_TIME_MS {

stackslib/src/chainstate/nakamoto/mod.rs

@@ -289,6 +289,14 @@ lazy_static! {
         );
     "#,
     ];
+
+    pub static ref NAKAMOTO_CHAINSTATE_SCHEMA_5: [&'static str; 2] = [
+    r#"
+        UPDATE db_config SET version = "8";
+    "#,
+    // Add an index for index block hash in nakamoto block headers
+    "CREATE INDEX IF NOT EXISTS index_block_hash ON nakamoto_block_headers(index_block_hash);",
+    ];
 }
 
 #[cfg(test)]

stackslib/src/chainstate/stacks/db/mod.rs

@@ -55,7 +55,7 @@ use crate::chainstate::burn::{ConsensusHash, ConsensusHashExtensions};
 use crate::chainstate::nakamoto::{
     HeaderTypeNames, NakamotoBlock, NakamotoBlockHeader, NakamotoChainState,
     NakamotoStagingBlocksConn, NAKAMOTO_CHAINSTATE_SCHEMA_1, NAKAMOTO_CHAINSTATE_SCHEMA_2,
-    NAKAMOTO_CHAINSTATE_SCHEMA_3, NAKAMOTO_CHAINSTATE_SCHEMA_4,
+    NAKAMOTO_CHAINSTATE_SCHEMA_3, NAKAMOTO_CHAINSTATE_SCHEMA_4, NAKAMOTO_CHAINSTATE_SCHEMA_5,
 };
 use crate::chainstate::stacks::address::StacksAddressExtensions;
 use crate::chainstate::stacks::boot::*;
@@ -299,14 +299,14 @@ impl DBConfig {
         });
         match epoch_id {
             StacksEpochId::Epoch10 => true,
-            StacksEpochId::Epoch20 => version_u32 >= 1 && version_u32 <= 7,
-            StacksEpochId::Epoch2_05 => version_u32 >= 2 && version_u32 <= 7,
-            StacksEpochId::Epoch21 => version_u32 >= 3 && version_u32 <= 7,
-            StacksEpochId::Epoch22 => version_u32 >= 3 && version_u32 <= 7,
-            StacksEpochId::Epoch23 => version_u32 >= 3 && version_u32 <= 7,
-            StacksEpochId::Epoch24 => version_u32 >= 3 && version_u32 <= 7,
-            StacksEpochId::Epoch25 => version_u32 >= 3 && version_u32 <= 7,
-            StacksEpochId::Epoch30 => version_u32 >= 3 && version_u32 <= 7,
+            StacksEpochId::Epoch20 => version_u32 >= 1 && version_u32 <= 8,
+            StacksEpochId::Epoch2_05 => version_u32 >= 2 && version_u32 <= 8,
+            StacksEpochId::Epoch21 => version_u32 >= 3 && version_u32 <= 8,
+            StacksEpochId::Epoch22 => version_u32 >= 3 && version_u32 <= 8,
+            StacksEpochId::Epoch23 => version_u32 >= 3 && version_u32 <= 8,
+            StacksEpochId::Epoch24 => version_u32 >= 3 && version_u32 <= 8,
+            StacksEpochId::Epoch25 => version_u32 >= 3 && version_u32 <= 8,
+            StacksEpochId::Epoch30 => version_u32 >= 3 && version_u32 <= 8,
         }
     }
 }
@@ -680,7 +680,7 @@ impl<'a> DerefMut for ChainstateTx<'a> {
     }
 }
 
-pub const CHAINSTATE_VERSION: &'static str = "7";
+pub const CHAINSTATE_VERSION: &'static str = "8";
 
 const CHAINSTATE_INITIAL_SCHEMA: &'static [&'static str] = &[
     "PRAGMA foreign_keys = ON;",
@@ -1087,28 +1087,24 @@ impl StacksChainState {
         while db_config.version != CHAINSTATE_VERSION {
             match db_config.version.as_str() {
                 "1" => {
-                    // migrate to 2
                     info!("Migrating chainstate schema from version 1 to 2");
                     for cmd in CHAINSTATE_SCHEMA_2.iter() {
                         tx.execute_batch(cmd)?;
                     }
                 }
                 "2" => {
-                    // migrate to 3
                     info!("Migrating chainstate schema from version 2 to 3");
                     for cmd in CHAINSTATE_SCHEMA_3.iter() {
                         tx.execute_batch(cmd)?;
                     }
                 }
                 "3" => {
-                    // migrate to nakamoto 1
                     info!("Migrating chainstate schema from version 3 to 4: nakamoto support");
                     for cmd in NAKAMOTO_CHAINSTATE_SCHEMA_1.iter() {
                         tx.execute_batch(cmd)?;
                     }
                 }
                 "4" => {
-                    // migrate to nakamoto 2
                     info!(
                         "Migrating chainstate schema from version 4 to 5: fix nakamoto tenure typo"
                     );
@@ -1117,21 +1113,27 @@ impl StacksChainState {
                     }
                 }
                 "5" => {
-                    // migrate to nakamoto 3
                     info!("Migrating chainstate schema from version 5 to 6: adds height_in_tenure field");
                     for cmd in NAKAMOTO_CHAINSTATE_SCHEMA_3.iter() {
                         tx.execute_batch(cmd)?;
                     }
                 }
                 "6" => {
-                    // migrate to nakamoto 3
                     info!(
                         "Migrating chainstate schema from version 6 to 7: adds signer_stats table"
                     );
                     for cmd in NAKAMOTO_CHAINSTATE_SCHEMA_4.iter() {
                         tx.execute_batch(cmd)?;
                     }
                 }
+                "7" => {
+                    info!(
+                        "Migrating chainstate schema from version 7 to 8: add index for nakamoto block headers"
+                    );
+                    for cmd in NAKAMOTO_CHAINSTATE_SCHEMA_5.iter() {
+                        tx.execute_batch(cmd)?;
+                    }
+                }
                 _ => {
                     error!(
                         "Invalid chain state database: expected version = {}, got {}",

stackslib/src/chainstate/stacks/tests/mod.rs

@@ -475,14 +475,14 @@ impl TestStacksNode {
             };
 
             if StacksChainState::has_stored_block(
-                &self.chainstate.db(),
+                self.chainstate.db(),
                 &self.chainstate.blocks_path,
                 &consensus_hash,
                 &bc.block_header_hash,
             )
             .unwrap()
                 && !StacksChainState::is_block_orphaned(
-                    &self.chainstate.db(),
+                    self.chainstate.db(),
                     &consensus_hash,
                     &bc.block_header_hash,
                 )