From a7ae33a4723afbef1fa0b0594d5f7528b2f14908 Mon Sep 17 00:00:00 2001 From: djordon Date: Mon, 12 Jan 2026 21:45:05 +0000 Subject: [PATCH 01/37] Add the postgres queries that update the status of bitcoin blocks --- signer/src/storage/mod.rs | 10 +++ signer/src/storage/postgres/write.rs | 129 ++++++++++++++++++++++++++- 2 files changed, 138 insertions(+), 1 deletion(-) diff --git a/signer/src/storage/mod.rs b/signer/src/storage/mod.rs index 8b901cf2a..8f552f29a 100644 --- a/signer/src/storage/mod.rs +++ b/signer/src/storage/mod.rs @@ -620,4 +620,14 @@ pub trait DbWrite { peer_id: &PeerId, address: Multiaddr, ) -> impl Future> + Send; + + /// Update the is_canonical status for all bitcoin blocks. + /// + /// Marks blocks reachable from the given chain tip as canonical + /// (is_canonical = TRUE) and marks all other blocks as non-canonical + /// (is_canonical = FALSE). + fn update_bitcoin_blocks_canonical_status( + &self, + chain_tip: &model::BitcoinBlockRef, + ) -> impl Future> + Send; } diff --git a/signer/src/storage/postgres/write.rs b/signer/src/storage/postgres/write.rs index 4e4079b47..02933697c 100644 --- a/signer/src/storage/postgres/write.rs +++ b/signer/src/storage/postgres/write.rs @@ -28,8 +28,9 @@ impl PgWrite { ( block_hash , block_height , parent_hash + , is_canonical ) - VALUES ($1, $2, $3) + VALUES ($1, $2, $3, NULL) ON CONFLICT DO NOTHING", ) .bind(block.block_hash) @@ -1001,6 +1002,113 @@ impl PgWrite { Ok(()) } + + /// This function finds the first block on the chain that is reachable + /// from the given chain tip that is already marked as a canonical + /// block in the database. + async fn find_canonical_root<'e, E>( + executor: &'e mut E, + chain_tip: &model::BitcoinBlockHash, + ) -> Result + where + E: 'static, + for<'c> &'c mut E: sqlx::PgExecutor<'c>, + { + // Walk backwards from chain tip to find the first block where + // is_canonical = TRUE + sqlx::query_scalar::<_, i64>( + r#" + -- find_canonical_root + WITH RECURSIVE chain_walk AS ( + -- Start from the chain tip + SELECT + block_hash + , block_height + , parent_hash + , is_canonical + FROM sbtc_signer.bitcoin_blocks + WHERE block_hash = $1 + + UNION ALL + + -- Recursively get parent blocks, stopping when we find a canonical block + SELECT + parent.block_hash + , parent.block_height + , parent.parent_hash + , parent.is_canonical + FROM sbtc_signer.bitcoin_blocks AS parent + JOIN chain_walk AS child + ON parent.block_hash = child.parent_hash + WHERE child.is_canonical IS NOT TRUE + ) + SELECT MIN(block_height) + FROM chain_walk + "#, + ) + .bind(chain_tip) + .fetch_optional(executor) + .await + .map_err(Error::SqlxQuery) + .map(|maybe_height| maybe_height.unwrap_or(0)) + } + + async fn update_bitcoin_blocks_canonical_status<'e, E>( + executor: &'e mut E, + chain_tip: &model::BitcoinBlockRef, + ) -> Result<(), Error> + where + E: 'static, + for<'c> &'c mut E: sqlx::PgExecutor<'c>, + { + // First, walk backwards from chain tip to find the first + // block where is_canonical = TRUE. This block height is the height + // where we'll need to update the canonical status of bitcoin + // blocks. + let min_height = Self::find_canonical_root(executor, &chain_tip.block_hash).await?; + + // Next, update blocks with height greater than min_height + // Build canonical chain and mark those blocks as canonical. + sqlx::query( + r#" + -- update_bitcoin_blocks_canonical_status + WITH RECURSIVE canonical_chain AS ( + -- Start from the chain tip + SELECT + block_hash + , block_height + , parent_hash + FROM sbtc_signer.bitcoin_blocks + WHERE block_hash = $1 + + UNION ALL + + -- Recursively get parent blocks + SELECT + parent.block_hash + , parent.block_height + , parent.parent_hash + FROM sbtc_signer.bitcoin_blocks AS parent + JOIN canonical_chain AS child + ON parent.block_hash = child.parent_hash + WHERE parent.block_height >= $2 + ) + UPDATE sbtc_signer.bitcoin_blocks + SET is_canonical = (block_hash IN ( + SELECT block_hash FROM canonical_chain + )) + WHERE block_height BETWEEN $2 AND $3 + "#, + ) + .bind(chain_tip.block_hash) + .bind(min_height) + .bind(chain_tip.block_height) + .execute(executor) + .await + .map_err(Error::SqlxQuery)?; + + Ok(()) + } } impl DbWrite for PgStore { @@ -1161,6 +1269,17 @@ impl DbWrite for PgStore { ) .await } + + async fn update_bitcoin_blocks_canonical_status( + &self, + chain_tip: &model::BitcoinBlockRef, + ) -> Result<(), Error> { + PgWrite::update_bitcoin_blocks_canonical_status( + self.get_connection().await?.as_mut(), + chain_tip, + ) + .await + } } impl DbWrite for PgTransaction<'_> { @@ -1332,4 +1451,12 @@ impl DbWrite for PgTransaction<'_> { let mut tx = self.tx.lock().await; PgWrite::update_peer_connection(tx.as_mut(), pub_key, peer_id, address).await } + + async fn update_bitcoin_blocks_canonical_status( + &self, + chain_tip: &model::BitcoinBlockRef, + ) -> Result<(), Error> { + let mut tx = self.tx.lock().await; + PgWrite::update_bitcoin_blocks_canonical_status(tx.as_mut(), chain_tip).await + } } From c6cb8eb269fe6da738befd62ef0326c9a05cd57a Mon Sep 17 00:00:00 2001 From: djordon Date: Mon, 12 Jan 2026 21:45:40 +0000 Subject: [PATCH 02/37] Update the in-memory database --- signer/src/storage/memory/store.rs | 3 ++ signer/src/storage/memory/write.rs | 45 ++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/signer/src/storage/memory/store.rs b/signer/src/storage/memory/store.rs index 58b55f45e..cc2296690 100644 --- a/signer/src/storage/memory/store.rs +++ b/signer/src/storage/memory/store.rs @@ -42,6 +42,9 @@ pub struct Store { /// Bitcoin blocks pub bitcoin_blocks: HashMap, + /// Bitcoin blocks + pub canonical_bitcoin_blocks: HashMap, + /// Stacks blocks pub stacks_blocks: HashMap, diff --git a/signer/src/storage/memory/write.rs b/signer/src/storage/memory/write.rs index 9fb8f4945..e8153843c 100644 --- a/signer/src/storage/memory/write.rs +++ b/signer/src/storage/memory/write.rs @@ -385,6 +385,42 @@ impl DbWrite for SharedStore { Ok(()) } + + async fn update_bitcoin_blocks_canonical_status( + &self, + chain_tip: &model::BitcoinBlockRef, + ) -> Result<(), Error> { + let mut store = self.lock().await; + store.version += 1; + + // Then, recursively mark all blocks reachable from the chain tip as canonical + if let Some(block) = store.bitcoin_blocks.get(&chain_tip.block_hash).cloned() { + store + .canonical_bitcoin_blocks + .insert(chain_tip.block_hash, block); + } + + let mut current_block_hash = chain_tip.block_hash; + while let Some(block) = store + .canonical_bitcoin_blocks + .get(¤t_block_hash) + .cloned() + { + match store.bitcoin_blocks.get(&block.parent_hash).cloned() { + Some(parent) => { + store + .canonical_bitcoin_blocks + .insert(block.parent_hash, parent); + current_block_hash = block.parent_hash; + } + None => { + break; + } + } + } + + Ok(()) + } } impl DbWrite for InMemoryTransaction { @@ -539,4 +575,13 @@ impl DbWrite for InMemoryTransaction { .update_peer_connection(pub_key, peer_id, address) .await } + + async fn update_bitcoin_blocks_canonical_status( + &self, + chain_tip: &model::BitcoinBlockRef, + ) -> Result<(), Error> { + self.store + .update_bitcoin_blocks_canonical_status(chain_tip) + .await + } } From 51299a699ba3628de4791943d4c9006bcae7fc65 Mon Sep 17 00:00:00 2001 From: djordon Date: Mon, 12 Jan 2026 21:47:26 +0000 Subject: [PATCH 03/37] Update bitcoin blocks in the block observer --- signer/src/bitcoin/rpc.rs | 9 +++++++++ signer/src/block_observer.rs | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/signer/src/bitcoin/rpc.rs b/signer/src/bitcoin/rpc.rs index 8b3bc7f43..18444ecaa 100644 --- a/signer/src/bitcoin/rpc.rs +++ b/signer/src/bitcoin/rpc.rs @@ -25,6 +25,7 @@ use url::Url; use crate::bitcoin::BitcoinInteract; use crate::error::Error; use crate::storage::model::BitcoinBlockHeight; +use crate::storage::model::BitcoinBlockRef; use super::GetTransactionFeeResult; use super::TransactionLookupHint; @@ -316,6 +317,14 @@ pub struct BitcoinBlockHeader { pub previous_block_hash: BlockHash, } +impl From for BitcoinBlockRef { + fn from(header: BitcoinBlockHeader) -> Self { + BitcoinBlockRef { + block_hash: header.hash.into(), + block_height: header.height, + } + } +} /// A struct representing the recommended fee, in sats per vbyte, from a /// particular source. #[derive(Debug, Clone, Copy, PartialEq)] diff --git a/signer/src/block_observer.rs b/signer/src/block_observer.rs index 1f5c5a571..8e6968585 100644 --- a/signer/src/block_observer.rs +++ b/signer/src/block_observer.rs @@ -294,7 +294,8 @@ impl BlockObserver { } /// Find the parent blocks from the given block that are also missing - /// from our database. + /// from our database. The results are returned from blocks with the + /// least height to the greatest height. /// /// # Notes /// @@ -363,13 +364,25 @@ impl BlockObserver { /// This means that if we stop processing blocks midway though, /// subsequent calls to this function will properly pick up from where /// we left off and update the database. + #[tracing::instrument(skip_all, fields(%block_hash))] async fn process_bitcoin_blocks_until(&self, block_hash: BlockHash) -> Result<(), Error> { let block_headers = self.next_headers_to_process(block_hash).await?; + // This should be the header for the above block hash. + let chain_tip = block_headers.last().cloned(); + for block_header in block_headers { self.process_bitcoin_block(block_header).await?; } + if let Some(chain_tip) = chain_tip.map(BitcoinBlockRef::from) { + tracing::info!("updating block canonical status"); + + let db = self.context.get_storage_mut(); + db.update_bitcoin_blocks_canonical_status(&chain_tip) + .await?; + } + Ok(()) } From 68e4de0136363a6d73f4ea88b290086077b69c60 Mon Sep 17 00:00:00 2001 From: djordon Date: Mon, 12 Jan 2026 21:47:59 +0000 Subject: [PATCH 04/37] Add a new column for canonical blocks --- .../0021__add_is_canonical_to_bitcoin_blocks.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 signer/migrations/0021__add_is_canonical_to_bitcoin_blocks.sql diff --git a/signer/migrations/0021__add_is_canonical_to_bitcoin_blocks.sql b/signer/migrations/0021__add_is_canonical_to_bitcoin_blocks.sql new file mode 100644 index 000000000..953b23c92 --- /dev/null +++ b/signer/migrations/0021__add_is_canonical_to_bitcoin_blocks.sql @@ -0,0 +1,9 @@ +-- Add is_canonical column to bitcoin_blocks table to track which blocks +-- are on the canonical Bitcoin blockchain. + +ALTER TABLE sbtc_signer.bitcoin_blocks +ADD COLUMN is_canonical BOOLEAN; + +-- Create an index on is_canonical for efficient queries filtering +-- canonical vs non-canonical blocks. +CREATE INDEX ix_bitcoin_blocks_is_canonical ON sbtc_signer.bitcoin_blocks(is_canonical); From 19de0322b481da673a22345e7cb31926939f86dd Mon Sep 17 00:00:00 2001 From: djordon Date: Mon, 12 Jan 2026 21:49:38 +0000 Subject: [PATCH 05/37] use a better name for the new trait query --- signer/src/block_observer.rs | 3 +-- signer/src/storage/memory/write.rs | 8 +++----- signer/src/storage/mod.rs | 2 +- signer/src/storage/postgres/write.rs | 17 +++++++---------- 4 files changed, 12 insertions(+), 18 deletions(-) diff --git a/signer/src/block_observer.rs b/signer/src/block_observer.rs index 8e6968585..e67b50198 100644 --- a/signer/src/block_observer.rs +++ b/signer/src/block_observer.rs @@ -379,8 +379,7 @@ impl BlockObserver { tracing::info!("updating block canonical status"); let db = self.context.get_storage_mut(); - db.update_bitcoin_blocks_canonical_status(&chain_tip) - .await?; + db.set_canonical_bitcoin_blockchain(&chain_tip).await?; } Ok(()) diff --git a/signer/src/storage/memory/write.rs b/signer/src/storage/memory/write.rs index e8153843c..f0b226e8b 100644 --- a/signer/src/storage/memory/write.rs +++ b/signer/src/storage/memory/write.rs @@ -386,7 +386,7 @@ impl DbWrite for SharedStore { Ok(()) } - async fn update_bitcoin_blocks_canonical_status( + async fn set_canonical_bitcoin_blockchain( &self, chain_tip: &model::BitcoinBlockRef, ) -> Result<(), Error> { @@ -576,12 +576,10 @@ impl DbWrite for InMemoryTransaction { .await } - async fn update_bitcoin_blocks_canonical_status( + async fn set_canonical_bitcoin_blockchain( &self, chain_tip: &model::BitcoinBlockRef, ) -> Result<(), Error> { - self.store - .update_bitcoin_blocks_canonical_status(chain_tip) - .await + self.store.set_canonical_bitcoin_blockchain(chain_tip).await } } diff --git a/signer/src/storage/mod.rs b/signer/src/storage/mod.rs index 8f552f29a..ca5a219e8 100644 --- a/signer/src/storage/mod.rs +++ b/signer/src/storage/mod.rs @@ -626,7 +626,7 @@ pub trait DbWrite { /// Marks blocks reachable from the given chain tip as canonical /// (is_canonical = TRUE) and marks all other blocks as non-canonical /// (is_canonical = FALSE). - fn update_bitcoin_blocks_canonical_status( + fn set_canonical_bitcoin_blockchain( &self, chain_tip: &model::BitcoinBlockRef, ) -> impl Future> + Send; diff --git a/signer/src/storage/postgres/write.rs b/signer/src/storage/postgres/write.rs index 02933697c..2d2c614fd 100644 --- a/signer/src/storage/postgres/write.rs +++ b/signer/src/storage/postgres/write.rs @@ -1053,7 +1053,7 @@ impl PgWrite { .map(|maybe_height| maybe_height.unwrap_or(0)) } - async fn update_bitcoin_blocks_canonical_status<'e, E>( + async fn set_canonical_bitcoin_blockchain<'e, E>( executor: &'e mut E, chain_tip: &model::BitcoinBlockRef, ) -> Result<(), Error> @@ -1071,7 +1071,7 @@ impl PgWrite { // Build canonical chain and mark those blocks as canonical. sqlx::query( r#" - -- update_bitcoin_blocks_canonical_status + -- set_canonical_bitcoin_blockchain WITH RECURSIVE canonical_chain AS ( -- Start from the chain tip SELECT @@ -1270,15 +1270,12 @@ impl DbWrite for PgStore { .await } - async fn update_bitcoin_blocks_canonical_status( + async fn set_canonical_bitcoin_blockchain( &self, chain_tip: &model::BitcoinBlockRef, ) -> Result<(), Error> { - PgWrite::update_bitcoin_blocks_canonical_status( - self.get_connection().await?.as_mut(), - chain_tip, - ) - .await + PgWrite::set_canonical_bitcoin_blockchain(self.get_connection().await?.as_mut(), chain_tip) + .await } } @@ -1452,11 +1449,11 @@ impl DbWrite for PgTransaction<'_> { PgWrite::update_peer_connection(tx.as_mut(), pub_key, peer_id, address).await } - async fn update_bitcoin_blocks_canonical_status( + async fn set_canonical_bitcoin_blockchain( &self, chain_tip: &model::BitcoinBlockRef, ) -> Result<(), Error> { let mut tx = self.tx.lock().await; - PgWrite::update_bitcoin_blocks_canonical_status(tx.as_mut(), chain_tip).await + PgWrite::set_canonical_bitcoin_blockchain(tx.as_mut(), chain_tip).await } } From 7c37e5ddde07643d3de3f0e83f93cc70ed5afabb Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 18:05:57 +0000 Subject: [PATCH 06/37] change the behavior of the update function --- signer/src/bitcoin/rpc.rs | 9 --------- signer/src/block_observer.rs | 5 +++-- signer/src/storage/memory/write.rs | 12 +++++------- signer/src/storage/mod.rs | 5 +++-- signer/src/storage/postgres/write.rs | 15 +++++++-------- 5 files changed, 18 insertions(+), 28 deletions(-) diff --git a/signer/src/bitcoin/rpc.rs b/signer/src/bitcoin/rpc.rs index 18444ecaa..8b3bc7f43 100644 --- a/signer/src/bitcoin/rpc.rs +++ b/signer/src/bitcoin/rpc.rs @@ -25,7 +25,6 @@ use url::Url; use crate::bitcoin::BitcoinInteract; use crate::error::Error; use crate::storage::model::BitcoinBlockHeight; -use crate::storage::model::BitcoinBlockRef; use super::GetTransactionFeeResult; use super::TransactionLookupHint; @@ -317,14 +316,6 @@ pub struct BitcoinBlockHeader { pub previous_block_hash: BlockHash, } -impl From for BitcoinBlockRef { - fn from(header: BitcoinBlockHeader) -> Self { - BitcoinBlockRef { - block_hash: header.hash.into(), - block_height: header.height, - } - } -} /// A struct representing the recommended fee, in sats per vbyte, from a /// particular source. #[derive(Debug, Clone, Copy, PartialEq)] diff --git a/signer/src/block_observer.rs b/signer/src/block_observer.rs index e67b50198..0468d7f03 100644 --- a/signer/src/block_observer.rs +++ b/signer/src/block_observer.rs @@ -368,14 +368,15 @@ impl BlockObserver { async fn process_bitcoin_blocks_until(&self, block_hash: BlockHash) -> Result<(), Error> { let block_headers = self.next_headers_to_process(block_hash).await?; - // This should be the header for the above block hash. let chain_tip = block_headers.last().cloned(); + // This should be the header for the above block hash. + debug_assert_eq!(chain_tip.as_ref().map(|h| h.hash), Some(block_hash)); for block_header in block_headers { self.process_bitcoin_block(block_header).await?; } - if let Some(chain_tip) = chain_tip.map(BitcoinBlockRef::from) { + if let Some(chain_tip) = chain_tip.map(|h| h.hash.into()) { tracing::info!("updating block canonical status"); let db = self.context.get_storage_mut(); diff --git a/signer/src/storage/memory/write.rs b/signer/src/storage/memory/write.rs index f0b226e8b..bebab55ad 100644 --- a/signer/src/storage/memory/write.rs +++ b/signer/src/storage/memory/write.rs @@ -388,19 +388,17 @@ impl DbWrite for SharedStore { async fn set_canonical_bitcoin_blockchain( &self, - chain_tip: &model::BitcoinBlockRef, + chain_tip: &model::BitcoinBlockHash, ) -> Result<(), Error> { let mut store = self.lock().await; store.version += 1; // Then, recursively mark all blocks reachable from the chain tip as canonical - if let Some(block) = store.bitcoin_blocks.get(&chain_tip.block_hash).cloned() { - store - .canonical_bitcoin_blocks - .insert(chain_tip.block_hash, block); + if let Some(block) = store.bitcoin_blocks.get(&chain_tip).cloned() { + store.canonical_bitcoin_blocks.insert(*chain_tip, block); } - let mut current_block_hash = chain_tip.block_hash; + let mut current_block_hash = *chain_tip; while let Some(block) = store .canonical_bitcoin_blocks .get(¤t_block_hash) @@ -578,7 +576,7 @@ impl DbWrite for InMemoryTransaction { async fn set_canonical_bitcoin_blockchain( &self, - chain_tip: &model::BitcoinBlockRef, + chain_tip: &model::BitcoinBlockHash, ) -> Result<(), Error> { self.store.set_canonical_bitcoin_blockchain(chain_tip).await } diff --git a/signer/src/storage/mod.rs b/signer/src/storage/mod.rs index ca5a219e8..ac301fba8 100644 --- a/signer/src/storage/mod.rs +++ b/signer/src/storage/mod.rs @@ -625,9 +625,10 @@ pub trait DbWrite { /// /// Marks blocks reachable from the given chain tip as canonical /// (is_canonical = TRUE) and marks all other blocks as non-canonical - /// (is_canonical = FALSE). + /// (is_canonical = FALSE). This includes blocks that may have a height + /// that is greater than the height of the given chain tip. fn set_canonical_bitcoin_blockchain( &self, - chain_tip: &model::BitcoinBlockRef, + chain_tip: &model::BitcoinBlockHash, ) -> impl Future> + Send; } diff --git a/signer/src/storage/postgres/write.rs b/signer/src/storage/postgres/write.rs index 2d2c614fd..5276750bb 100644 --- a/signer/src/storage/postgres/write.rs +++ b/signer/src/storage/postgres/write.rs @@ -1006,7 +1006,7 @@ impl PgWrite { /// This function finds the first block on the chain that is reachable /// from the given chain tip that is already marked as a canonical /// block in the database. - async fn find_canonical_root<'e, E>( + async fn find_bitcoin_canonical_root<'e, E>( executor: &'e mut E, chain_tip: &model::BitcoinBlockHash, ) -> Result @@ -1055,7 +1055,7 @@ impl PgWrite { async fn set_canonical_bitcoin_blockchain<'e, E>( executor: &'e mut E, - chain_tip: &model::BitcoinBlockRef, + chain_tip: &model::BitcoinBlockHash, ) -> Result<(), Error> where E: 'static, @@ -1065,7 +1065,7 @@ impl PgWrite { // block where is_canonical = TRUE. This block height is the height // where we'll need to update the canonical status of bitcoin // blocks. - let min_height = Self::find_canonical_root(executor, &chain_tip.block_hash).await?; + let min_height = Self::find_bitcoin_canonical_root(executor, chain_tip).await?; // Next, update blocks with height greater than min_height // Build canonical chain and mark those blocks as canonical. @@ -1097,12 +1097,11 @@ impl PgWrite { SET is_canonical = (block_hash IN ( SELECT block_hash FROM canonical_chain )) - WHERE block_height BETWEEN $2 AND $3 + WHERE block_height >= $2 "#, ) - .bind(chain_tip.block_hash) + .bind(chain_tip) .bind(min_height) - .bind(chain_tip.block_height) .execute(executor) .await .map_err(Error::SqlxQuery)?; @@ -1272,7 +1271,7 @@ impl DbWrite for PgStore { async fn set_canonical_bitcoin_blockchain( &self, - chain_tip: &model::BitcoinBlockRef, + chain_tip: &model::BitcoinBlockHash, ) -> Result<(), Error> { PgWrite::set_canonical_bitcoin_blockchain(self.get_connection().await?.as_mut(), chain_tip) .await @@ -1451,7 +1450,7 @@ impl DbWrite for PgTransaction<'_> { async fn set_canonical_bitcoin_blockchain( &self, - chain_tip: &model::BitcoinBlockRef, + chain_tip: &model::BitcoinBlockHash, ) -> Result<(), Error> { let mut tx = self.tx.lock().await; PgWrite::set_canonical_bitcoin_blockchain(tx.as_mut(), chain_tip).await From d577cf2af44dd2030e1d788b74284f3adfa6f111 Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 20:45:49 +0000 Subject: [PATCH 07/37] Add two integration tests for the new query --- signer/src/testing/blocks.rs | 57 +++++-- signer/tests/integration/postgres.rs | 217 +++++++++++++++++++++++++++ 2 files changed, 265 insertions(+), 9 deletions(-) diff --git a/signer/src/testing/blocks.rs b/signer/src/testing/blocks.rs index 86872f773..2281e7e16 100644 --- a/signer/src/testing/blocks.rs +++ b/signer/src/testing/blocks.rs @@ -1,6 +1,9 @@ //! Various utilities for generating and manipulating chains of bitcoin and //! stacks blocks for testing purposes. +use std::collections::BTreeMap; +use std::collections::btree_map; + use fake::Fake as _; use fake::Faker; @@ -13,14 +16,15 @@ use crate::storage::model::StacksBlockHeight; /// Represents a naive, sequential chain of bitcoin blocks and provides basic /// functionality for manipulation. Does not handle forks/branches. -pub struct BitcoinChain(Vec); +#[derive(Debug, Clone)] +pub struct BitcoinChain(BTreeMap); impl<'a> IntoIterator for &'a BitcoinChain { type Item = &'a BitcoinBlock; - type IntoIter = std::slice::Iter<'a, BitcoinBlock>; + type IntoIter = std::collections::btree_map::Values<'a, BitcoinBlockHeight, BitcoinBlock>; fn into_iter(self) -> Self::IntoIter { - self.0.iter() + self.0.values() } } @@ -40,7 +44,7 @@ impl BitcoinChain { /// block will have a height one greater than the previous block and a /// parent hash equal to the hash of the previous block. pub fn new() -> Self { - Self(vec![BitcoinBlock::new_genesis()]) + Self(BTreeMap::from([(0u64.into(), BitcoinBlock::new_genesis())])) } /// Generate a new chain of bitcoin blocks with a length equal to `length`. @@ -63,25 +67,26 @@ impl BitcoinChain { pub fn generate_blocks(&mut self, length: usize) -> Vec<&BitcoinBlock> { for _ in 0..length { let new_block = self.chain_tip().new_child(); - self.0.push(new_block); + self.0.insert(new_block.block_height, new_block); } - self.0[(self.0.len() - length)..].iter().collect() + let start: BitcoinBlockHeight = ((self.0.len() - length) as u64).into(); + self.0.range(start..).map(|(_, block)| block).collect() } /// Gets the first block in the chain. pub fn first_block(&self) -> &BitcoinBlock { - self.0.first().unwrap() + self.0.first_key_value().unwrap().1 } /// Gets the last block in the chain. pub fn chain_tip(&self) -> &BitcoinBlock { - self.0.last().unwrap() + self.0.last_key_value().unwrap().1 } /// Gets the nth block in the chain, if it exists. pub fn nth_block_checked(&self, height: BitcoinBlockHeight) -> Option<&BitcoinBlock> { - self.0.get(*height as usize) + self.0.get(&height) } /// Gets the nth block in the chain, panicking if it does not exist. @@ -89,6 +94,40 @@ impl BitcoinChain { self.nth_block_checked(height) .expect("no nth bitcoin block (index out of range)") } + + /// Create a new fork of the chain at the given height. + /// + /// The fork will contain the blocks up to, but not including, the + /// given height. It will then generate `num_blocks` new blocks, and + /// add them to the chain starting at the given height, returning the + /// resulting chain. + pub fn fork_at_height(&self, height: H, num_blocks: usize) -> Self + where + H: Into, + { + let fork: BTreeMap = self + .0 + .range(..height.into()) + .map(|(height, block)| (*height, block.clone())) + .collect(); + + let mut fork_chain = Self(fork); + + for _ in 0..num_blocks { + let new_block = fork_chain.chain_tip().new_child(); + fork_chain.0.insert(new_block.block_height, new_block); + } + + fork_chain + } + + /// Get a range of blocks from the chain. + pub fn range(&self, range: R) -> btree_map::Range<'_, BitcoinBlockHeight, BitcoinBlock> + where + R: std::ops::RangeBounds, + { + self.0.range(range) + } } impl BitcoinBlock { diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index 3441f63db..356234e3f 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -7733,3 +7733,220 @@ mod sqlx_transactions { Ok(()) } } + +mod canonical_bitcoin_blockchain { + use super::*; + use signer::testing::blocks::BitcoinChain; + + // Returns the value of the is_canonical column for the given block + // hash is true or false. If the given block hash is missing than this + // function panics. + async fn is_block_canonical(db: &PgStore, block_hash: &BitcoinBlockHash) -> Option { + sqlx::query_scalar( + "SELECT is_canonical FROM sbtc_signer.bitcoin_blocks WHERE block_hash = $1", + ) + .bind(block_hash) + .fetch_one(db.pool()) + .await + .unwrap() + } + + /// Set the is_canonical column to NULL for all bitcoin blocks. + async fn clear_is_canonical_bitcoin_blocks(db: &PgStore) { + sqlx::query("UPDATE sbtc_signer.bitcoin_blocks SET is_canonical = NULL") + .execute(db.pool()) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_set_canonical_bitcoin_blockchain() { + let db = testing::storage::new_test_database().await; + + // Create a chain of 5 bitcoin blocks + let canonical_chain = BitcoinChain::new_with_length(5); + + // Insert all blocks from the canonical chain into the database + for block in &canonical_chain { + db.write_bitcoin_block(block).await.unwrap(); + } + + // Get the chain tip from the canonical chain + let chain_tip = canonical_chain.chain_tip().block_hash; + + // Initially, all blocks should have is_canonical = NULL + // Verify that no blocks have is_canonical IS NOT NULL + let has_non_null_canonical: bool = sqlx::query_scalar( + "SELECT EXISTS( + SELECT 1 + FROM sbtc_signer.bitcoin_blocks + WHERE is_canonical IS NOT NULL + )", + ) + .fetch_one(db.pool()) + .await + .unwrap(); + + assert!(!has_non_null_canonical); + + // Call set_canonical_bitcoin_blockchain with the chain tip + db.set_canonical_bitcoin_blockchain(&chain_tip) + .await + .unwrap(); + + // Verify that all blocks in the canonical chain are marked as canonical + for block in &canonical_chain { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + + assert_eq!(is_canonical, Some(true)); + } + + // Create a fork: blocks at the same height as some canonical blocks but with different hashes + // This fork starts at height 2 (the third block) + let fork_chain = canonical_chain.fork_at_height(2u64, 2); + + for block in &fork_chain { + db.write_bitcoin_block(block).await.unwrap(); + } + + let fork_block_1 = fork_chain.nth_block(2u64.into()); + let fork_block_2 = fork_chain.nth_block(3u64.into()); + + // Verify that fork blocks are marked as non-canonical + let fork_1_is_canonical = is_block_canonical(&db, &fork_block_1.block_hash).await; + assert_eq!(fork_1_is_canonical, None); + + let fork_2_is_canonical = is_block_canonical(&db, &fork_block_2.block_hash).await; + assert_eq!(fork_2_is_canonical, None); + + // These new blocks aren't part of the canonical chain, so they + // should be marked as non-canonical. In order to make sure that + // they get marked as such, we need to clear the is_canonical + // column from some of the blocks in the table. + clear_is_canonical_bitcoin_blocks(&db).await; + db.set_canonical_bitcoin_blockchain(&chain_tip) + .await + .unwrap(); + + // Verify that fork blocks are marked as non-canonical + let fork_1_is_canonical = is_block_canonical(&db, &fork_block_1.block_hash).await; + assert_eq!(fork_1_is_canonical, Some(false)); + + let fork_2_is_canonical = is_block_canonical(&db, &fork_block_2.block_hash).await; + assert_eq!(fork_2_is_canonical, Some(false)); + + // Verify that all blocks at the same height as fork blocks but in the canonical chain + // are marked as canonical + let canonical_block_at_fork_height = canonical_chain.nth_block(fork_block_1.block_height); + let canonical_at_fork_height_is_canonical = + is_block_canonical(&db, &canonical_block_at_fork_height.block_hash).await; + assert_eq!(canonical_at_fork_height_is_canonical, Some(true)); + + testing::storage::drop_db(db).await; + } + + #[tokio::test] + async fn test_set_canonical_bitcoin_blockchain_with_fork() { + let db = testing::storage::new_test_database().await; + + // Create two blockchains, one of length 10 and another of length + // 20, where one forks the other at height 3 (the block with height + // 3 differs from one to the other but their earlier blocks are the + // same) + let main_chain = BitcoinChain::new_with_length(10); + let fork_chain = main_chain.fork_at_height(3u64, 17); + // These are the ranges for the block heights that didn't change, + // so blocks from both chains should always be canonical. + let stable_block_heights = ..BitcoinBlockHeight::from(3u64); + // These are the ranges for the block heights that changed, so + // blocks from one chain will always be non-canonical. + let forked_block_heights = BitcoinBlockHeight::from(3u64)..; + + // Write both chains to the database + for block in (&main_chain).into_iter().chain(&fork_chain) { + db.write_bitcoin_block(block).await.unwrap(); + } + + // Set the main chain as canonical + let main_chain_tip = main_chain.chain_tip().block_hash; + db.set_canonical_bitcoin_blockchain(&main_chain_tip) + .await + .unwrap(); + + // Verify that all blocks in the main chain are marked as canonical + for block in &main_chain { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + assert_eq!(is_canonical, Some(true)); + } + + // Check that fork chain blocks with height 0 to 2 are canonical, + // since they are also part of the main chain + for (_, block) in fork_chain.range(stable_block_heights.clone()) { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + assert_eq!(is_canonical, Some(true)); + } + for (_, block) in fork_chain.range(forked_block_heights.clone()) { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + assert_eq!(is_canonical, Some(false)); + } + + // Set the fork chain as canonical from its chain tip + let fork_chain_tip = fork_chain.chain_tip().block_hash; + db.set_canonical_bitcoin_blockchain(&fork_chain_tip) + .await + .unwrap(); + + // Check that all fork chain blocks are canonical + for block in &fork_chain { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + assert_eq!(is_canonical, Some(true)); + } + + // Check that original main chain blocks with height 0 to 2 are canonical + for (_, block) in main_chain.range(stable_block_heights.clone()) { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + assert_eq!(is_canonical, Some(true)); + } + + // Check that original main chain blocks with height 3 to 9 are + // non-canonical + for (_, block) in main_chain.range(forked_block_heights.clone()) { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + assert_eq!(is_canonical, Some(false)); + } + + // Okay, now let's make the main chain canonical again, just to + // make sure that things switch back as expected. + db.set_canonical_bitcoin_blockchain(&main_chain_tip) + .await + .unwrap(); + + // Verify that all blocks in the main chain are marked as canonical + // again. + for block in &main_chain { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + assert_eq!(is_canonical, Some(true)); + } + + // Check that fork chain blocks with height 0 to 2 are canonical, + // since they are also part of the main chain + for (_, block) in fork_chain.range(stable_block_heights.clone()) { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + assert_eq!(is_canonical, Some(true)); + } + for (_, block) in fork_chain.range(forked_block_heights.clone()) { + let is_canonical = is_block_canonical(&db, &block.block_hash).await; + assert_eq!(is_canonical, Some(false)); + } + + // Now to make sure that things aren't messed up because the ranges + // are empty iterators. + assert_eq!(fork_chain.range(stable_block_heights.clone()).count(), 3); + assert_eq!(fork_chain.range(forked_block_heights.clone()).count(), 17); + + assert_eq!(main_chain.range(stable_block_heights).count(), 3); + assert_eq!(main_chain.range(forked_block_heights).count(), 7); + + testing::storage::drop_db(db).await; + } +} From d5ef4e03516c370e96286431968e75ce5876a6c4 Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 21:42:42 +0000 Subject: [PATCH 08/37] fix up the postgres tests by moving the helper function to a more central place --- signer/src/testing/storage/postgres.rs | 15 +++++++ signer/tests/integration/postgres.rs | 59 +++++++++++++------------- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/signer/src/testing/storage/postgres.rs b/signer/src/testing/storage/postgres.rs index 317ac6d86..0f2f3cf32 100644 --- a/signer/src/testing/storage/postgres.rs +++ b/signer/src/testing/storage/postgres.rs @@ -36,4 +36,19 @@ impl PgStore { .await .map_err(Error::SqlxQuery) } + + /// Returns the value of the is_canonical column for the given block + /// hash. If the given block hash is missing then an error is returned. + pub async fn is_block_canonical( + &self, + block_hash: &model::BitcoinBlockHash, + ) -> Result, Error> { + sqlx::query_scalar( + "SELECT is_canonical FROM sbtc_signer.bitcoin_blocks WHERE block_hash = $1", + ) + .bind(block_hash) + .fetch_one(self.pool()) + .await + .map_err(Error::SqlxQuery) + } } diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index 356234e3f..a53ff41ee 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -7738,19 +7738,6 @@ mod canonical_bitcoin_blockchain { use super::*; use signer::testing::blocks::BitcoinChain; - // Returns the value of the is_canonical column for the given block - // hash is true or false. If the given block hash is missing than this - // function panics. - async fn is_block_canonical(db: &PgStore, block_hash: &BitcoinBlockHash) -> Option { - sqlx::query_scalar( - "SELECT is_canonical FROM sbtc_signer.bitcoin_blocks WHERE block_hash = $1", - ) - .bind(block_hash) - .fetch_one(db.pool()) - .await - .unwrap() - } - /// Set the is_canonical column to NULL for all bitcoin blocks. async fn clear_is_canonical_bitcoin_blocks(db: &PgStore) { sqlx::query("UPDATE sbtc_signer.bitcoin_blocks SET is_canonical = NULL") @@ -7796,7 +7783,7 @@ mod canonical_bitcoin_blockchain { // Verify that all blocks in the canonical chain are marked as canonical for block in &canonical_chain { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } @@ -7813,10 +7800,16 @@ mod canonical_bitcoin_blockchain { let fork_block_2 = fork_chain.nth_block(3u64.into()); // Verify that fork blocks are marked as non-canonical - let fork_1_is_canonical = is_block_canonical(&db, &fork_block_1.block_hash).await; + let fork_1_is_canonical = db + .is_block_canonical(&fork_block_1.block_hash) + .await + .unwrap(); assert_eq!(fork_1_is_canonical, None); - let fork_2_is_canonical = is_block_canonical(&db, &fork_block_2.block_hash).await; + let fork_2_is_canonical = db + .is_block_canonical(&fork_block_2.block_hash) + .await + .unwrap(); assert_eq!(fork_2_is_canonical, None); // These new blocks aren't part of the canonical chain, so they @@ -7829,17 +7822,25 @@ mod canonical_bitcoin_blockchain { .unwrap(); // Verify that fork blocks are marked as non-canonical - let fork_1_is_canonical = is_block_canonical(&db, &fork_block_1.block_hash).await; + let fork_1_is_canonical = db + .is_block_canonical(&fork_block_1.block_hash) + .await + .unwrap(); assert_eq!(fork_1_is_canonical, Some(false)); - let fork_2_is_canonical = is_block_canonical(&db, &fork_block_2.block_hash).await; + let fork_2_is_canonical = db + .is_block_canonical(&fork_block_2.block_hash) + .await + .unwrap(); assert_eq!(fork_2_is_canonical, Some(false)); // Verify that all blocks at the same height as fork blocks but in the canonical chain // are marked as canonical let canonical_block_at_fork_height = canonical_chain.nth_block(fork_block_1.block_height); - let canonical_at_fork_height_is_canonical = - is_block_canonical(&db, &canonical_block_at_fork_height.block_hash).await; + let canonical_at_fork_height_is_canonical = db + .is_block_canonical(&canonical_block_at_fork_height.block_hash) + .await + .unwrap(); assert_eq!(canonical_at_fork_height_is_canonical, Some(true)); testing::storage::drop_db(db).await; @@ -7875,18 +7876,18 @@ mod canonical_bitcoin_blockchain { // Verify that all blocks in the main chain are marked as canonical for block in &main_chain { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } // Check that fork chain blocks with height 0 to 2 are canonical, // since they are also part of the main chain for (_, block) in fork_chain.range(stable_block_heights.clone()) { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } for (_, block) in fork_chain.range(forked_block_heights.clone()) { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(false)); } @@ -7898,20 +7899,20 @@ mod canonical_bitcoin_blockchain { // Check that all fork chain blocks are canonical for block in &fork_chain { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } // Check that original main chain blocks with height 0 to 2 are canonical for (_, block) in main_chain.range(stable_block_heights.clone()) { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } // Check that original main chain blocks with height 3 to 9 are // non-canonical for (_, block) in main_chain.range(forked_block_heights.clone()) { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(false)); } @@ -7924,18 +7925,18 @@ mod canonical_bitcoin_blockchain { // Verify that all blocks in the main chain are marked as canonical // again. for block in &main_chain { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } // Check that fork chain blocks with height 0 to 2 are canonical, // since they are also part of the main chain for (_, block) in fork_chain.range(stable_block_heights.clone()) { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } for (_, block) in fork_chain.range(forked_block_heights.clone()) { - let is_canonical = is_block_canonical(&db, &block.block_hash).await; + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(false)); } From c94649a3aa5083feea1e095bea03c672f498c7ae Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 21:44:49 +0000 Subject: [PATCH 09/37] add a block observer test --- signer/tests/integration/block_observer.rs | 203 +++++++++++++++++++++ 1 file changed, 203 insertions(+) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index fad38693c..96eaa8a5a 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -1874,3 +1874,206 @@ fn make_coinbase_deposit_request( }; (deposit_tx, req, info) } + +/// This test checks that the block observer marks the canonical status of +/// bitcoin blocks in the database whenever a new block is observed. +#[tokio::test] +async fn block_observer_marks_bitcoin_blocks_as_canonical() { + let (rpc, faucet) = regtest::initialize_blockchain(); + let db = testing::storage::new_test_database().await; + + let emily_client = EmilyClient::try_new( + &Url::parse("http://testApiKey@localhost:3031").unwrap(), + Duration::from_secs(1), + None, + ) + .unwrap(); + + let ctx = TestContext::builder() + .with_storage(db.clone()) + .with_first_bitcoin_core_client() + .with_emily_client(emily_client.clone()) + .with_mocked_stacks_client() + .build(); + + // Set up the stacks client + ctx.with_stacks_client(|client| { + client + .expect_get_tenure_info() + .returning(move || Box::pin(std::future::ready(Ok(DUMMY_TENURE_INFO.clone())))); + client.expect_get_block().returning(|_| { + let response = Ok(NakamotoBlock { + header: NakamotoBlockHeader::empty(), + txs: Vec::new(), + }); + Box::pin(std::future::ready(response)) + }); + client + .expect_get_tenure_headers() + .returning(|_| Box::pin(std::future::ready(TenureBlockHeaders::nearly_empty()))); + client.expect_get_epoch_status().returning(|| { + Box::pin(std::future::ready(Ok(StacksEpochStatus::PostNakamoto { + nakamoto_start_height: BitcoinBlockHeight::from(232_u32), + }))) + }); + client + .expect_get_sortition_info() + .returning(move |_| Box::pin(std::future::ready(Ok(DUMMY_SORTITION_INFO.clone())))); + client.expect_get_contract_source().returning(|_, _| { + Box::pin(async { + Err(Error::StacksNodeResponse( + mock_reqwests_status_code_error(404).await, + )) + }) + }); + }) + .await; + + // Helper function to get all blocks in the chain from a given block hash + async fn get_chain_blocks( + db: &PgStore, + chain_tip: &BitcoinBlockHash, + ) -> Vec<(BitcoinBlockHash, Option)> { + sqlx::query_as::<_, (BitcoinBlockHash, Option)>( + r#" + WITH RECURSIVE chain_blocks AS ( + SELECT + block_hash, + is_canonical, + parent_hash + FROM sbtc_signer.bitcoin_blocks + WHERE block_hash = $1 + + UNION ALL + + SELECT + parent.block_hash, + parent.is_canonical, + parent.parent_hash + FROM sbtc_signer.bitcoin_blocks AS parent + JOIN chain_blocks AS child + ON parent.block_hash = child.parent_hash + ) + SELECT block_hash, is_canonical + FROM chain_blocks + "#, + ) + .bind(chain_tip) + .fetch_all(db.pool()) + .await + .unwrap() + } + + // Start the block observer + let start_flag = Arc::new(AtomicBool::new(false)); + let flag = start_flag.clone(); + + let bitcoin_block_source = BitcoinChainTipPoller::start_for_regtest().await; + let block_observer = BlockObserver { + context: ctx.clone(), + bitcoin_block_source, + }; + + // We need at least one receiver + let _signal = ctx.get_signal_receiver(); + + tokio::spawn(async move { + flag.store(true, Ordering::Relaxed); + block_observer.run().await + }); + + // Wait for the task to start + while !start_flag.load(Ordering::SeqCst) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + + // Generate a new block and wait for the block observer to process it + let chain_tip_before_invalidation: BitcoinBlockHash = faucet.generate_block().into(); + + ctx.wait_for_signal(Duration::from_secs(3), |signal| { + matches!( + signal, + SignerSignal::Event(SignerEvent::BitcoinBlockObserved(block_ref)) + if block_ref.block_hash == chain_tip_before_invalidation + ) + }) + .await + .unwrap(); + + // We need to generate a new bitcoin block to make sure that bitcoin + // core feels that the block that we want to invalidate has been + // accepted. Maybe bitcoin core needs some time to process everything + // when there is a fork so we wait for a bit. + faucet.generate_block(); + tokio::time::sleep(Duration::from_secs(1)).await; + + // Verify the block is in the database + let db_block = db + .get_bitcoin_block(&chain_tip_before_invalidation) + .await + .unwrap(); + assert!(db_block.is_some()); + + // Check that all blocks on the chain have is_canonical = true + let chain_blocks = get_chain_blocks(&db, &chain_tip_before_invalidation).await; + for (_, is_canonical) in chain_blocks { + assert_eq!(is_canonical, Some(true)); + } + let chain_tip_before_invalidation_status = db + .is_block_canonical(&chain_tip_before_invalidation) + .await + .unwrap(); + assert_eq!(chain_tip_before_invalidation_status, Some(true)); + + // Now invalidate the chain tip + rpc.invalidate_block(&*chain_tip_before_invalidation) + .unwrap(); + + let new_block_1: BitcoinBlockHash = faucet.generate_block().into(); + let new_block_2: BitcoinBlockHash = faucet.generate_block().into(); + + ctx.wait_for_signal(Duration::from_secs(8), |signal| { + matches!( + signal, + SignerSignal::Event(SignerEvent::BitcoinBlockObserved(block_ref)) + if block_ref.block_hash == new_block_2 + ) + }) + .await + .unwrap(); + + // Verify the new blocks are in the database + let db_new_block_1 = db.get_bitcoin_block(&new_block_1).await.unwrap(); + assert!(db_new_block_1.is_some()); + let db_new_block_2 = db.get_bitcoin_block(&new_block_2).await.unwrap(); + assert!(db_new_block_2.is_some()); + + // Check that the new blocks have is_canonical = true + assert_eq!( + db.is_block_canonical(&new_block_1).await.unwrap(), + Some(true) + ); + assert_eq!( + db.is_block_canonical(&new_block_2).await.unwrap(), + Some(true) + ); + + // Check that the old chain tip has is_canonical = false + let invalidated_chain_tip_status = db + .is_block_canonical(&chain_tip_before_invalidation) + .await + .unwrap(); + assert_eq!(invalidated_chain_tip_status, Some(false)); + + // Verify all blocks on the new chain are canonical + let new_chain_blocks = get_chain_blocks(&db, &new_block_2).await; + for (block_hash, is_canonical) in new_chain_blocks { + // The old chain tip should not be in the new chain + if block_hash == chain_tip_before_invalidation { + panic!("The old chain tip was invalidated, and should not be on the new chain"); + } + assert_eq!(is_canonical, Some(true)); + } + + testing::storage::drop_db(db).await; +} From 0d10f728b8be2e41b4b41f81e177ecd6ea0d5289 Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 21:47:22 +0000 Subject: [PATCH 10/37] cargo clippy fixes --- signer/src/storage/memory/write.rs | 2 +- signer/tests/integration/block_observer.rs | 2 +- signer/tests/integration/postgres.rs | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/signer/src/storage/memory/write.rs b/signer/src/storage/memory/write.rs index bebab55ad..ad7c81f46 100644 --- a/signer/src/storage/memory/write.rs +++ b/signer/src/storage/memory/write.rs @@ -394,7 +394,7 @@ impl DbWrite for SharedStore { store.version += 1; // Then, recursively mark all blocks reachable from the chain tip as canonical - if let Some(block) = store.bitcoin_blocks.get(&chain_tip).cloned() { + if let Some(block) = store.bitcoin_blocks.get(chain_tip).cloned() { store.canonical_bitcoin_blocks.insert(*chain_tip, block); } diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index 96eaa8a5a..0de354cec 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -2026,7 +2026,7 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { assert_eq!(chain_tip_before_invalidation_status, Some(true)); // Now invalidate the chain tip - rpc.invalidate_block(&*chain_tip_before_invalidation) + rpc.invalidate_block(&chain_tip_before_invalidation) .unwrap(); let new_block_1: BitcoinBlockHash = faucet.generate_block().into(); diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index a53ff41ee..50ee2d315 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -7882,7 +7882,7 @@ mod canonical_bitcoin_blockchain { // Check that fork chain blocks with height 0 to 2 are canonical, // since they are also part of the main chain - for (_, block) in fork_chain.range(stable_block_heights.clone()) { + for (_, block) in fork_chain.range(stable_block_heights) { let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } @@ -7904,7 +7904,7 @@ mod canonical_bitcoin_blockchain { } // Check that original main chain blocks with height 0 to 2 are canonical - for (_, block) in main_chain.range(stable_block_heights.clone()) { + for (_, block) in main_chain.range(stable_block_heights) { let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } @@ -7931,7 +7931,7 @@ mod canonical_bitcoin_blockchain { // Check that fork chain blocks with height 0 to 2 are canonical, // since they are also part of the main chain - for (_, block) in fork_chain.range(stable_block_heights.clone()) { + for (_, block) in fork_chain.range(stable_block_heights) { let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); assert_eq!(is_canonical, Some(true)); } @@ -7942,7 +7942,7 @@ mod canonical_bitcoin_blockchain { // Now to make sure that things aren't messed up because the ranges // are empty iterators. - assert_eq!(fork_chain.range(stable_block_heights.clone()).count(), 3); + assert_eq!(fork_chain.range(stable_block_heights).count(), 3); assert_eq!(fork_chain.range(forked_block_heights.clone()).count(), 17); assert_eq!(main_chain.range(stable_block_heights).count(), 3); From 1e261ef4feba5ce848db1765aae0d500e8046e52 Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 21:58:33 +0000 Subject: [PATCH 11/37] update the field comment --- signer/src/storage/memory/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signer/src/storage/memory/store.rs b/signer/src/storage/memory/store.rs index cc2296690..4a007807e 100644 --- a/signer/src/storage/memory/store.rs +++ b/signer/src/storage/memory/store.rs @@ -42,7 +42,7 @@ pub struct Store { /// Bitcoin blocks pub bitcoin_blocks: HashMap, - /// Bitcoin blocks + /// Bitcoin blocks that are on the canonical bitcoin blockchain. pub canonical_bitcoin_blocks: HashMap, /// Stacks blocks From 479d888fe99a8a42331280c8e5f4d807ed7b7df1 Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 22:10:31 +0000 Subject: [PATCH 12/37] fix load_latest_deposit_requests_persists_requests_from_past, hopefully --- signer/tests/integration/block_observer.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index 0de354cec..ea943ca9c 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -214,20 +214,12 @@ async fn load_latest_deposit_requests_persists_requests_from_past(blocks_ago: u6 // We need to wait for the bitcoin-core to send us all the // notifications so that we are up to date with the expected chain tip. // For that we just wait until we know that we're up-to-date - let mut current_chain_tip = db2.get_bitcoin_canonical_chain_tip().await.unwrap(); - - let waiting_fut = async { - let db2 = db2.clone(); - while current_chain_tip != Some(chain_tip) { - current_chain_tip = db2.get_bitcoin_canonical_chain_tip().await.unwrap(); - tokio::time::sleep(Duration::from_millis(250)).await; - } - }; - - tokio::time::timeout(Duration::from_secs(3), waiting_fut) - .await - .unwrap(); - + ctx.wait_for_signal(Duration::from_secs(10), |signal| { + matches!(signal, SignerSignal::Event(SignerEvent::BitcoinBlockObserved(block_ref)) + if block_ref.block_hash == chain_tip) + }) + .await + .unwrap(); // Okay now lets check if we have these deposit requests in our // database. It should also have bitcoin blockchain data From 78b0d28775db14b1d257bbc0160aa52a4e8b57bb Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 22:29:44 +0000 Subject: [PATCH 13/37] generate the blocks after invalidation differently --- signer/tests/integration/block_observer.rs | 23 ++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index ea943ca9c..135995474 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -72,6 +72,7 @@ use signer::transaction_coordinator::should_run_dkg; use signer::transaction_signer::assert_allow_dkg_begin; use url::Url; +use crate::bitcoin_forks::GenerateBlockJson; use crate::setup::IntoEmilyTestingConfig as _; use crate::setup::TestSweepSetup; use crate::setup::fetch_canonical_bitcoin_blockchain; @@ -1982,7 +1983,7 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { // Generate a new block and wait for the block observer to process it let chain_tip_before_invalidation: BitcoinBlockHash = faucet.generate_block().into(); - ctx.wait_for_signal(Duration::from_secs(3), |signal| { + ctx.wait_for_signal(Duration::from_secs(8), |signal| { matches!( signal, SignerSignal::Event(SignerEvent::BitcoinBlockObserved(block_ref)) @@ -1997,7 +1998,7 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { // accepted. Maybe bitcoin core needs some time to process everything // when there is a fork so we wait for a bit. faucet.generate_block(); - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_secs(2)).await; // Verify the block is in the database let db_block = db @@ -2021,8 +2022,22 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { rpc.invalidate_block(&chain_tip_before_invalidation) .unwrap(); - let new_block_1: BitcoinBlockHash = faucet.generate_block().into(); - let new_block_2: BitcoinBlockHash = faucet.generate_block().into(); + let [new_block_1, new_block_2] = (0..2) + .map(|_| { + rpc.call::( + "generateblock", + &[ + faucet.address.to_string().into(), + serde_json::to_value::<&[String; 0]>(&[]).unwrap(), + ], + ) + .unwrap() + .hash + .into() + }) + .collect::>() + .try_into() + .unwrap(); ctx.wait_for_signal(Duration::from_secs(8), |signal| { matches!( From 233b3fd571ba0fc9d31f6ffd1e21f69f1ad33f3b Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 22:36:37 +0000 Subject: [PATCH 14/37] this wasn't importable or something, quick fix --- signer/tests/integration/block_observer.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index 135995474..b9ce180c5 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -72,7 +72,6 @@ use signer::transaction_coordinator::should_run_dkg; use signer::transaction_signer::assert_allow_dkg_begin; use url::Url; -use crate::bitcoin_forks::GenerateBlockJson; use crate::setup::IntoEmilyTestingConfig as _; use crate::setup::TestSweepSetup; use crate::setup::fetch_canonical_bitcoin_blockchain; @@ -1868,6 +1867,11 @@ fn make_coinbase_deposit_request( (deposit_tx, req, info) } +#[derive(serde::Deserialize, Clone, PartialEq, Eq, Debug)] +pub struct GenerateBlockJson { + pub hash: bitcoin::BlockHash, +} + /// This test checks that the block observer marks the canonical status of /// bitcoin blocks in the database whenever a new block is observed. #[tokio::test] From 0c61eb945055d24dcf82221c73bc626ce9963f0f Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 22:41:17 +0000 Subject: [PATCH 15/37] Okay, this can cause problems if some test fails --- signer/src/block_observer.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/signer/src/block_observer.rs b/signer/src/block_observer.rs index 0468d7f03..c24d7b206 100644 --- a/signer/src/block_observer.rs +++ b/signer/src/block_observer.rs @@ -369,8 +369,6 @@ impl BlockObserver { let block_headers = self.next_headers_to_process(block_hash).await?; let chain_tip = block_headers.last().cloned(); - // This should be the header for the above block hash. - debug_assert_eq!(chain_tip.as_ref().map(|h| h.hash), Some(block_hash)); for block_header in block_headers { self.process_bitcoin_block(block_header).await?; From 1bdc690ecd82e78c25c38b48b72045d6ccd33976 Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 23:01:55 +0000 Subject: [PATCH 16/37] bitcoin-core sensitivity or something? --- signer/tests/integration/block_observer.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index b9ce180c5..85aed251f 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -1997,13 +1997,6 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { .await .unwrap(); - // We need to generate a new bitcoin block to make sure that bitcoin - // core feels that the block that we want to invalidate has been - // accepted. Maybe bitcoin core needs some time to process everything - // when there is a fork so we wait for a bit. - faucet.generate_block(); - tokio::time::sleep(Duration::from_secs(2)).await; - // Verify the block is in the database let db_block = db .get_bitcoin_block(&chain_tip_before_invalidation) From ca0b237680113cc56f9fcae8241ba73fe19fa324 Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 13 Jan 2026 23:13:38 +0000 Subject: [PATCH 17/37] ask for the chain tip after invalidation --- signer/tests/integration/block_observer.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index 85aed251f..c3a50e1cd 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -2019,6 +2019,8 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { rpc.invalidate_block(&chain_tip_before_invalidation) .unwrap(); + let _ = rpc.get_best_block_hash().unwrap(); + let [new_block_1, new_block_2] = (0..2) .map(|_| { rpc.call::( From 685ab43c829726584990f04c005727dce02e579e Mon Sep 17 00:00:00 2001 From: djordon Date: Wed, 14 Jan 2026 01:37:44 +0000 Subject: [PATCH 18/37] Let's try sleeping, twice and invalidating twice --- signer/tests/integration/block_observer.rs | 28 +++++++--------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index c3a50e1cd..5d89d0ee1 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -1867,11 +1867,6 @@ fn make_coinbase_deposit_request( (deposit_tx, req, info) } -#[derive(serde::Deserialize, Clone, PartialEq, Eq, Debug)] -pub struct GenerateBlockJson { - pub hash: bitcoin::BlockHash, -} - /// This test checks that the block observer marks the canonical status of /// bitcoin blocks in the database whenever a new block is observed. #[tokio::test] @@ -2018,22 +2013,15 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { // Now invalidate the chain tip rpc.invalidate_block(&chain_tip_before_invalidation) .unwrap(); + tokio::time::sleep(Duration::from_millis(500)).await; + rpc.invalidate_block(&chain_tip_before_invalidation) + .unwrap(); + tokio::time::sleep(Duration::from_millis(500)).await; - let _ = rpc.get_best_block_hash().unwrap(); - - let [new_block_1, new_block_2] = (0..2) - .map(|_| { - rpc.call::( - "generateblock", - &[ - faucet.address.to_string().into(), - serde_json::to_value::<&[String; 0]>(&[]).unwrap(), - ], - ) - .unwrap() - .hash - .into() - }) + let [new_block_1, new_block_2] = faucet + .generate_blocks(2) + .into_iter() + .map(From::from) .collect::>() .try_into() .unwrap(); From a3fbcf0ba2cd56c8eedfd9239e6d43a0ed2b3f25 Mon Sep 17 00:00:00 2001 From: djordon Date: Wed, 14 Jan 2026 01:47:20 +0000 Subject: [PATCH 19/37] 4 second sleep? --- signer/tests/integration/block_observer.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index 5d89d0ee1..b2067347a 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -2013,10 +2013,7 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { // Now invalidate the chain tip rpc.invalidate_block(&chain_tip_before_invalidation) .unwrap(); - tokio::time::sleep(Duration::from_millis(500)).await; - rpc.invalidate_block(&chain_tip_before_invalidation) - .unwrap(); - tokio::time::sleep(Duration::from_millis(500)).await; + tokio::time::sleep(Duration::from_secs(4)).await; let [new_block_1, new_block_2] = faucet .generate_blocks(2) From dad31d65d97556ab56a7fd171cf2c11bdc0d842c Mon Sep 17 00:00:00 2001 From: djordon Date: Wed, 14 Jan 2026 02:31:27 +0000 Subject: [PATCH 20/37] generate blocks to a new random address --- signer/tests/integration/block_observer.rs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index b2067347a..3a945cd48 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -1874,6 +1874,8 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { let (rpc, faucet) = regtest::initialize_blockchain(); let db = testing::storage::new_test_database().await; + let mut rng = get_rng(); + let emily_client = EmilyClient::try_new( &Url::parse("http://testApiKey@localhost:3031").unwrap(), Duration::from_secs(1), @@ -2013,15 +2015,14 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { // Now invalidate the chain tip rpc.invalidate_block(&chain_tip_before_invalidation) .unwrap(); - tokio::time::sleep(Duration::from_secs(4)).await; - - let [new_block_1, new_block_2] = faucet - .generate_blocks(2) - .into_iter() - .map(From::from) - .collect::>() - .try_into() - .unwrap(); + + // Sometimes bitcoin-core will generate an invalid block (maybe the + // same block?), after invalidating the chain tip. One wait to combat + // this is to generate new blocks to a new address, ensuring the new + // block is distinct from the previous ones. + let random_address = Recipient::new_with_rng(AddressType::P2wpkh, &mut rng).address; + let new_blocks = rpc.generate_to_address(2, &random_address).unwrap(); + let [new_block_1, new_block_2] = <[_; 2]>::try_from(new_blocks).unwrap().map(From::from); ctx.wait_for_signal(Duration::from_secs(8), |signal| { matches!( From 2d6a9137639c59777b1097ac501dd7288135e2cf Mon Sep 17 00:00:00 2001 From: djordon Date: Wed, 14 Jan 2026 20:17:32 +0000 Subject: [PATCH 21/37] Update the comments --- signer/src/storage/mod.rs | 15 ++- signer/src/storage/postgres/write.rs | 11 +++ signer/tests/integration/postgres.rs | 134 ++++++++++++++++++++++++++- 3 files changed, 154 insertions(+), 6 deletions(-) diff --git a/signer/src/storage/mod.rs b/signer/src/storage/mod.rs index ac301fba8..39374b32c 100644 --- a/signer/src/storage/mod.rs +++ b/signer/src/storage/mod.rs @@ -621,12 +621,17 @@ pub trait DbWrite { address: Multiaddr, ) -> impl Future> + Send; - /// Update the is_canonical status for all bitcoin blocks. + /// Ensure that each and only blocks along the chain identified by the + /// given chain tip have their is_canonical set to TRUE. /// - /// Marks blocks reachable from the given chain tip as canonical - /// (is_canonical = TRUE) and marks all other blocks as non-canonical - /// (is_canonical = FALSE). This includes blocks that may have a height - /// that is greater than the height of the given chain tip. + /// # Notes + /// + /// This function should mark all blocks that are reachable from the + /// given chain tip as canonical (is_canonical = TRUE), and no other + /// block should have is_canonical set to TRUE. That means that the + /// DbWrite::set_canonical_bitcoin_blockchain() function need not set + /// the is_canonical column to FALSE if it is a non-canonical block -- + /// it can just leave the value of that column as NULL. fn set_canonical_bitcoin_blockchain( &self, chain_tip: &model::BitcoinBlockHash, diff --git a/signer/src/storage/postgres/write.rs b/signer/src/storage/postgres/write.rs index 5276750bb..0b60cb428 100644 --- a/signer/src/storage/postgres/write.rs +++ b/signer/src/storage/postgres/write.rs @@ -1053,6 +1053,17 @@ impl PgWrite { .map(|maybe_height| maybe_height.unwrap_or(0)) } + /// Update the is_canonical status for all blocks with height greater + /// than the current "canonical root height" (the block height + /// identified by the `find_bitcoin_canonical_root` function). + /// + /// # Notes + /// + /// If a forked block is added to the database and the next chain tip + /// builds off of a block where is_canonical is TRUE, and that block + /// has height greater than this forked block, then this function will + /// not set the is_canonical column for the forked block. This is okay, + /// since we just need to make sure that it is not set to TRUE. async fn set_canonical_bitcoin_blockchain<'e, E>( executor: &'e mut E, chain_tip: &model::BitcoinBlockHash, diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index 50ee2d315..4f785a170 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -7746,6 +7746,13 @@ mod canonical_bitcoin_blockchain { .unwrap(); } + /// In this test, we check that the + /// DbWrite::set_canonical_bitcoin_blockchain() function sets + /// is_canonical = TRUE in the bitcoin_blocks table for all blocks in + /// the chain along the chain tip. We also check that if forked blocks + /// are added to the database along with canonical blocks, that the + /// forked blocks are marked as non-canonical, assuming that neither + /// the canonical or non-canonical blocks have been marked at all. #[tokio::test] async fn test_set_canonical_bitcoin_blockchain() { let db = testing::storage::new_test_database().await; @@ -7846,8 +7853,16 @@ mod canonical_bitcoin_blockchain { testing::storage::drop_db(db).await; } + /// Similar to the above test, we have two chains each with differing + /// lengths, both in the database. The "main" when + /// DbWrite::set_canonical_bitcoin_blockchain() is called is the longer + /// chain with the main chain tip as input, all of those blocks are + /// marked as canonical and all other blocks are marked as + /// non-canonical. If we chose the fork chain tip, the blocks that are + /// part of the fork are then marked as canonical while the main + /// chain's blocks are then marked as non-canonical. #[tokio::test] - async fn test_set_canonical_bitcoin_blockchain_with_fork() { + async fn test_set_canonical_bitcoin_blockchain_with_fork_known_immediately() { let db = testing::storage::new_test_database().await; // Create two blockchains, one of length 10 and another of length @@ -7950,4 +7965,121 @@ mod canonical_bitcoin_blockchain { testing::storage::drop_db(db).await; } + + /// In this test, we check that it is not important whether the forked + /// blocks are already stored in the database or not when + /// DbWrite::set_canonical_bitcoin_blockchain() is called for the first + /// time, it still correctly mark the blocks as canonical or + /// non-canonical if they are part of the chain identified by the given + /// chain tip. + #[tokio::test] + async fn test_set_canonical_bitcoin_blockchain_with_fork_known_later() { + let db = testing::storage::new_test_database().await; + + // Create two blockchains, one of length 10 and another of length + // 20, where one forks the other at height 3 (the block with height + // 3 differs from one to the other but their earlier blocks are the + // same) + let main_chain = BitcoinChain::new_with_length(10); + let fork_chain = main_chain.fork_at_height(2u64, 15); + // These are the ranges for the block heights that didn't change, + // so blocks from both chains should always be canonical. + let stable_block_heights = ..BitcoinBlockHeight::from(2u64); + // These are the ranges for the block heights that changed, so + // blocks from one chain will always be non-canonical. + let forked_block_heights = BitcoinBlockHeight::from(2u64)..; + + // Write main chains to the database + for block in &main_chain { + db.write_bitcoin_block(block).await.unwrap(); + } + + // Set the main chain as canonical + let main_chain_tip = main_chain.chain_tip().block_hash; + db.set_canonical_bitcoin_blockchain(&main_chain_tip) + .await + .unwrap(); + + // Verify that all blocks in the main chain are marked as canonical + for block in &main_chain { + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + assert_eq!(is_canonical, Some(true)); + } + + // Write fork chains to the database + for block in &fork_chain { + db.write_bitcoin_block(block).await.unwrap(); + } + + // These were already written to the database, they are part of the main chain. + for (_, block) in fork_chain.range(stable_block_heights) { + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + assert_eq!(is_canonical, Some(true)); + } + + // These are new blocks, not yet seen before. + for (_, block) in fork_chain.range(forked_block_heights.clone()) { + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + assert_eq!(is_canonical, None); + } + + // Set the fork chain as canonical from its chain tip + let fork_chain_tip = fork_chain.chain_tip().block_hash; + db.set_canonical_bitcoin_blockchain(&fork_chain_tip) + .await + .unwrap(); + + // Check that all fork chain blocks are canonical + for block in &fork_chain { + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + assert_eq!(is_canonical, Some(true)); + } + + // Check that original main chain blocks with height 0 to 1 are canonical + for (_, block) in main_chain.range(stable_block_heights) { + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + assert_eq!(is_canonical, Some(true)); + } + + // Check that original main chain blocks with height 1 to 9 are + // non-canonical + for (_, block) in main_chain.range(forked_block_heights.clone()) { + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + assert_eq!(is_canonical, Some(false)); + } + + // Okay, now let's make the main chain canonical again, just to + // make sure that things switch back as expected. + db.set_canonical_bitcoin_blockchain(&main_chain_tip) + .await + .unwrap(); + + // Verify that all blocks in the main chain are marked as canonical + // again. + for block in &main_chain { + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + assert_eq!(is_canonical, Some(true)); + } + + // Check that fork chain blocks with height 0 to 1 are canonical, + // since they are also part of the main chain + for (_, block) in fork_chain.range(stable_block_heights) { + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + assert_eq!(is_canonical, Some(true)); + } + for (_, block) in fork_chain.range(forked_block_heights.clone()) { + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + assert_eq!(is_canonical, Some(false)); + } + + // Now to make sure that things aren't messed up because the ranges + // are empty iterators. + assert_eq!(fork_chain.range(stable_block_heights).count(), 2); + assert_eq!(fork_chain.range(forked_block_heights.clone()).count(), 15); + + assert_eq!(main_chain.range(stable_block_heights).count(), 2); + assert_eq!(main_chain.range(forked_block_heights).count(), 8); + + testing::storage::drop_db(db).await; + } } From 8ea706d581e668f4b384aac346b5c5377730da2e Mon Sep 17 00:00:00 2001 From: djordon Date: Wed, 14 Jan 2026 20:28:08 +0000 Subject: [PATCH 22/37] Simplify the two queries into one --- signer/src/storage/mod.rs | 2 +- signer/src/storage/postgres/write.rs | 82 ++++++---------------------- 2 files changed, 17 insertions(+), 67 deletions(-) diff --git a/signer/src/storage/mod.rs b/signer/src/storage/mod.rs index 39374b32c..70f6ce08c 100644 --- a/signer/src/storage/mod.rs +++ b/signer/src/storage/mod.rs @@ -625,7 +625,7 @@ pub trait DbWrite { /// given chain tip have their is_canonical set to TRUE. /// /// # Notes - /// + /// /// This function should mark all blocks that are reachable from the /// given chain tip as canonical (is_canonical = TRUE), and no other /// block should have is_canonical set to TRUE. That means that the diff --git a/signer/src/storage/postgres/write.rs b/signer/src/storage/postgres/write.rs index 0b60cb428..843316e4c 100644 --- a/signer/src/storage/postgres/write.rs +++ b/signer/src/storage/postgres/write.rs @@ -1003,62 +1003,12 @@ impl PgWrite { Ok(()) } - /// This function finds the first block on the chain that is reachable - /// from the given chain tip that is already marked as a canonical - /// block in the database. - async fn find_bitcoin_canonical_root<'e, E>( - executor: &'e mut E, - chain_tip: &model::BitcoinBlockHash, - ) -> Result - where - E: 'static, - for<'c> &'c mut E: sqlx::PgExecutor<'c>, - { - // Walk backwards from chain tip to find the first block where - // is_canonical = TRUE - sqlx::query_scalar::<_, i64>( - r#" - -- find_canonical_root - WITH RECURSIVE chain_walk AS ( - -- Start from the chain tip - SELECT - block_hash - , block_height - , parent_hash - , is_canonical - FROM sbtc_signer.bitcoin_blocks - WHERE block_hash = $1 - - UNION ALL - - -- Recursively get parent blocks, stopping when we find a canonical block - SELECT - parent.block_hash - , parent.block_height - , parent.parent_hash - , parent.is_canonical - FROM sbtc_signer.bitcoin_blocks AS parent - JOIN chain_walk AS child - ON parent.block_hash = child.parent_hash - WHERE child.is_canonical IS NOT TRUE - ) - SELECT MIN(block_height) - FROM chain_walk - "#, - ) - .bind(chain_tip) - .fetch_optional(executor) - .await - .map_err(Error::SqlxQuery) - .map(|maybe_height| maybe_height.unwrap_or(0)) - } - /// Update the is_canonical status for all blocks with height greater - /// than the current "canonical root height" (the block height - /// identified by the `find_bitcoin_canonical_root` function). - /// + /// than the current "canonical root height" (the first block on the chain + /// reachable from the chain tip that is already marked as canonical). + /// /// # Notes - /// + /// /// If a forked block is added to the database and the next chain tip /// builds off of a block where is_canonical is TRUE, and that block /// has height greater than this forked block, then this function will @@ -1072,14 +1022,9 @@ impl PgWrite { E: 'static, for<'c> &'c mut E: sqlx::PgExecutor<'c>, { - // First, walk backwards from chain tip to find the first - // block where is_canonical = TRUE. This block height is the height - // where we'll need to update the canonical status of bitcoin - // blocks. - let min_height = Self::find_bitcoin_canonical_root(executor, chain_tip).await?; - - // Next, update blocks with height greater than min_height - // Build canonical chain and mark those blocks as canonical. + // Walk backwards from chain tip to find the canonical root and + // build the canonical chain in a single recursive query, then + // update all blocks with height >= the minimum height found. sqlx::query( r#" -- set_canonical_bitcoin_blockchain @@ -1089,30 +1034,35 @@ impl PgWrite { block_hash , block_height , parent_hash + , is_canonical FROM sbtc_signer.bitcoin_blocks WHERE block_hash = $1 UNION ALL - -- Recursively get parent blocks + -- Recursively get parent blocks, stopping when we find a canonical block SELECT parent.block_hash , parent.block_height , parent.parent_hash + , parent.is_canonical FROM sbtc_signer.bitcoin_blocks AS parent JOIN canonical_chain AS child ON parent.block_hash = child.parent_hash - WHERE parent.block_height >= $2 + WHERE child.is_canonical IS NOT TRUE + ), + min_height AS ( + SELECT COALESCE(MIN(block_height), 0) AS height + FROM canonical_chain ) UPDATE sbtc_signer.bitcoin_blocks SET is_canonical = (block_hash IN ( SELECT block_hash FROM canonical_chain )) - WHERE block_height >= $2 + WHERE block_height >= (SELECT height FROM min_height) "#, ) .bind(chain_tip) - .bind(min_height) .execute(executor) .await .map_err(Error::SqlxQuery)?; From 2b5494a979f7d0954072b02d9ebc24690a7711e1 Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 15 Jan 2026 16:44:00 +0000 Subject: [PATCH 23/37] only call set_canonical_bitcoin_blockchain in a new function designed specifically for the bitcoin chain tip --- signer/src/block_observer.rs | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/signer/src/block_observer.rs b/signer/src/block_observer.rs index c24d7b206..6b36bb92e 100644 --- a/signer/src/block_observer.rs +++ b/signer/src/block_observer.rs @@ -170,7 +170,7 @@ where ) .increment(1); - if let Err(error) = self.process_bitcoin_blocks_until(block_hash).await { + if let Err(error) = self.process_bitcoin_chain_tip(block_hash).await { tracing::warn!(%error, %block_hash, "could not process bitcoin blocks"); } @@ -348,6 +348,28 @@ impl BlockObserver { Ok(headers.into()) } + /// Process the bitcoin chain tip by fetching all unknown block headers + /// from the given chain tip back until the nakamoto start height. Then + /// mark all blocks reachable from the chain tip as canonical in the + /// database. + /// + /// # Notes + /// + /// This function must only be called with the bitcoin chain tip since + /// it updates all blocks that are reachable from the given block hash + /// as canonical and may update blocks not reachable as non-canonical. + #[tracing::instrument(skip_all, fields(%chain_tip))] + async fn process_bitcoin_chain_tip(&self, chain_tip: BlockHash) -> Result<(), Error> { + self.process_bitcoin_blocks_until(chain_tip).await?; + + let db = self.context.get_storage_mut(); + + tracing::info!("updating canonical bitcoin blockchain to chain tip"); + let chain_tip: model::BitcoinBlockHash = chain_tip.into(); + db.set_canonical_bitcoin_blockchain(&chain_tip).await?; + Ok(()) + } + /// Process bitcoin blocks until we get caught up to the given /// `block_hash`. /// @@ -364,23 +386,13 @@ impl BlockObserver { /// This means that if we stop processing blocks midway though, /// subsequent calls to this function will properly pick up from where /// we left off and update the database. - #[tracing::instrument(skip_all, fields(%block_hash))] async fn process_bitcoin_blocks_until(&self, block_hash: BlockHash) -> Result<(), Error> { let block_headers = self.next_headers_to_process(block_hash).await?; - let chain_tip = block_headers.last().cloned(); - for block_header in block_headers { self.process_bitcoin_block(block_header).await?; } - if let Some(chain_tip) = chain_tip.map(|h| h.hash.into()) { - tracing::info!("updating block canonical status"); - - let db = self.context.get_storage_mut(); - db.set_canonical_bitcoin_blockchain(&chain_tip).await?; - } - Ok(()) } From 4a7522e8f9f7ad8cbcf273bab5a9825ac84fd22a Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 27 Jan 2026 21:23:25 +0000 Subject: [PATCH 24/37] apparently, this was lost in the merge --- signer/tests/integration/block_observer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index 2cb60909d..c00913616 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -54,6 +54,7 @@ use signer::storage::model::TxPrevout; use signer::storage::model::TxPrevoutType; use signer::storage::postgres::PgStore; use signer::testing::btc::get_canonical_chain_tip; +use signer::testing::stacks::DUMMY_SORTITION_INFO; use signer::testing::stacks::DUMMY_TENURE_INFO; use testing_emily_client::apis::testing_api; From ea231ed61dc545209bfc9257873a5f1cdaf31244 Mon Sep 17 00:00:00 2001 From: djordon Date: Tue, 27 Jan 2026 21:46:28 +0000 Subject: [PATCH 25/37] We need to make sure stacks blocks are anchored to bitcoin blocks on the canonical chain --- signer/tests/integration/block_observer.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index c00913616..77681765d 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -1899,6 +1899,7 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { .build(); // Set up the stacks client + let anchor = get_canonical_chain_tip(rpc); ctx.with_stacks_client(|client| { client .expect_get_tenure_info() @@ -1910,9 +1911,12 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { }); Box::pin(std::future::ready(response)) }); + + let tenure_headers = TenureBlockHeaders::from_anchor(&anchor); + client .expect_get_tenure_headers() - .returning(|_| Box::pin(std::future::ready(TenureBlockHeaders::nearly_empty()))); + .returning(move |_| Box::pin(std::future::ready(Ok(tenure_headers.clone())))); client.expect_get_epoch_status().returning(|| { Box::pin(std::future::ready(Ok(StacksEpochStatus::PostNakamoto { nakamoto_start_height: BitcoinBlockHeight::from(232_u32), From 2662d9df2985a6fc529af284ac6fc752857d3553 Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 15:25:58 +0000 Subject: [PATCH 26/37] slight simplification --- signer/src/block_observer.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/signer/src/block_observer.rs b/signer/src/block_observer.rs index de31f8a19..537c18827 100644 --- a/signer/src/block_observer.rs +++ b/signer/src/block_observer.rs @@ -366,8 +366,7 @@ impl BlockObserver { tracing::info!("updating canonical bitcoin blockchain to chain tip"); let chain_tip: model::BitcoinBlockHash = chain_tip.into(); - db.set_canonical_bitcoin_blockchain(&chain_tip).await?; - Ok(()) + db.set_canonical_bitcoin_blockchain(&chain_tip).await } /// Process bitcoin blocks until we get caught up to the given From 726d0eaf9c710d0812e995201eb87ff818527ef7 Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 15:26:32 +0000 Subject: [PATCH 27/37] forgot to clear the in-memory canonical blocks --- signer/src/storage/memory/write.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/signer/src/storage/memory/write.rs b/signer/src/storage/memory/write.rs index ad7c81f46..71aa80368 100644 --- a/signer/src/storage/memory/write.rs +++ b/signer/src/storage/memory/write.rs @@ -393,6 +393,7 @@ impl DbWrite for SharedStore { let mut store = self.lock().await; store.version += 1; + store.canonical_bitcoin_blocks.clear(); // Then, recursively mark all blocks reachable from the chain tip as canonical if let Some(block) = store.bitcoin_blocks.get(chain_tip).cloned() { store.canonical_bitcoin_blocks.insert(*chain_tip, block); From 7185893b1c911c6cbca2a5e68ef68afcbac70a84 Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 15:29:19 +0000 Subject: [PATCH 28/37] cleaner block generation in fork_at_height --- signer/src/testing/blocks.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/signer/src/testing/blocks.rs b/signer/src/testing/blocks.rs index 2281e7e16..0f59899bd 100644 --- a/signer/src/testing/blocks.rs +++ b/signer/src/testing/blocks.rs @@ -113,10 +113,7 @@ impl BitcoinChain { let mut fork_chain = Self(fork); - for _ in 0..num_blocks { - let new_block = fork_chain.chain_tip().new_child(); - fork_chain.0.insert(new_block.block_height, new_block); - } + fork_chain.generate_blocks(num_blocks); fork_chain } From 69b569809f34b9dbb0963951554687e22f68f222 Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 15:40:13 +0000 Subject: [PATCH 29/37] Just add another transaction instead of generating a block to a new address --- signer/tests/integration/block_observer.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index 77681765d..128ded32f 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -1882,8 +1882,6 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { let (rpc, faucet) = regtest::initialize_blockchain(); let db = testing::storage::new_test_database().await; - let mut rng = get_rng(); - let emily_client = EmilyClient::try_new( &Url::parse("http://testApiKey@localhost:3031").unwrap(), Duration::from_secs(1), @@ -2029,11 +2027,11 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { .unwrap(); // Sometimes bitcoin-core will generate an invalid block (maybe the - // same block?), after invalidating the chain tip. One wait to combat - // this is to generate new blocks to a new address, ensuring the new - // block is distinct from the previous ones. - let random_address = Recipient::new_with_rng(AddressType::P2wpkh, &mut rng).address; - let new_blocks = rpc.generate_to_address(2, &random_address).unwrap(); + // same block?), after invalidating the chain tip. One way to combat + // this is to create a new transaction to ensure that the new block is + // distinct from the previous ones. + faucet.send_to(1001, &faucet.address); + let new_blocks = faucet.generate_blocks(2); let [new_block_1, new_block_2] = <[_; 2]>::try_from(new_blocks).unwrap().map(From::from); ctx.wait_for_signal(Duration::from_secs(8), |signal| { From 43feedd9d5a14b82fe1fc250643a8a46a0f4e590 Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 15:42:10 +0000 Subject: [PATCH 30/37] update a comment --- signer/tests/integration/postgres.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index 55678656f..aa8582517 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -7944,7 +7944,7 @@ mod canonical_bitcoin_blockchain { let fork_block_1 = fork_chain.nth_block(2u64.into()); let fork_block_2 = fork_chain.nth_block(3u64.into()); - // Verify that fork blocks are marked as non-canonical + // Verify that fork blocks are not marked as canonical let fork_1_is_canonical = db .is_block_canonical(&fork_block_1.block_hash) .await From 5f1803571ad19c7a7526d375477c15d9a729bb36 Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 15:55:00 +0000 Subject: [PATCH 31/37] add an additional check for canonicalness in one of the tests --- signer/tests/integration/postgres.rs | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index aa8582517..5eb7d6d44 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -7957,10 +7957,27 @@ mod canonical_bitcoin_blockchain { .unwrap(); assert_eq!(fork_2_is_canonical, None); - // These new blocks aren't part of the canonical chain, so they - // should be marked as non-canonical. In order to make sure that - // they get marked as such, we need to clear the is_canonical - // column from some of the blocks in the table. + // Let's set the canonical chain again and verity that the new + // blocks are still not marked as canonical. Note that the + // guarantees of the set_canonical_bitcoin_blockchaion function is + // that non-canonical blocks are not marked as canonical, not that + // they are definitively marked as non-canonical. + db.set_canonical_bitcoin_blockchain(&chain_tip).await.unwrap(); + let fork_1_is_canonical = db + .is_block_canonical(&fork_block_1.block_hash) + .await + .unwrap(); + assert_ne!(fork_1_is_canonical, Some(true)); + + let fork_2_is_canonical = db + .is_block_canonical(&fork_block_2.block_hash) + .await + .unwrap(); + assert_ne!(fork_2_is_canonical, Some(true)); + + // In order to make sure that non-canonical blocks get marked as + // such, we need to clear the is_canonical column from some of the + // blocks in the table. clear_is_canonical_bitcoin_blocks(&db).await; db.set_canonical_bitcoin_blockchain(&chain_tip) .await From d2fff18b422a9c53e51fc1895c36901b51d96e1d Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 15:55:44 +0000 Subject: [PATCH 32/37] comment update --- signer/tests/integration/postgres.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index 5eb7d6d44..b8f6cdbb9 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -7944,7 +7944,7 @@ mod canonical_bitcoin_blockchain { let fork_block_1 = fork_chain.nth_block(2u64.into()); let fork_block_2 = fork_chain.nth_block(3u64.into()); - // Verify that fork blocks are not marked as canonical + // Verify that forked blocks are not marked as canonical let fork_1_is_canonical = db .is_block_canonical(&fork_block_1.block_hash) .await From a98677393b12a1ce6a08e2b5c27bba8b77cbf5bc Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 20:00:08 +0000 Subject: [PATCH 33/37] parameterize the test --- signer/tests/integration/block_observer.rs | 35 +++++++++++----------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index 128ded32f..c644899f1 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -1877,8 +1877,11 @@ fn make_coinbase_deposit_request( /// This test checks that the block observer marks the canonical status of /// bitcoin blocks in the database whenever a new block is observed. +#[test_case::test_case(3; "fork generating three blocks")] +#[test_case::test_case(2; "fork generating two blocks")] +#[test_case::test_case(1; "fork generating one block")] #[tokio::test] -async fn block_observer_marks_bitcoin_blocks_as_canonical() { +async fn block_observer_marks_bitcoin_blocks_as_canonical(fork_generating_blocks: u64) { let (rpc, faucet) = regtest::initialize_blockchain(); let db = testing::storage::new_test_database().await; @@ -2031,35 +2034,31 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { // this is to create a new transaction to ensure that the new block is // distinct from the previous ones. faucet.send_to(1001, &faucet.address); - let new_blocks = faucet.generate_blocks(2); - let [new_block_1, new_block_2] = <[_; 2]>::try_from(new_blocks).unwrap().map(From::from); + let new_blocks = faucet + .generate_blocks(fork_generating_blocks) + .into_iter() + .map(BitcoinBlockHash::from) + .collect::>(); + let last_new_block = new_blocks.last().cloned().unwrap(); ctx.wait_for_signal(Duration::from_secs(8), |signal| { matches!( signal, SignerSignal::Event(SignerEvent::BitcoinBlockObserved(block_ref)) - if block_ref.block_hash == new_block_2 + if block_ref.block_hash == last_new_block ) }) .await .unwrap(); // Verify the new blocks are in the database - let db_new_block_1 = db.get_bitcoin_block(&new_block_1).await.unwrap(); - assert!(db_new_block_1.is_some()); - let db_new_block_2 = db.get_bitcoin_block(&new_block_2).await.unwrap(); - assert!(db_new_block_2.is_some()); + for new_block in new_blocks { + let db_new_block = db.get_bitcoin_block(&new_block).await.unwrap(); + assert!(db_new_block.is_some()); + assert_eq!(db.is_block_canonical(&new_block).await.unwrap(), Some(true)); + } // Check that the new blocks have is_canonical = true - assert_eq!( - db.is_block_canonical(&new_block_1).await.unwrap(), - Some(true) - ); - assert_eq!( - db.is_block_canonical(&new_block_2).await.unwrap(), - Some(true) - ); - // Check that the old chain tip has is_canonical = false let invalidated_chain_tip_status = db .is_block_canonical(&chain_tip_before_invalidation) @@ -2068,7 +2067,7 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical() { assert_eq!(invalidated_chain_tip_status, Some(false)); // Verify all blocks on the new chain are canonical - let new_chain_blocks = get_chain_blocks(&db, &new_block_2).await; + let new_chain_blocks = get_chain_blocks(&db, &last_new_block).await; for (block_hash, is_canonical) in new_chain_blocks { // The old chain tip should not be in the new chain if block_hash == chain_tip_before_invalidation { From 7d313ff008a2d6bf08c0b6e66857b51efafd49c4 Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 20:00:28 +0000 Subject: [PATCH 34/37] cargo fmt --- signer/tests/integration/postgres.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index b8f6cdbb9..fbc9c2550 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -7962,7 +7962,9 @@ mod canonical_bitcoin_blockchain { // guarantees of the set_canonical_bitcoin_blockchaion function is // that non-canonical blocks are not marked as canonical, not that // they are definitively marked as non-canonical. - db.set_canonical_bitcoin_blockchain(&chain_tip).await.unwrap(); + db.set_canonical_bitcoin_blockchain(&chain_tip) + .await + .unwrap(); let fork_1_is_canonical = db .is_block_canonical(&fork_block_1.block_hash) .await From a2d57b97fdb62d0fa12f84edbe09f5645b23b32b Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 21:21:26 +0000 Subject: [PATCH 35/37] Use the new testcontainer infra --- signer/tests/integration/block_observer.rs | 29 ++++++++-------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/signer/tests/integration/block_observer.rs b/signer/tests/integration/block_observer.rs index 3639f7dd3..354f875f3 100644 --- a/signer/tests/integration/block_observer.rs +++ b/signer/tests/integration/block_observer.rs @@ -1897,20 +1897,18 @@ fn make_coinbase_deposit_request( #[test_case::test_case(1; "fork generating one block")] #[tokio::test] async fn block_observer_marks_bitcoin_blocks_as_canonical(fork_generating_blocks: u64) { - let (rpc, faucet) = regtest::initialize_blockchain(); - let db = testing::storage::new_test_database().await; + let stack = TestContainersBuilder::start_bitcoin().await; + let bitcoin = stack.bitcoin().await; + let rpc = bitcoin.rpc(); + let faucet = &bitcoin.get_faucet(); + let (emily_client, emily_tables) = new_emily_setup().await; - let emily_client = EmilyClient::try_new( - &Url::parse("http://testApiKey@localhost:3031").unwrap(), - Duration::from_secs(1), - None, - ) - .unwrap(); + let db = testing::storage::new_test_database().await; let ctx = TestContext::builder() .with_storage(db.clone()) - .with_first_bitcoin_core_client() - .with_emily_client(emily_client.clone()) + .with_bitcoin_client(bitcoin.get_client()) + .with_emily_client(emily_client) .with_mocked_stacks_client() .build(); @@ -1920,13 +1918,6 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical(fork_generating_blocks client .expect_get_tenure_info() .returning(move || Box::pin(std::future::ready(Ok(DUMMY_TENURE_INFO.clone())))); - client.expect_get_block().returning(|_| { - let response = Ok(NakamotoBlock { - header: NakamotoBlockHeader::empty(), - txs: Vec::new(), - }); - Box::pin(std::future::ready(response)) - }); let tenure_headers = TenureBlockHeaders::from_anchor(&anchor); @@ -1990,10 +1981,9 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical(fork_generating_blocks let start_flag = Arc::new(AtomicBool::new(false)); let flag = start_flag.clone(); - let bitcoin_block_source = BitcoinChainTipPoller::start_for_regtest().await; let block_observer = BlockObserver { context: ctx.clone(), - bitcoin_block_source, + bitcoin_block_source: bitcoin.start_chain_tip_poller().await, }; // We need at least one receiver @@ -2092,4 +2082,5 @@ async fn block_observer_marks_bitcoin_blocks_as_canonical(fork_generating_blocks } testing::storage::drop_db(db).await; + clean_emily_setup(emily_tables).await; } From c8d789644811d30941aceb1349e78787eacaef8e Mon Sep 17 00:00:00 2001 From: djordon Date: Thu, 29 Jan 2026 21:43:17 +0000 Subject: [PATCH 36/37] add another test using testdata --- signer/tests/integration/postgres.rs | 90 ++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index 2c45dec31..fc3226c3d 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -8233,4 +8233,94 @@ mod canonical_bitcoin_blockchain { testing::storage::drop_db(db).await; } + + /// Like the test_set_canonical_bitcoin_blockchain test above, this + /// test checks that the DbWrite::set_canonical_bitcoin_blockchain() + /// function sets is_canonical = TRUE in the bitcoin_blocks table for + /// all blocks in the chain along the chain tip. It also check that if + /// forked blocks are added to the database along with canonical + /// blocks, that the forked blocks are marked as non-canonical. + /// + /// The difference with this test is that it uses the TestData struct + /// to generate many random blockchains and insert them all into the + /// database. + #[tokio::test] + async fn test_set_canonical_bitcoin_blockchain_v2() { + let db = testing::storage::new_test_database().await; + + let mut rng = get_rng(); + let signer_keys = testing::wsts::generate_signer_set_public_keys(&mut rng, 7); + + // Create many random blockchains and insert them all into the database + let params = testing::storage::model::Params { + num_bitcoin_blocks: 100, + num_stacks_blocks_per_bitcoin_block: 0, + num_deposit_requests_per_block: 0, + num_withdraw_requests_per_block: 0, + num_signers_per_request: 0, + consecutive_blocks: false, + }; + let blockchains = TestData::generate(&mut rng, &signer_keys, ¶ms); + + // Insert all blocks from into the database + for block in blockchains.bitcoin_blocks.iter() { + db.write_bitcoin_block(block).await.unwrap(); + } + + // Get the chain tip of the canonical chain + let chain_tip = db + .get_bitcoin_canonical_chain_tip_ref() + .await + .unwrap() + .unwrap(); + + // Initially, all blocks should have is_canonical = NULL + // Verify that no blocks have is_canonical IS NOT NULL + let has_non_null_canonical: bool = sqlx::query_scalar( + "SELECT EXISTS( + SELECT 1 + FROM sbtc_signer.bitcoin_blocks + WHERE is_canonical IS NOT NULL + )", + ) + .fetch_one(db.pool()) + .await + .unwrap(); + + assert!(!has_non_null_canonical); + + // Call set_canonical_bitcoin_blockchain with the chain tip + db.set_canonical_bitcoin_blockchain(&chain_tip.block_hash) + .await + .unwrap(); + + // To make sure that not everything is marked as non-canonical, we + // have a flag that at least one block is canonical. + let mut some_canonical_block_exists = false; + + // Verify that only blocks in the canonical chain are marked as canonical + for block in blockchains.bitcoin_blocks.iter() { + // This function checks whether the given block is on the + // canonical chain by recursing backwards from the chain tip to + // the given block. + let on_canonical_chain = db + .in_canonical_bitcoin_blockchain(&chain_tip, &block.into()) + .await + .unwrap(); + // This function checks the is_canonical column for the given + // block hash. + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + + // Every block should have their is_canonical column set to + // some value, and it should match what the + // in_canonical_bitcoin_blockchain function returns. + assert_eq!(Some(on_canonical_chain), is_canonical); + + if on_canonical_chain { + some_canonical_block_exists = true; + } + } + assert!(some_canonical_block_exists); + testing::storage::drop_db(db).await; + } } From 69bcd3c0505144f726b9bc4b51202071ff903de2 Mon Sep 17 00:00:00 2001 From: djordon Date: Fri, 30 Jan 2026 15:59:03 +0000 Subject: [PATCH 37/37] re-order the tests --- signer/tests/integration/postgres.rs | 187 ++++++++++++++------------- 1 file changed, 97 insertions(+), 90 deletions(-) diff --git a/signer/tests/integration/postgres.rs b/signer/tests/integration/postgres.rs index fc3226c3d..175ef7837 100644 --- a/signer/tests/integration/postgres.rs +++ b/signer/tests/integration/postgres.rs @@ -8004,6 +8004,103 @@ mod canonical_bitcoin_blockchain { testing::storage::drop_db(db).await; } + /// Like the test_set_canonical_bitcoin_blockchain test above, this + /// test checks that the DbWrite::set_canonical_bitcoin_blockchain() + /// function sets is_canonical = TRUE in the bitcoin_blocks table for + /// all blocks in the chain along the chain tip. It also check that if + /// forked blocks are added to the database along with canonical + /// blocks, that the forked blocks are marked as non-canonical after a + /// call to DbWrite::set_canonical_bitcoin_blockchain. + /// + /// The difference with this test is that it uses the TestData struct + /// to generate many random blockchains and insert them all into the + /// database. + #[tokio::test] + async fn test_set_canonical_bitcoin_blockchain_v2() { + let db = testing::storage::new_test_database().await; + + let mut rng = get_rng(); + let signer_keys = testing::wsts::generate_signer_set_public_keys(&mut rng, 7); + + // Create many random blockchains and insert them all into the database + let params = testing::storage::model::Params { + num_bitcoin_blocks: 128, + num_stacks_blocks_per_bitcoin_block: 0, + num_deposit_requests_per_block: 0, + num_withdraw_requests_per_block: 0, + num_signers_per_request: 0, + consecutive_blocks: false, + }; + let blockchains = TestData::generate(&mut rng, &signer_keys, ¶ms); + + // Insert all blocks from into the database + for block in blockchains.bitcoin_blocks.iter() { + db.write_bitcoin_block(block).await.unwrap(); + } + + // Get the chain tip of the canonical chain + let chain_tip = db + .get_bitcoin_canonical_chain_tip_ref() + .await + .unwrap() + .unwrap(); + + // Initially, all blocks should have is_canonical = NULL + // Verify that no blocks have is_canonical IS NOT NULL + let has_non_null_canonical: bool = sqlx::query_scalar( + "SELECT EXISTS( + SELECT 1 + FROM sbtc_signer.bitcoin_blocks + WHERE is_canonical IS NOT NULL + )", + ) + .fetch_one(db.pool()) + .await + .unwrap(); + + assert!(!has_non_null_canonical); + + // Call set_canonical_bitcoin_blockchain with the chain tip + db.set_canonical_bitcoin_blockchain(&chain_tip.block_hash) + .await + .unwrap(); + + // To make sure that not everything is marked as canonical or + // non-canonical, we have flags that tracks if at least one block + // is canonical and at least one block is non-canonical. + let mut some_canonical_block_exists = false; + let mut some_non_canonical_block_exists = false; + + // Verify that only blocks in the canonical chain are marked as canonical + for block in blockchains.bitcoin_blocks.iter() { + // This function checks whether the given block is on the + // canonical chain by recursing backwards from the chain tip to + // the given block. + let on_canonical_chain = db + .in_canonical_bitcoin_blockchain(&chain_tip, &block.into()) + .await + .unwrap(); + // This function checks the is_canonical column for the given + // block hash. + let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); + + // For this test, every block should have their is_canonical + // column set to either TRUE or FALSE, no NULL values. And in + // general, the value set should match the value returned by + // the in_canonical_bitcoin_blockchain function. + assert_eq!(Some(on_canonical_chain), is_canonical); + + if on_canonical_chain { + some_canonical_block_exists = true; + } else { + some_non_canonical_block_exists = true; + } + } + assert!(some_canonical_block_exists); + assert!(some_non_canonical_block_exists); + testing::storage::drop_db(db).await; + } + /// Similar to the above test, we have two chains each with differing /// lengths, both in the database. The "main" when /// DbWrite::set_canonical_bitcoin_blockchain() is called is the longer @@ -8233,94 +8330,4 @@ mod canonical_bitcoin_blockchain { testing::storage::drop_db(db).await; } - - /// Like the test_set_canonical_bitcoin_blockchain test above, this - /// test checks that the DbWrite::set_canonical_bitcoin_blockchain() - /// function sets is_canonical = TRUE in the bitcoin_blocks table for - /// all blocks in the chain along the chain tip. It also check that if - /// forked blocks are added to the database along with canonical - /// blocks, that the forked blocks are marked as non-canonical. - /// - /// The difference with this test is that it uses the TestData struct - /// to generate many random blockchains and insert them all into the - /// database. - #[tokio::test] - async fn test_set_canonical_bitcoin_blockchain_v2() { - let db = testing::storage::new_test_database().await; - - let mut rng = get_rng(); - let signer_keys = testing::wsts::generate_signer_set_public_keys(&mut rng, 7); - - // Create many random blockchains and insert them all into the database - let params = testing::storage::model::Params { - num_bitcoin_blocks: 100, - num_stacks_blocks_per_bitcoin_block: 0, - num_deposit_requests_per_block: 0, - num_withdraw_requests_per_block: 0, - num_signers_per_request: 0, - consecutive_blocks: false, - }; - let blockchains = TestData::generate(&mut rng, &signer_keys, ¶ms); - - // Insert all blocks from into the database - for block in blockchains.bitcoin_blocks.iter() { - db.write_bitcoin_block(block).await.unwrap(); - } - - // Get the chain tip of the canonical chain - let chain_tip = db - .get_bitcoin_canonical_chain_tip_ref() - .await - .unwrap() - .unwrap(); - - // Initially, all blocks should have is_canonical = NULL - // Verify that no blocks have is_canonical IS NOT NULL - let has_non_null_canonical: bool = sqlx::query_scalar( - "SELECT EXISTS( - SELECT 1 - FROM sbtc_signer.bitcoin_blocks - WHERE is_canonical IS NOT NULL - )", - ) - .fetch_one(db.pool()) - .await - .unwrap(); - - assert!(!has_non_null_canonical); - - // Call set_canonical_bitcoin_blockchain with the chain tip - db.set_canonical_bitcoin_blockchain(&chain_tip.block_hash) - .await - .unwrap(); - - // To make sure that not everything is marked as non-canonical, we - // have a flag that at least one block is canonical. - let mut some_canonical_block_exists = false; - - // Verify that only blocks in the canonical chain are marked as canonical - for block in blockchains.bitcoin_blocks.iter() { - // This function checks whether the given block is on the - // canonical chain by recursing backwards from the chain tip to - // the given block. - let on_canonical_chain = db - .in_canonical_bitcoin_blockchain(&chain_tip, &block.into()) - .await - .unwrap(); - // This function checks the is_canonical column for the given - // block hash. - let is_canonical = db.is_block_canonical(&block.block_hash).await.unwrap(); - - // Every block should have their is_canonical column set to - // some value, and it should match what the - // in_canonical_bitcoin_blockchain function returns. - assert_eq!(Some(on_canonical_chain), is_canonical); - - if on_canonical_chain { - some_canonical_block_exists = true; - } - } - assert!(some_canonical_block_exists); - testing::storage::drop_db(db).await; - } }