diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 4fc71c50f..e805701a8 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -23,3 +23,8 @@ serde = ["dep:serde", "bitcoin/serde", "hashbrown?/serde"] [dev-dependencies] bdk_chain = { path = "../chain" } bdk_testenv = { path = "../testenv", default-features = false } +criterion = { version = "0.2" } + +[[bench]] +name = "checkpoint_skiplist" +harness = false diff --git a/crates/core/benches/checkpoint_skiplist.rs b/crates/core/benches/checkpoint_skiplist.rs new file mode 100644 index 000000000..ddf109a63 --- /dev/null +++ b/crates/core/benches/checkpoint_skiplist.rs @@ -0,0 +1,209 @@ +use bdk_core::CheckPoint; +use bitcoin::hashes::Hash; +use bitcoin::BlockHash; +use criterion::{black_box, criterion_group, criterion_main, Bencher, Criterion}; + +/// Create a checkpoint chain with the given length +fn create_checkpoint_chain(length: u32) -> CheckPoint { + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + for height in 1..=length { + let hash = BlockHash::from_byte_array([(height % 256) as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + cp +} + +/// Benchmark get() operations at various depths +fn bench_checkpoint_get(c: &mut Criterion) { + // Small chain - get near start + c.bench_function("get_100_near_start", |b: &mut Bencher| { + let cp = create_checkpoint_chain(100); + let target = 10; + b.iter(|| { + black_box(cp.get(target)); + }); + }); + + // Medium chain - get middle + c.bench_function("get_1000_middle", |b: &mut Bencher| { + let cp = create_checkpoint_chain(1000); + let target = 500; + b.iter(|| { + black_box(cp.get(target)); + }); + }); + + // Large chain - get near end + c.bench_function("get_10000_near_end", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 9000; + b.iter(|| { + black_box(cp.get(target)); + }); + }); + + // Large chain - get near start (best case for skiplist) + c.bench_function("get_10000_near_start", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 100; + b.iter(|| { + black_box(cp.get(target)); + }); + }); +} + +/// Benchmark floor_at() operations +fn bench_checkpoint_floor_at(c: &mut Criterion) { + c.bench_function("floor_at_1000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(1000); + let target = 750; // Target that might not exist exactly + b.iter(|| { + black_box(cp.floor_at(target)); + }); + }); + + c.bench_function("floor_at_10000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 7500; + b.iter(|| { + black_box(cp.floor_at(target)); + }); + }); +} + +/// Benchmark range() iteration +fn bench_checkpoint_range(c: &mut Criterion) { + c.bench_function("range_1000_20pct", |b: &mut Bencher| { + let cp = create_checkpoint_chain(1000); + let start = 400; + let end = 600; + b.iter(|| { + let range: Vec<_> = cp.range(start..=end).collect(); + black_box(range); + }); + }); + + c.bench_function("range_10000_to_end", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let from = 5000; + b.iter(|| { + let range: Vec<_> = cp.range(from..).collect(); + black_box(range); + }); + }); +} + +/// Benchmark insert() operations +fn bench_checkpoint_insert(c: &mut Criterion) { + c.bench_function("insert_sparse_1000", |b: &mut Bencher| { + // Create a sparse chain + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + for i in 1..=100 { + let height = i * 10; + let hash = BlockHash::from_byte_array([(height % 256) as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + let insert_height = 505; + let insert_hash = BlockHash::from_byte_array([255; 32]); + + b.iter(|| { + let result = cp.clone().insert(insert_height, insert_hash); + black_box(result); + }); + }); +} + +/// Compare linear traversal vs skiplist-enhanced get() +fn bench_traversal_comparison(c: &mut Criterion) { + // Linear traversal benchmark + c.bench_function("linear_traversal_10000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 100; // Near the beginning + + b.iter(|| { + let mut current = cp.clone(); + while current.height() > target { + if let Some(prev) = current.prev() { + current = prev; + } else { + break; + } + } + black_box(current); + }); + }); + + // Skiplist-enhanced get() for comparison + c.bench_function("skiplist_get_10000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 100; // Same target + + b.iter(|| { + black_box(cp.get(target)); + }); + }); +} + +/// Analyze skip pointer distribution and usage +fn bench_skip_pointer_analysis(c: &mut Criterion) { + c.bench_function("count_skip_pointers_10000", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + + b.iter(|| { + let mut count = 0; + let mut current = cp.clone(); + loop { + if current.skip().is_some() { + count += 1; + } + if let Some(prev) = current.prev() { + current = prev; + } else { + break; + } + } + black_box(count); + }); + }); + + // Measure actual skip pointer usage during traversal + c.bench_function("skip_usage_in_traversal", |b: &mut Bencher| { + let cp = create_checkpoint_chain(10000); + let target = 100; + + b.iter(|| { + let mut current = cp.clone(); + let mut skips_used = 0; + + while current.height() > target { + if let Some(skip_cp) = current.skip() { + if skip_cp.height() >= target { + current = skip_cp; + skips_used += 1; + continue; + } + } + + if let Some(prev) = current.prev() { + current = prev; + } else { + break; + } + } + black_box((current, skips_used)); + }); + }); +} + +criterion_group!( + benches, + bench_checkpoint_get, + bench_checkpoint_floor_at, + bench_checkpoint_range, + bench_checkpoint_insert, + bench_traversal_comparison, + bench_skip_pointer_analysis +); + +criterion_main!(benches); diff --git a/crates/core/src/checkpoint.rs b/crates/core/src/checkpoint.rs index d0a9bacd7..0469cd932 100644 --- a/crates/core/src/checkpoint.rs +++ b/crates/core/src/checkpoint.rs @@ -6,6 +6,9 @@ use bitcoin::{block::Header, BlockHash}; use crate::BlockId; +/// Interval for skiplist pointers based on checkpoint index. +const CHECKPOINT_SKIP_INTERVAL: u32 = 100; + /// A checkpoint is a node of a reference-counted linked list of [`BlockId`]s. /// /// Checkpoints are cheaply cloneable and are useful to find the agreement point between two sparse @@ -28,6 +31,10 @@ struct CPInner { data: D, /// Previous checkpoint (if any). prev: Option>>, + /// Skip pointer for fast traversals. + skip: Option>>, + /// Index of this checkpoint (number of checkpoints from the first). + index: u32, } /// When a `CPInner` is dropped we need to go back down the chain and manually remove any @@ -125,6 +132,16 @@ impl CheckPoint { self.0.prev.clone().map(CheckPoint) } + /// Get the index of this checkpoint (number of checkpoints from the first). + pub fn index(&self) -> u32 { + self.0.index + } + + /// Get the skip pointer checkpoint if it exists. + pub fn skip(&self) -> Option> { + self.0.skip.clone().map(CheckPoint) + } + /// Iterate from this checkpoint in descending height. pub fn iter(&self) -> CheckPointIter { self.clone().into_iter() @@ -134,7 +151,47 @@ impl CheckPoint { /// /// Returns `None` if checkpoint at `height` does not exist`. pub fn get(&self, height: u32) -> Option { - self.range(height..=height).next() + // Quick path for current height + if self.height() == height { + return Some(self.clone()); + } + + // Use skip pointers for efficient traversal + let mut current = self.clone(); + + // First, use skip pointers to get close + while current.height() > height { + // Try to use skip pointer if it won't overshoot + if let Some(skip_cp) = current.skip() { + if skip_cp.height() >= height { + current = skip_cp; + continue; + } + } + + // Fall back to regular traversal + match current.prev() { + Some(prev) => { + if prev.height() < height { + // Height doesn't exist in the chain + return None; + } + current = prev; + } + None => return None, + } + + if current.height() == height { + return Some(current); + } + } + + // Check if we found the height after the loop + if current.height() == height { + Some(current) + } else { + None + } } /// Iterate checkpoints over a height range. @@ -147,17 +204,42 @@ impl CheckPoint { { let start_bound = range.start_bound().cloned(); let end_bound = range.end_bound().cloned(); - self.iter() - .skip_while(move |cp| match end_bound { - core::ops::Bound::Included(inc_bound) => cp.height() > inc_bound, - core::ops::Bound::Excluded(exc_bound) => cp.height() >= exc_bound, - core::ops::Bound::Unbounded => false, - }) - .take_while(move |cp| match start_bound { - core::ops::Bound::Included(inc_bound) => cp.height() >= inc_bound, - core::ops::Bound::Excluded(exc_bound) => cp.height() > exc_bound, - core::ops::Bound::Unbounded => true, - }) + + // Fast-path to find starting point using skip pointers + let mut current = self.clone(); + + // Skip past checkpoints that are above the end bound + while match end_bound { + core::ops::Bound::Included(inc_bound) => current.height() > inc_bound, + core::ops::Bound::Excluded(exc_bound) => current.height() >= exc_bound, + core::ops::Bound::Unbounded => false, + } { + // Try to use skip pointer if it won't overshoot + if let Some(skip_cp) = current.skip() { + let use_skip = match end_bound { + core::ops::Bound::Included(inc_bound) => skip_cp.height() > inc_bound, + core::ops::Bound::Excluded(exc_bound) => skip_cp.height() >= exc_bound, + core::ops::Bound::Unbounded => false, + }; + if use_skip { + current = skip_cp; + continue; + } + } + + // Fall back to regular traversal + match current.prev() { + Some(prev) => current = prev, + None => break, + } + } + + // Now iterate normally from the found starting point + current.into_iter().take_while(move |cp| match start_bound { + core::ops::Bound::Included(inc_bound) => cp.height() >= inc_bound, + core::ops::Bound::Excluded(exc_bound) => cp.height() > exc_bound, + core::ops::Bound::Unbounded => true, + }) } /// Returns the checkpoint at `height` if one exists, otherwise the nearest checkpoint at a @@ -167,7 +249,38 @@ impl CheckPoint { /// /// Returns `None` if no checkpoint exists at or below the given height. pub fn floor_at(&self, height: u32) -> Option { - self.range(..=height).next() + // Quick path for current height or higher + if self.height() <= height { + return Some(self.clone()); + } + + // Use skip pointers for efficient traversal + let mut current = self.clone(); + + while current.height() > height { + // Try to use skip pointer if it won't undershoot + if let Some(skip_cp) = current.skip() { + if skip_cp.height() > height { + current = skip_cp; + continue; + } + } + + // Fall back to regular traversal + match current.prev() { + Some(prev) => { + // If prev is at or below height, we've found our floor + if prev.height() <= height { + return Some(prev); + } + current = prev; + } + None => return None, + } + } + + // Current is at or below height + Some(current) } /// Returns the checkpoint located a number of heights below this one. @@ -205,6 +318,8 @@ where }, data, prev: None, + skip: None, + index: 0, })) } @@ -269,8 +384,69 @@ where cp = cp.prev().expect("will break before genesis block"); }; - base.extend(core::iter::once((height, data)).chain(tail.into_iter().rev())) - .expect("tail is in order") + // Rebuild the chain with proper indices + let mut result = base.clone(); + let base_index = result.index(); + + // First insert the new block + result = result + .push_with_index(height, data, base_index + 1) + .expect("height is valid"); + + // Then re-add all the tail blocks with updated indices + let mut current_index = base_index + 2; + for (h, d) in tail.into_iter().rev() { + result = result + .push_with_index(h, d, current_index) + .expect("tail is in order"); + current_index += 1; + } + + result + } + + // Helper method to push with a specific index (internal use) + fn push_with_index(self, height: u32, data: D, new_index: u32) -> Result { + if self.height() < height { + // Calculate skip pointer + let skip = if new_index >= CHECKPOINT_SKIP_INTERVAL + && new_index % CHECKPOINT_SKIP_INTERVAL == 0 + { + // Navigate back CHECKPOINT_SKIP_INTERVAL checkpoints + let target_index = new_index - CHECKPOINT_SKIP_INTERVAL; + let mut current = Some(self.0.clone()); + loop { + match current { + Some(ref cp) if cp.index == target_index => break, + Some(ref cp) if cp.index < target_index => { + // We've gone too far back, skip pointer not available + current = None; + break; + } + Some(ref cp) => { + current = cp.prev.clone(); + } + None => break, + } + } + current + } else { + None + }; + + Ok(Self(Arc::new(CPInner { + block_id: BlockId { + height, + hash: data.to_blockhash(), + }, + data, + prev: Some(self.0), + skip, + index: new_index, + }))) + } else { + Err(self) + } } /// Puts another checkpoint onto the linked list representing the blockchain. @@ -279,6 +455,35 @@ where /// one you are pushing on to. pub fn push(self, height: u32, data: D) -> Result { if self.height() < height { + let new_index = self.0.index + 1; + + // Calculate skip pointer + let skip = if new_index >= CHECKPOINT_SKIP_INTERVAL + && new_index % CHECKPOINT_SKIP_INTERVAL == 0 + { + // Navigate back CHECKPOINT_SKIP_INTERVAL checkpoints + let mut current = Some(self.0.clone()); + let mut steps = 0; + loop { + match current { + Some(ref cp) if cp.index == new_index - CHECKPOINT_SKIP_INTERVAL => break, + Some(ref cp) => { + current = cp.prev.clone(); + steps += 1; + // Safety check to avoid infinite loop + if steps > CHECKPOINT_SKIP_INTERVAL { + current = None; + break; + } + } + None => break, + } + } + current + } else { + None + }; + Ok(Self(Arc::new(CPInner { block_id: BlockId { height, @@ -286,6 +491,8 @@ where }, data, prev: Some(self.0), + skip, + index: new_index, }))) } else { Err(self) diff --git a/crates/core/tests/test_checkpoint_skiplist.rs b/crates/core/tests/test_checkpoint_skiplist.rs new file mode 100644 index 000000000..c92ee642a --- /dev/null +++ b/crates/core/tests/test_checkpoint_skiplist.rs @@ -0,0 +1,196 @@ +use bdk_core::CheckPoint; +use bitcoin::hashes::Hash; +use bitcoin::BlockHash; + +#[test] +fn test_skiplist_indices() { + // Create a long chain to test skiplist + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + assert_eq!(cp.index(), 0); + + for height in 1..=500 { + let hash = BlockHash::from_byte_array([height as u8; 32]); + cp = cp.push(height, hash).unwrap(); + assert_eq!(cp.index(), height); + } + + // Test that skip pointers are set correctly + // At index 100, 200, 300, 400, 500 we should have skip pointers + assert_eq!(cp.index(), 500); + + // Navigate to index 400 and check skip pointer + let mut current = cp.clone(); + for _ in 0..100 { + current = current.prev().unwrap(); + } + assert_eq!(current.index(), 400); + + // Check that skip pointer exists at index 400 + if let Some(skip) = current.skip() { + assert_eq!(skip.index(), 300); + } else { + panic!("Expected skip pointer at index 400"); + } + + // Navigate to index 300 and check skip pointer + for _ in 0..100 { + current = current.prev().unwrap(); + } + assert_eq!(current.index(), 300); + + if let Some(skip) = current.skip() { + assert_eq!(skip.index(), 200); + } else { + panic!("Expected skip pointer at index 300"); + } + + // Navigate to index 100 and check skip pointer + for _ in 0..200 { + current = current.prev().unwrap(); + } + assert_eq!(current.index(), 100); + + if let Some(skip) = current.skip() { + assert_eq!(skip.index(), 0); + } else { + panic!("Expected skip pointer at index 100"); + } +} + +#[test] +fn test_skiplist_get_performance() { + // Create a very long chain + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + + for height in 1..=1000 { + let hash = BlockHash::from_byte_array([(height % 256) as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + // Test that get() can find checkpoints efficiently + // This should use skip pointers to navigate quickly + + // Verify the chain was built correctly + assert_eq!(cp.height(), 1000); + assert_eq!(cp.index(), 1000); + + // Find checkpoint near the beginning + if let Some(found) = cp.get(50) { + assert_eq!(found.height(), 50); + assert_eq!(found.index(), 50); + } else { + // Debug: print the first few checkpoints + let mut current = cp.clone(); + println!("First 10 checkpoints:"); + for _ in 0..10 { + println!("Height: {}, Index: {}", current.height(), current.index()); + if let Some(prev) = current.prev() { + current = prev; + } else { + break; + } + } + panic!("Could not find checkpoint at height 50"); + } + + // Find checkpoint in the middle + if let Some(found) = cp.get(500) { + assert_eq!(found.height(), 500); + assert_eq!(found.index(), 500); + } else { + panic!("Could not find checkpoint at height 500"); + } + + // Find checkpoint near the end + if let Some(found) = cp.get(950) { + assert_eq!(found.height(), 950); + assert_eq!(found.index(), 950); + } else { + panic!("Could not find checkpoint at height 950"); + } + + // Test non-existent checkpoint + assert!(cp.get(1001).is_none()); +} + +#[test] +fn test_skiplist_floor_at() { + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + + // Create sparse chain with gaps + for height in [10, 50, 100, 150, 200, 300, 400, 500] { + let hash = BlockHash::from_byte_array([height as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + // Test floor_at with skip pointers + let floor = cp.floor_at(250).unwrap(); + assert_eq!(floor.height(), 200); + + let floor = cp.floor_at(99).unwrap(); + assert_eq!(floor.height(), 50); + + let floor = cp.floor_at(500).unwrap(); + assert_eq!(floor.height(), 500); + + let floor = cp.floor_at(600).unwrap(); + assert_eq!(floor.height(), 500); +} + +#[test] +fn test_skiplist_insert_maintains_indices() { + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + + // Build initial chain + for height in [10, 20, 30, 40, 50] { + let hash = BlockHash::from_byte_array([height as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + // Insert a block in the middle + let hash = BlockHash::from_byte_array([25; 32]); + cp = cp.insert(25, hash); + + // Check that indices are maintained correctly + let check = cp.get(50).unwrap(); + assert_eq!(check.index(), 6); // 0, 10, 20, 25, 30, 40, 50 + + let check = cp.get(25).unwrap(); + assert_eq!(check.index(), 3); + + // Check the full chain has correct indices + let mut current = cp.clone(); + let expected_heights = [50, 40, 30, 25, 20, 10, 0]; + let expected_indices = [6, 5, 4, 3, 2, 1, 0]; + + for (expected_height, expected_index) in expected_heights.iter().zip(expected_indices.iter()) { + assert_eq!(current.height(), *expected_height); + assert_eq!(current.index(), *expected_index); + if *expected_height > 0 { + current = current.prev().unwrap(); + } + } +} + +#[test] +fn test_skiplist_range_uses_skip_pointers() { + let mut cp = CheckPoint::new(0, BlockHash::all_zeros()); + + // Create a chain with 500 checkpoints + for height in 1..=500 { + let hash = BlockHash::from_byte_array([(height % 256) as u8; 32]); + cp = cp.push(height, hash).unwrap(); + } + + // Test range iteration + let range_items: Vec<_> = cp.range(100..=200).collect(); + assert_eq!(range_items.len(), 101); + assert_eq!(range_items.first().unwrap().height(), 200); + assert_eq!(range_items.last().unwrap().height(), 100); + + // Test open range + let range_items: Vec<_> = cp.range(450..).collect(); + assert_eq!(range_items.len(), 51); + assert_eq!(range_items.first().unwrap().height(), 500); + assert_eq!(range_items.last().unwrap().height(), 450); +}