Create a fast IBD client

.gitignore

@@ -3,3 +3,6 @@ Cargo.lock
 !justfile
 *.db
 *.hints
+/node/chainstate
+/node/blocks
+/node/blockfiles

Cargo.toml

@@ -1,7 +1,9 @@
 [workspace]
-members = ["accumulator", "hintfile", "network"]
+members = ["accumulator", "hintfile", "network", "node"]
 default-members = ["accumulator", "network"]
 resolver = "2"
 
 [workspace.dependencies]
 bitcoin = { git = "https://github.com/rust-bitcoin/rust-bitcoin", default-features = false, rev = "16cc257c3695dea0e7301a5fa9cab44b8ed60598" }
+kernel = { package = "bitcoinkernel", git = "https://github.com/alexanderwiederin/rust-bitcoinkernel.git", rev = "353533221e3ba91d672418eab1ae7b83a61214f9" }
+p2p = { package = "bitcoin-p2p", git =  "https://github.com/2140-dev/bitcoin-p2p.git", rev = "4f67fbeced98e3ddcc65a1333c46823a2b56332a" }

accumulator/src/lib.rs

@@ -34,6 +34,13 @@ fn split_in_half(a: [u8; 32]) -> ([u8; 16], [u8; 16]) {
     (high, low)
 }
 
+/// Update an accumulator by adding or spending a pre-hashed outpoint
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum AccumulatorUpdate {
+    Add([u8; 32]),
+    Spent([u8; 32]),
+}
+
 impl Accumulator {
     /// The zero accumulator
     pub const ZERO: Accumulator = Accumulator { high: 0, low: 0 };
@@ -59,6 +66,14 @@ impl Accumulator {
         *self = Self { high, low };
     }
 
+    /// Update the accumulator
+    pub fn update(&mut self, update: AccumulatorUpdate) {
+        match update {
+            AccumulatorUpdate::Add(added) => self.add_hashed_outpoint(added),
+            AccumulatorUpdate::Spent(spent) => self.spend_hashed_outpoint(spent),
+        }
+    }
+
     /// Spend the inputs in a block by subtracing them from the accumulator.
     pub fn spend(&mut self, outpoint: OutPoint) {
         let hash = hash_outpoint(outpoint);
@@ -226,4 +241,27 @@ mod tests {
         acc.spend_hashed_outpoint(hash_two);
         acc.spend_hashed_outpoint(hash_three);
     }
+
+    #[test]
+    fn test_update_method() {
+        let [outpoint_one, outpoint_two, outpoint_three, outpoint_four, outpoint_five] =
+            make_five_outpoint();
+        let hash_one = hash_outpoint(outpoint_one);
+        let hash_two = hash_outpoint(outpoint_two);
+        let hash_three = hash_outpoint(outpoint_three);
+        let hash_four = hash_outpoint(outpoint_four);
+        let hash_five = hash_outpoint(outpoint_five);
+        let mut acc = Accumulator::default();
+        acc.update(AccumulatorUpdate::Add(hash_one));
+        acc.update(AccumulatorUpdate::Add(hash_two));
+        acc.update(AccumulatorUpdate::Add(hash_three));
+        acc.update(AccumulatorUpdate::Add(hash_four));
+        acc.update(AccumulatorUpdate::Add(hash_five));
+        acc.update(AccumulatorUpdate::Spent(hash_five));
+        acc.update(AccumulatorUpdate::Spent(hash_four));
+        acc.update(AccumulatorUpdate::Spent(hash_three));
+        acc.update(AccumulatorUpdate::Spent(hash_two));
+        acc.update(AccumulatorUpdate::Spent(hash_one));
+        assert!(acc.is_zero());
+    }
 }

hintfile/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2021"
 
 [dependencies]
 bitcoin = { workspace = true }
-kernel = { package = "bitcoinkernel", git = "https://github.com/alexanderwiederin/rust-bitcoinkernel.git", rev = "b7b42d65a70f2c0e5fc8d1d024549bf857211046" }
+kernel = { workspace = true }
 
 [[bin]]
 name = "construct"

hintfile/src/bin/construct.rs

@@ -3,7 +3,7 @@ use std::{fs::File, io::Write, sync::Arc};
 use hintfile::write_compact_size;
 use kernel::{ChainType, ChainstateManager, ChainstateManagerOptions, ContextBuilder, KernelError};
 
-const CHAIN_TYPE: ChainType = ChainType::MAINNET;
+const CHAIN_TYPE: ChainType = ChainType::SIGNET;
 
 fn main() {
     let mut file = File::create("./bitcoin.hints").unwrap();

network/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "peers"
+name = "network"
 version = "0.1.0"
 edition = "2021"
 

network/tests/test.rs

@@ -1,6 +1,6 @@
 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 
-use peers::dns::DnsQuery;
+use network::dns::DnsQuery;
 
 #[tokio::test]
 #[cfg(feature = "tokio")]

node/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "node"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+accumulator = { path = "../accumulator/" }
+bitcoin = { workspace = true }
+kernel = { workspace = true }
+hintfile = { path = "../hintfile/" }
+network = { path = "../network/" }
+p2p = { workspace = true }
+
+tracing = "0.1"
+tracing-subscriber = "0.3"
+
+[[bin]]
+name = "ibd"

node/justfile

@@ -0,0 +1,5 @@
+ibd:
+  cargo run --bin ibd ./bitcoin.hints --release
+
+delete:
+  rm -rf blocks chainstate blockfiles

node/src/bin/ibd.rs

@@ -0,0 +1,92 @@
+use std::{
+    fs::File,
+    path::Path,
+    sync::{mpsc::channel, Arc, Mutex},
+    time::Instant,
+};
+
+use bitcoin::Network;
+use hintfile::Hints;
+use kernel::{ChainType, ChainstateManager, ChainstateManagerOptions, ContextBuilder};
+
+use node::{
+    bootstrap_dns, elapsed_time, get_blocks_for_range, hashes_from_chain, sync_block_headers,
+    AccumulatorState,
+};
+
+const CHAIN_TYPE: ChainType = ChainType::SIGNET;
+const NETWORK: Network = Network::Signet;
+const TASKS: usize = 256;
+const BLOCK_FILE_PATH: &str = "./blockfiles";
+
+fn main() {
+    let mut args = std::env::args();
+    let _ = args.next();
+    let hint_path = args.next().expect("Usage: <path_to_hints_file>");
+    // Logging
+    let subscriber = tracing_subscriber::FmtSubscriber::new();
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+    let hintfile_start_time = Instant::now();
+    tracing::info!("Reading in {hint_path}");
+    let mut hintfile = File::open(hint_path).expect("invalid hintfile path");
+    let hints = Arc::new(Hints::from_file(&mut hintfile));
+    elapsed_time(hintfile_start_time);
+    let block_file_path = Path::new(BLOCK_FILE_PATH);
+    std::fs::create_dir(block_file_path).expect("could not create block file directory");
+    let stop_hash = hints.stop_hash();
+    tracing::info!("Assume valid hash: {stop_hash}");
+    tracing::info!("Finding peers with DNS");
+    let dns_start_time = Instant::now();
+    let peers = bootstrap_dns(NETWORK);
+    elapsed_time(dns_start_time);
+    tracing::info!("Initializing bitcoin kernel");
+    let kernel_start_time = Instant::now();
+    let ctx = ContextBuilder::new()
+        .chain_type(CHAIN_TYPE)
+        .build()
+        .unwrap();
+    let options = ChainstateManagerOptions::new(&ctx, ".", "./blocks").unwrap();
+    let chainman = ChainstateManager::new(options).unwrap();
+    elapsed_time(kernel_start_time);
+    let tip = chainman.best_header().height();
+    tracing::info!("Kernel best header: {tip}");
+    let chain = Arc::new(chainman);
+    sync_block_headers(stop_hash, &peers, Arc::clone(&chain), NETWORK);
+    tracing::info!("Assume valid height: {}", chain.best_header().height());
+    let (tx, rx) = channel();
+    let main_routine_time = Instant::now();
+    let mut accumulator_state = AccumulatorState::new(rx);
+    let acc_task = std::thread::spawn(move || accumulator_state.verify());
+    let peers = Arc::new(Mutex::new(peers));
+    let mut tasks = Vec::new();
+    let chunk_size = chain.best_header().height() as usize / TASKS;
+    let hashes = hashes_from_chain(Arc::clone(&chain), chunk_size);
+    for (task_id, chunk) in hashes.into_iter().enumerate() {
+        let chain = Arc::clone(&chain);
+        let tx = tx.clone();
+        let peers = Arc::clone(&peers);
+        let hints = Arc::clone(&hints);
+        let block_task = std::thread::spawn(move || {
+            get_blocks_for_range(
+                task_id as u32,
+                NETWORK,
+                block_file_path,
+                chain,
+                &hints,
+                peers,
+                tx,
+                chunk,
+            )
+        });
+        tasks.push(block_task);
+    }
+    for task in tasks {
+        if let Err(e) = task.join() {
+            tracing::warn!("{:?}", e.downcast::<String>());
+        }
+    }
+    drop(tx);
+    let acc_result = acc_task.join().unwrap();
+    tracing::info!("Verified: {acc_result}");
+    elapsed_time(main_routine_time);
+}