write scanning test for bdk_electrum

Author: vladimirfomene

crates/electrum/Cargo.toml

@@ -14,3 +14,7 @@ readme = "README.md"
 [dependencies]
 bdk_chain = { path = "../chain", version = "0.4.0", features = ["serde", "miniscript"] }
 electrum-client = { version = "0.12" }
+
+[dev-dependencies]
+anyhow = "1"
+electrsd = { version = "0.22.0", features = ["legacy", "bitcoind_22_0"] }

crates/electrum/tests/test_electrum_scanning.rs

@@ -0,0 +1,379 @@
+use bdk_chain::{ keychain::KeychainTxOutIndex, collections::BTreeMap, bitcoin::{ BlockHash, OutPoint, Txid, hashes::Hash, Transaction}, miniscript::{Descriptor, DescriptorPublicKey}, bitcoin::{Amount, Address, Network::Regtest, Script}, sparse_chain::ChainPosition, TxHeight, chain_graph::ChainGraph};
+use bdk_electrum::{ ElectrumExt };
+use electrsd::{ ElectrsD, bitcoind::{self, BitcoinD, bitcoincore_rpc::{bitcoincore_rpc_json::AddressType, RpcApi}}, electrum_client::ElectrumApi};
+use electrum_client::{ Client };
+use std::{ env, time::{ Duration, Instant } };
+use anyhow::Result;
+
+#[derive(Debug, Clone, PartialOrd, PartialEq, Ord, Eq)]
+enum Keychain {
+    External,
+    Internal
+}
+
+#[test]
+fn test_scanning_stop_gap() -> Result<()> {
+    let (bitcoin_daemon, electrs_daemon, tcp_client) = init_test_tools(None, None);
+    premine(&bitcoin_daemon, &electrs_daemon, 101);
+
+    
+    let local_chain:BTreeMap<u32, BlockHash> = BTreeMap::new();
+    let mut txout_index = init_txout_index();
+    let chain_graph = ChainGraph::default();
+    
+
+    let (tx, revealed_spks) = send_to_revealed_script(&mut txout_index, &bitcoin_daemon, 19, 19);
+    let (_, to_script) = revealed_spks.get(19).unwrap().to_owned();
+
+    generate_blocks_and_wait(&bitcoin_daemon, &electrs_daemon, 1);
+
+    let mut spks = BTreeMap::new();
+    spks.insert(Keychain::External, revealed_spks.into_iter());
+    let electrum_update = tcp_client.scan(&local_chain, spks, [], [], 20, 5).unwrap();
+    let new_txs = tcp_client.batch_transaction_get(electrum_update.missing_full_txs(&chain_graph))?;
+    let keychain_scan = electrum_update.into_keychain_scan(new_txs, &chain_graph)?;
+    println!("Last Active indices {:?}", keychain_scan.last_active_indices);
+    assert_eq!(*keychain_scan.last_active_indices.get(&Keychain::External).unwrap(), 19);
+    let (&conf, chain_tx) = keychain_scan.update.get_tx_in_chain(tx.txid()).unwrap();
+    assert_eq!(tx, chain_tx.clone());
+    assert!(conf.is_confirmed());
+    assert_eq!(conf.height(), TxHeight::Confirmed(103));
+    let (output_vout, _) = tx.output.iter().enumerate().find(|(_idx, out)| out.script_pubkey == to_script).unwrap();
+    let full_txout = keychain_scan.update.full_txout(bdk_chain::bitcoin::OutPoint { txid: tx.txid(), vout: output_vout as u32 });
+    assert_eq!(full_txout.unwrap().txout.value, 10000);
+
+
+    let (tx, revealed_spks) = send_to_revealed_script(&mut txout_index, &bitcoin_daemon, 38, 18);
+    let (_, to_script) = revealed_spks.get(18).unwrap().to_owned();
+
+    generate_blocks_and_wait(&bitcoin_daemon, &electrs_daemon, 1);
+
+    let mut spks = BTreeMap::new();
+    spks.insert(Keychain::External, revealed_spks.into_iter());
+    let electrum_update = tcp_client.scan(&local_chain, spks, [], [], 20, 5).unwrap();
+    let new_txs = tcp_client.batch_transaction_get(electrum_update.missing_full_txs(&chain_graph))?;
+    let keychain_scan = electrum_update.into_keychain_scan(new_txs, &chain_graph)?;
+    println!("Last Active indices {:?}", keychain_scan.last_active_indices);
+    assert_eq!(*keychain_scan.last_active_indices.get(&Keychain::External).unwrap(), 38);
+    let (&conf, chain_tx) = keychain_scan.update.get_tx_in_chain(tx.txid()).unwrap();
+    assert_eq!(tx, chain_tx.clone());
+    assert!(conf.is_confirmed());
+    assert_eq!(conf.height(), TxHeight::Confirmed(104));
+    let (output_vout, _) = tx.output.iter().enumerate().find(|(_idx, out)| out.script_pubkey == to_script).unwrap();
+    let full_txout = keychain_scan.update.full_txout(bdk_chain::bitcoin::OutPoint { txid: tx.txid(), vout: output_vout as u32 });
+    assert_eq!(full_txout.unwrap().txout.value, 10000);
+
+    let (tx, revealed_spks) = send_to_revealed_script(&mut txout_index, &bitcoin_daemon, 59, 20);
+    let (_, to_script) = revealed_spks.get(20).unwrap().to_owned();
+    
+    generate_blocks_and_wait(&bitcoin_daemon, &electrs_daemon, 1);
+
+    let mut spks = BTreeMap::new();
+    spks.insert(Keychain::External, revealed_spks.into_iter());
+    let electrum_update = tcp_client.scan(&local_chain, spks, [], [], 20, 5).unwrap();
+    println!("Last Active indices {:?}", keychain_scan.last_active_indices);
+    assert!(keychain_scan.last_active_indices.get(&Keychain::External).is_none());
+    assert!(keychain_scan.update.get_tx_in_chain(tx.txid()).is_none());
+    let (output_vout, _) = tx.output.iter().enumerate().find(|(_idx, out)| out.script_pubkey == to_script).unwrap();
+    assert!(keychain_scan.update.full_txout(bdk_chain::bitcoin::OutPoint { txid: tx.txid(), vout: output_vout as u32 }).is_none());
+
+    Ok(())
+
+}
+
+#[test]
+fn test_reorg() -> Result<()> {
+    let mut conf = bitcoind::Conf::default();
+    conf.p2p = bitcoind::P2P::Yes;
+    let (bitcoin_daemon, electrs_daemon, tcp_client) = init_test_tools( Some(conf), None);
+    let mut miner_conf = bitcoind::Conf::default();
+    miner_conf.p2p = bitcoin_daemon.p2p_connect(true).unwrap();
+    let miner_node = setup_bitcoind(Some(miner_conf));
+    premine(&bitcoin_daemon, &electrs_daemon, 101);
+
+    let local_chain:BTreeMap<u32, BlockHash> = BTreeMap::new();
+    let mut txout_index = init_txout_index();
+    let chain_graph = ChainGraph::default();
+
+    let (tx, revealed_spks) = send_to_revealed_script(&mut txout_index, &bitcoin_daemon, 0, 0);
+    let (_, revealed_spk) = revealed_spks.get(0).unwrap().to_owned();
+    
+
+    // Get the transaction confirmed above
+    generate_blocks_and_wait(&bitcoin_daemon, &electrs_daemon, 1);
+
+    let mut spks = BTreeMap::new();
+    spks.insert(Keychain::External, [(0, revealed_spk.clone())].into_iter());
+    let electrum_update = tcp_client.scan(&local_chain, spks, [], [], 20, 5).unwrap();
+    let new_txs = tcp_client.batch_transaction_get(electrum_update.missing_full_txs(&chain_graph))?;
+    let keychain_scan = electrum_update.into_keychain_scan(new_txs, &chain_graph)?;
+    assert_eq!(*keychain_scan.last_active_indices.get(&Keychain::External).unwrap(), 0);
+    let (&conf, chain_tx) = keychain_scan.update.get_tx_in_chain(tx.txid()).unwrap();
+    assert_eq!(tx, chain_tx.clone());
+    assert!(conf.is_confirmed());
+    assert_eq!(conf.height(), TxHeight::Confirmed(103));
+    let (output_vout, _) = tx.output.iter().enumerate().find(|(_idx, out)| out.value == 10000).unwrap();
+    let full_txout = keychain_scan.update.full_txout(bdk_chain::bitcoin::OutPoint { txid: tx.txid(), vout: output_vout as u32 });
+    assert_eq!(full_txout.unwrap().txout.value, 10000);
+
+    // Reorg blocks on miner chain
+    reorg(3, &miner_node)?;
+    // Generate more blocks on the miner node, thereby making it the chain with the most
+    // work, so the bitcoin_daemon chain has to catch up on this chain which doesn't
+    // have a transaction above.
+    generate_blocks_and_wait(&miner_node, &electrs_daemon, 5);
+
+    
+    let mut spks = BTreeMap::new();
+    spks.insert(Keychain::External, [(0, revealed_spk.clone())].into_iter());
+    let electrum_update = tcp_client.scan(&local_chain, spks, [], [], 20, 5).unwrap();
+    let new_txs = tcp_client.batch_transaction_get(electrum_update.missing_full_txs(&chain_graph))?;
+    let keychain_scan = electrum_update.into_keychain_scan(new_txs, &chain_graph)?;
+    let (conf, _tx_chain) = keychain_scan.update.get_tx_in_chain(tx.txid()).unwrap();
+    assert!(!conf.is_confirmed());
+    let (output_vout, _) = tx.output.iter().enumerate().find(|(_idx, out)| out.value == 10000).unwrap();
+    let full_txout = keychain_scan.update.full_txout(bdk_chain::bitcoin::OutPoint { txid: tx.txid(), vout: output_vout as u32 }).unwrap();
+    assert!(!full_txout.chain_position.is_confirmed());
+
+    Ok(())
+
+}
+
+#[test]
+fn test_scan_with_txids() -> Result<()> {
+    let (bitcoin_daemon, electrs_daemon, tcp_client) = init_test_tools(None, None);
+    premine(&bitcoin_daemon, &electrs_daemon, 101);
+
+    let local_chain:BTreeMap<u32, BlockHash> = BTreeMap::new();
+    let mut txout_index = init_txout_index();
+    let chain_graph = ChainGraph::default();
+
+    let (tx_1, _) = send_to_revealed_script(&mut txout_index, &bitcoin_daemon, 0, 0);
+
+    generate_blocks_and_wait(&bitcoin_daemon, &electrs_daemon, 1);
+
+    let electrum_update = tcp_client.scan(&local_chain, BTreeMap::<Keychain, Vec<(u32, Script)>>::new(), [tx_1.txid()], [], 20, 5).unwrap();
+    let new_txs = tcp_client.batch_transaction_get(electrum_update.missing_full_txs(&chain_graph))?;
+    let keychain_scan = electrum_update.into_keychain_scan(new_txs, &chain_graph)?;
+    assert!(keychain_scan.last_active_indices.get(&Keychain::External).is_none());
+    let (conf, tx1_chain) = keychain_scan.update.get_tx_in_chain(tx_1.txid()).unwrap();
+    assert_eq!(tx_1, tx1_chain.clone());
+    assert!(conf.is_confirmed());
+    assert_eq!(conf.height(), TxHeight::Confirmed(103));
+    let checkpoint = keychain_scan.update.chain().checkpoint_at(103).unwrap();
+    assert_eq!(checkpoint.hash, bitcoin_daemon.client.get_block_hash(103).unwrap());
+
+    let (tx_2, _) = send_to_revealed_script(&mut txout_index, &bitcoin_daemon, 1, 0);
+
+    wait_for_tx_appears_in_esplora(5, &electrs_daemon, &tx_2.txid());
+    let electrum_update = tcp_client.scan(&local_chain, BTreeMap::<Keychain, Vec<(u32, Script)>>::new(), [tx_2.txid()], [], 20, 5).unwrap();
+    let new_txs = tcp_client.batch_transaction_get(electrum_update.missing_full_txs(&chain_graph))?;
+    let keychain_scan = electrum_update.into_keychain_scan(new_txs, &chain_graph)?;
+    let (conf, _tx_chain) = keychain_scan.update.get_tx_in_chain(tx_2.txid()).unwrap();
+    assert!(!conf.is_confirmed());
+
+    Ok(())
+}
+
+#[test]
+fn test_scan_with_outpoints() -> Result<()> {
+    let (bitcoin_daemon, electrs_daemon, tcp_client) = init_test_tools(None, None);
+    premine(&bitcoin_daemon, &electrs_daemon, 101);
+
+
+    let local_chain:BTreeMap<u32, BlockHash> = BTreeMap::new();
+    let mut txout_index = init_txout_index();
+    let chain_graph = ChainGraph::default();
+
+    let mut outpoints: [OutPoint; 2] = [OutPoint::null(), OutPoint::null()];
+    let mut txids: [Txid; 2] = [Txid::from_inner([0x00; 32]), Txid::from_inner([0x00; 32])];
+    for i in 0..=1 {
+        let (tx, revealed_spks) = send_to_revealed_script(&mut txout_index, &bitcoin_daemon, i, 0);
+        let (_, revealed_spk) = revealed_spks.get(0).unwrap().to_owned();
+        let (output_vout, _) = tx.output.iter().enumerate().find(|(_idx, out)| out.script_pubkey == revealed_spk.clone()).unwrap();
+        outpoints[i as usize] = OutPoint::new(tx.txid(), output_vout as u32);
+        txids[i as usize] = tx.txid();
+    }
+
+    let electrum_update = tcp_client.scan(&local_chain, BTreeMap::<Keychain, Vec<(u32, Script)>>::new(), [], outpoints, 20, 5).unwrap();
+    let new_txs = tcp_client.batch_transaction_get(electrum_update.missing_full_txs(&chain_graph))?;
+    let keychain_scan = electrum_update.into_keychain_scan(new_txs, &chain_graph)?;
+    for i in 0..=1 {
+        assert!(keychain_scan.update.get_tx_in_chain(txids[i]).is_none());
+        assert!(keychain_scan.update.full_txout(outpoints[i]).is_none());
+    }
+
+    generate_blocks_and_wait(&bitcoin_daemon, &electrs_daemon, 1);
+
+    let electrum_update = tcp_client.scan(&local_chain, BTreeMap::<Keychain, Vec<(u32, Script)>>::new(), [], outpoints, 20, 5).unwrap();
+    let new_txs = tcp_client.batch_transaction_get(electrum_update.missing_full_txs(&chain_graph))?;
+    let keychain_scan = electrum_update.into_keychain_scan(new_txs, &chain_graph)?;
+    for i in 0..=1 {
+        let (conf, tx) = keychain_scan.update.get_tx_in_chain(txids[i]).unwrap();
+        assert!(conf.is_confirmed());
+        assert_eq!(tx.txid(), txids[i]);
+        let full_txout = keychain_scan.update.full_txout(outpoints[i]).unwrap();
+        assert!(full_txout.chain_position.is_confirmed());
+    }
+
+    Ok(())
+
+}
+
+fn init_txout_index() ->  KeychainTxOutIndex::<Keychain> {
+    let mut txout_index = KeychainTxOutIndex::<Keychain>::default();
+    let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::default();
+    let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
+    let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
+
+    txout_index.add_keychain(Keychain::External, external_descriptor);
+    txout_index.add_keychain(Keychain::Internal, internal_descriptor);
+    txout_index
+}
+
+fn init_test_tools(bitcoind_conf: Option<bitcoind::Conf>, electrsd_conf: Option<electrsd::Conf>) -> (BitcoinD, ElectrsD, Client){
+    let bitcoin_daemon = setup_bitcoind(bitcoind_conf);
+    let electrs_daemon = setup_electrsd(electrsd_conf, &bitcoin_daemon);
+    let tcp_client = setup_client(&electrs_daemon);
+
+    (bitcoin_daemon, electrs_daemon, tcp_client)
+}
+
+fn send_to_revealed_script(txout_index: &mut KeychainTxOutIndex<Keychain>, bitcoin_daemon: &BitcoinD, reveal_target: u32, last_revealed_idx: usize) -> (Transaction, Vec<(u32, Script)>) {
+    let revealed_spks = txout_index.reveal_to_target(&Keychain::External, reveal_target).0.collect::<Vec<(u32, Script)>>();
+    println!("Revealed Spks {:?}", revealed_spks);
+    let (_idx, script) = revealed_spks.get(last_revealed_idx).unwrap().to_owned();
+    let address = Address::from_script(&script, Regtest).unwrap();
+    let txid = bitcoin_daemon
+        .client
+        .send_to_address(
+            &address,
+            Amount::from_sat(10000),
+            None,
+            None,
+            None,
+            None,
+            None,
+            None,
+        )
+        .unwrap();
+
+    (bitcoin_daemon.client.get_transaction(&txid, Some(false)).unwrap().transaction().unwrap(), revealed_spks)
+}
+
+fn wait_for_block(electrs_daemon: &ElectrsD, min_height: usize) {
+    let mut header = electrs_daemon.client.block_headers_subscribe().unwrap();
+    loop {
+        if header.height >= min_height {
+            break;
+        }
+        header = exponential_backoff_poll(|| {
+            electrs_daemon.trigger().unwrap();
+            electrs_daemon.client.ping().unwrap();
+            electrs_daemon.client.block_headers_pop().unwrap()
+        });
+    }
+}
+
+fn wait_for_tx_appears_in_esplora(wait_seconds: u64, electrs_daemon: &ElectrsD, txid: &bdk_chain::bitcoin::Txid) -> bool {
+    let instant = Instant::now();
+    loop {
+        let wait_tx = electrs_daemon.client.transaction_get(txid);
+        if wait_tx.is_ok() {
+            return true;
+        }
+       
+        if instant.elapsed() >= Duration::from_secs(wait_seconds) {
+            return false;
+        }
+    }
+}
+
+fn generate_blocks(bitcoin_daemon: &BitcoinD, num: usize) {
+    let address = bitcoin_daemon
+        .client
+        .get_new_address(Some("test"), Some(AddressType::Bech32))
+        .unwrap();
+    let _block_hashes = bitcoin_daemon
+        .client
+        .generate_to_address(num as u64, &address)
+        .unwrap();
+}
+
+fn premine(bitcoin_daemon: &BitcoinD, electrs_daemon: &ElectrsD, num_blocks: usize) {
+    generate_blocks_and_wait(bitcoin_daemon, electrs_daemon, num_blocks);
+}
+
+fn generate_blocks_and_wait(bitcoin_daemon: &BitcoinD, electrs_daemon: &ElectrsD, num: usize) {
+    let curr_height = bitcoin_daemon.client.get_block_count().unwrap();
+    generate_blocks(bitcoin_daemon, num);
+    wait_for_block(electrs_daemon, curr_height as usize + num);
+}
+
+
+fn reorg(num_blocks: usize, bitcoin_daemon: &BitcoinD) -> Result<()> {
+    let best_hash = bitcoin_daemon.client.get_best_block_hash()?;
+    let initial_height = bitcoin_daemon.client.get_block_info(&best_hash)?.height;
+
+    let mut to_invalidate = best_hash;
+    for i in 1..=num_blocks {
+        println!(
+            "Invalidating block {}/{} ({})",
+            i, num_blocks, to_invalidate
+        );
+
+        bitcoin_daemon.client.invalidate_block(&to_invalidate)?;
+        to_invalidate = bitcoin_daemon.client.get_best_block_hash()?;
+    }
+
+    println!(
+        "Invalidated {} blocks to new height of {}",
+        num_blocks,
+        initial_height - num_blocks as usize
+    );
+
+    Ok(())
+}
+
+
+fn setup_client(electrs_daemon: &ElectrsD) -> Client {
+    Client::new(electrs_daemon.electrum_url.as_str()).unwrap()
+}
+
+fn setup_bitcoind(conf: Option<bitcoind::Conf>) -> BitcoinD {
+    let bitcoind_exe = env::var("BITCOIND_EXE")
+        .ok()
+        .or_else(|| bitcoind::downloaded_exe_path().ok())
+        .expect(
+            "you need to provide an env var BITCOIND_EXE or specify a bitcoind version feature",
+        );
+    let conf = conf.unwrap_or_else(|| bitcoind::Conf::default());
+
+    BitcoinD::with_conf(bitcoind_exe, &conf).unwrap()
+}
+
+fn setup_electrsd(conf: Option<electrsd::Conf>, bitcoin_daemon: &BitcoinD) -> ElectrsD {
+    let electrs_exe = env::var("ELECTRS_EXE")
+        .ok()
+        .or_else(electrsd::downloaded_exe_path)
+        .expect("you need to provide env var ELECTRS_EXE or specify an electrsd version feature");
+        let mut conf = conf.unwrap_or_else(|| electrsd::Conf::default());
+        conf.http_enabled = true;
+        ElectrsD::with_conf(electrs_exe, &bitcoin_daemon, &conf).unwrap()
+}
+
+fn exponential_backoff_poll<T, F>(mut poll: F) -> T
+    where
+        F: FnMut() -> Option<T>,
+    {
+        let mut delay = Duration::from_millis(64);
+        loop {
+            match poll() {
+                Some(data) => break data,
+                None if delay.as_millis() < 512 => delay = delay.mul_f32(2.0),
+                None => {}
+            }
+    
+            std::thread::sleep(delay);
+        }
+}