From 254025837c681a9d88829a414d634d70c3eeaad5 Mon Sep 17 00:00:00 2001 From: Wei Chen Date: Thu, 11 Apr 2024 17:57:14 -0400 Subject: [PATCH 1/3] refactor(electrum): remove `RelevantTxids` and track txs in `TxGraph` This PR removes `RelevantTxids` from the electrum crate and tracks transactions in a `TxGraph`. This removes the need to separately construct a `TxGraph` after a `full_scan` or `sync`. --- crates/electrum/src/electrum_ext.rs | 244 ++++++++------------ crates/electrum/src/lib.rs | 11 +- crates/electrum/tests/test_electrum.rs | 39 +++- example-crates/example_electrum/src/main.rs | 37 +-- example-crates/wallet_electrum/src/main.rs | 13 +- 5 files changed, 159 insertions(+), 185 deletions(-) diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 5a6c5d116..7ad2ae270 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -1,122 +1,16 @@ use bdk_chain::{ - bitcoin::{OutPoint, ScriptBuf, Transaction, Txid}, + bitcoin::{OutPoint, ScriptBuf, Txid}, + collections::{HashMap, HashSet}, local_chain::CheckPoint, - tx_graph::{self, TxGraph}, + tx_graph::TxGraph, Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, }; -use electrum_client::{Client, ElectrumApi, Error, HeaderNotification}; -use std::{ - collections::{BTreeMap, BTreeSet, HashMap, HashSet}, - fmt::Debug, - str::FromStr, -}; +use electrum_client::{ElectrumApi, Error, HeaderNotification}; +use std::{collections::BTreeMap, fmt::Debug, str::FromStr}; /// We include a chain suffix of a certain length for the purpose of robustness. const CHAIN_SUFFIX_LENGTH: u32 = 8; -/// Represents updates fetched from an Electrum server, but excludes full transactions. -/// -/// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to -/// determine the full transactions missing from [`TxGraph`]. Then call [`Self::into_tx_graph`] to -/// fetch the full transactions from Electrum and finalize the update. -#[derive(Debug, Default, Clone)] -pub struct RelevantTxids(HashMap>); - -impl RelevantTxids { - /// Determine the full transactions that are missing from `graph`. - /// - /// Refer to [`RelevantTxids`] for more details. - pub fn missing_full_txs(&self, graph: &TxGraph) -> Vec { - self.0 - .keys() - .filter(move |&&txid| graph.as_ref().get_tx(txid).is_none()) - .cloned() - .collect() - } - - /// Finalizes the [`TxGraph`] update by fetching `missing` txids from the `client`. - /// - /// Refer to [`RelevantTxids`] for more details. - pub fn into_tx_graph( - self, - client: &Client, - missing: Vec, - ) -> Result, Error> { - let new_txs = client.batch_transaction_get(&missing)?; - let mut graph = TxGraph::::new(new_txs); - for (txid, anchors) in self.0 { - for anchor in anchors { - let _ = graph.insert_anchor(txid, anchor); - } - } - Ok(graph) - } - - /// Finalizes the update by fetching `missing` txids from the `client`, where the - /// resulting [`TxGraph`] has anchors of type [`ConfirmationTimeHeightAnchor`]. - /// - /// Refer to [`RelevantTxids`] for more details. - /// - /// **Note:** The confirmation time might not be precisely correct if there has been a reorg. - // Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to - // use it. - pub fn into_confirmation_time_tx_graph( - self, - client: &Client, - missing: Vec, - ) -> Result, Error> { - let graph = self.into_tx_graph(client, missing)?; - - let relevant_heights = { - let mut visited_heights = HashSet::new(); - graph - .all_anchors() - .iter() - .map(|(a, _)| a.confirmation_height_upper_bound()) - .filter(move |&h| visited_heights.insert(h)) - .collect::>() - }; - - let height_to_time = relevant_heights - .clone() - .into_iter() - .zip( - client - .batch_block_header(relevant_heights)? - .into_iter() - .map(|bh| bh.time as u64), - ) - .collect::>(); - - let graph_changeset = { - let old_changeset = TxGraph::default().apply_update(graph); - tx_graph::ChangeSet { - txs: old_changeset.txs, - txouts: old_changeset.txouts, - last_seen: old_changeset.last_seen, - anchors: old_changeset - .anchors - .into_iter() - .map(|(height_anchor, txid)| { - let confirmation_height = height_anchor.confirmation_height; - let confirmation_time = height_to_time[&confirmation_height]; - let time_anchor = ConfirmationTimeHeightAnchor { - anchor_block: height_anchor.anchor_block, - confirmation_height, - confirmation_time, - }; - (time_anchor, txid) - }) - .collect(), - } - }; - - let mut new_graph = TxGraph::default(); - new_graph.apply_changeset(graph_changeset); - Ok(new_graph) - } -} - /// Combination of chain and transactions updates from electrum /// /// We have to update the chain and the txids at the same time since we anchor the txids to @@ -125,25 +19,27 @@ impl RelevantTxids { pub struct ElectrumUpdate { /// Chain update pub chain_update: CheckPoint, - /// Transaction updates from electrum - pub relevant_txids: RelevantTxids, + /// Tracks electrum updates in TxGraph + pub graph_update: TxGraph, } -/// Trait to extend [`Client`] functionality. +/// Trait to extend [`electrum_client::Client`] functionality. pub trait ElectrumExt { /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and /// returns updates for [`bdk_chain`] data structures. /// /// - `prev_tip`: the most recent blockchain tip present locally /// - `keychain_spks`: keychains that we want to scan transactions for + /// - `full_txs`: [`TxGraph`] that contains all previously known transactions /// /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a /// single batch request. - fn full_scan( + fn full_scan( &self, prev_tip: CheckPoint, keychain_spks: BTreeMap>, + full_txs: Option<&TxGraph>, stop_gap: usize, batch_size: usize, ) -> Result<(ElectrumUpdate, BTreeMap), Error>; @@ -153,7 +49,8 @@ pub trait ElectrumExt { /// /// - `prev_tip`: the most recent blockchain tip present locally /// - `misc_spks`: an iterator of scripts we want to sync transactions for - /// - `txids`: transactions for which we want updated [`Anchor`]s + /// - `full_txs`: [`TxGraph`] that contains all previously known transactions + /// - `txids`: transactions for which we want updated [`bdk_chain::Anchor`]s /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we /// want to include in the update /// @@ -164,21 +61,23 @@ pub trait ElectrumExt { /// may include scripts that have been used, use [`full_scan`] with the keychain. /// /// [`full_scan`]: ElectrumExt::full_scan - fn sync( + fn sync( &self, prev_tip: CheckPoint, misc_spks: impl IntoIterator, + full_txs: Option<&TxGraph>, txids: impl IntoIterator, outpoints: impl IntoIterator, batch_size: usize, ) -> Result; } -impl ElectrumExt for A { - fn full_scan( +impl ElectrumExt for E { + fn full_scan( &self, prev_tip: CheckPoint, keychain_spks: BTreeMap>, + full_txs: Option<&TxGraph>, stop_gap: usize, batch_size: usize, ) -> Result<(ElectrumUpdate, BTreeMap), Error> { @@ -190,7 +89,14 @@ impl ElectrumExt for A { let (electrum_update, keychain_update) = loop { let (tip, _) = construct_update_tip(self, prev_tip.clone())?; - let mut relevant_txids = RelevantTxids::default(); + let mut tx_graph = TxGraph::::default(); + if let Some(txs) = full_txs { + let _ = + tx_graph.apply_update(txs.clone().map_anchors(|a| ConfirmationHeightAnchor { + anchor_block: a.anchor_block(), + confirmation_height: a.confirmation_height_upper_bound(), + })); + } let cps = tip .iter() .take(10) @@ -202,7 +108,7 @@ impl ElectrumExt for A { scanned_spks.append(&mut populate_with_spks( self, &cps, - &mut relevant_txids, + &mut tx_graph, &mut scanned_spks .iter() .map(|(i, (spk, _))| (i.clone(), spk.clone())), @@ -215,7 +121,7 @@ impl ElectrumExt for A { populate_with_spks( self, &cps, - &mut relevant_txids, + &mut tx_graph, keychain_spks, stop_gap, batch_size, @@ -234,6 +140,8 @@ impl ElectrumExt for A { let chain_update = tip; + let graph_update = into_confirmation_time_tx_graph(self, &tx_graph)?; + let keychain_update = request_spks .into_keys() .filter_map(|k| { @@ -248,7 +156,7 @@ impl ElectrumExt for A { break ( ElectrumUpdate { chain_update, - relevant_txids, + graph_update, }, keychain_update, ); @@ -257,10 +165,11 @@ impl ElectrumExt for A { Ok((electrum_update, keychain_update)) } - fn sync( + fn sync( &self, prev_tip: CheckPoint, misc_spks: impl IntoIterator, + full_txs: Option<&TxGraph>, txids: impl IntoIterator, outpoints: impl IntoIterator, batch_size: usize, @@ -273,6 +182,7 @@ impl ElectrumExt for A { let (mut electrum_update, _) = self.full_scan( prev_tip.clone(), [((), spk_iter)].into(), + full_txs, usize::MAX, batch_size, )?; @@ -284,10 +194,12 @@ impl ElectrumExt for A { .map(|cp| (cp.height(), cp)) .collect::>(); - populate_with_txids(self, &cps, &mut electrum_update.relevant_txids, txids)?; - - let _txs = - populate_with_outpoints(self, &cps, &mut electrum_update.relevant_txids, outpoints)?; + let mut tx_graph = TxGraph::::default(); + populate_with_txids(self, &cps, &mut tx_graph, txids)?; + populate_with_outpoints(self, &cps, &mut tx_graph, outpoints)?; + let _ = electrum_update + .graph_update + .apply_update(into_confirmation_time_tx_graph(self, &tx_graph)?); Ok(electrum_update) } @@ -411,10 +323,9 @@ fn determine_tx_anchor( fn populate_with_outpoints( client: &impl ElectrumApi, cps: &BTreeMap, - relevant_txids: &mut RelevantTxids, + tx_graph: &mut TxGraph, outpoints: impl IntoIterator, -) -> Result, Error> { - let mut full_txs = HashMap::new(); +) -> Result<(), Error> { for outpoint in outpoints { let txid = outpoint.txid; let tx = client.transaction_get(&txid)?; @@ -437,17 +348,19 @@ fn populate_with_outpoints( continue; } has_residing = true; - full_txs.insert(res.tx_hash, tx.clone()); + if tx_graph.get_tx(res.tx_hash).is_none() { + let _ = tx_graph.insert_tx(tx.clone()); + } } else { if has_spending { continue; } - let res_tx = match full_txs.get(&res.tx_hash) { + let res_tx = match tx_graph.get_tx(res.tx_hash) { Some(tx) => tx, None => { let res_tx = client.transaction_get(&res.tx_hash)?; - full_txs.insert(res.tx_hash, res_tx); - full_txs.get(&res.tx_hash).expect("just inserted") + let _ = tx_graph.insert_tx(res_tx); + tx_graph.get_tx(res.tx_hash).expect("just inserted") } }; has_spending = res_tx @@ -459,20 +372,18 @@ fn populate_with_outpoints( } }; - let anchor = determine_tx_anchor(cps, res.height, res.tx_hash); - let tx_entry = relevant_txids.0.entry(res.tx_hash).or_default(); - if let Some(anchor) = anchor { - tx_entry.insert(anchor); + if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { + let _ = tx_graph.insert_anchor(res.tx_hash, anchor); } } } - Ok(full_txs) + Ok(()) } fn populate_with_txids( client: &impl ElectrumApi, cps: &BTreeMap, - relevant_txids: &mut RelevantTxids, + tx_graph: &mut TxGraph, txids: impl IntoIterator, ) -> Result<(), Error> { for txid in txids { @@ -497,9 +408,11 @@ fn populate_with_txids( None => continue, }; - let tx_entry = relevant_txids.0.entry(txid).or_default(); + if tx_graph.get_tx(txid).is_none() { + let _ = tx_graph.insert_tx(tx); + } if let Some(anchor) = anchor { - tx_entry.insert(anchor); + let _ = tx_graph.insert_anchor(txid, anchor); } } Ok(()) @@ -508,7 +421,7 @@ fn populate_with_txids( fn populate_with_spks( client: &impl ElectrumApi, cps: &BTreeMap, - relevant_txids: &mut RelevantTxids, + tx_graph: &mut TxGraph, spks: &mut impl Iterator, stop_gap: usize, batch_size: usize, @@ -541,11 +454,50 @@ fn populate_with_spks( } for tx in spk_history { - let tx_entry = relevant_txids.0.entry(tx.tx_hash).or_default(); + let mut update = TxGraph::::default(); + + if tx_graph.get_tx(tx.tx_hash).is_none() { + let full_tx = client.transaction_get(&tx.tx_hash)?; + update = TxGraph::::new([full_tx]); + } + if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) { - tx_entry.insert(anchor); + let _ = update.insert_anchor(tx.tx_hash, anchor); } + + let _ = tx_graph.apply_update(update); } } } } + +fn into_confirmation_time_tx_graph( + client: &impl ElectrumApi, + tx_graph: &TxGraph, +) -> Result, Error> { + let relevant_heights = tx_graph + .all_anchors() + .iter() + .map(|(a, _)| a.confirmation_height) + .collect::>(); + + let height_to_time = relevant_heights + .clone() + .into_iter() + .zip( + client + .batch_block_header(relevant_heights)? + .into_iter() + .map(|bh| bh.time as u64), + ) + .collect::>(); + + let new_graph = tx_graph + .clone() + .map_anchors(|a| ConfirmationTimeHeightAnchor { + anchor_block: a.anchor_block, + confirmation_height: a.confirmation_height, + confirmation_time: height_to_time[&a.confirmation_height], + }); + Ok(new_graph) +} diff --git a/crates/electrum/src/lib.rs b/crates/electrum/src/lib.rs index 87c0e4618..f645653e4 100644 --- a/crates/electrum/src/lib.rs +++ b/crates/electrum/src/lib.rs @@ -7,19 +7,10 @@ //! keychain where the range of possibly used scripts is not known. In this case it is necessary to //! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a //! sync or full scan the user receives relevant blockchain data and output updates for -//! [`bdk_chain`] including [`RelevantTxids`]. -//! -//! The [`RelevantTxids`] only includes `txid`s and not full transactions. The caller is responsible -//! for obtaining full transactions before applying new data to their [`bdk_chain`]. This can be -//! done with these steps: -//! -//! 1. Determine which full transactions are missing. Use [`RelevantTxids::missing_full_txs`]. -//! -//! 2. Obtaining the full transactions. To do this via electrum use [`ElectrumApi::batch_transaction_get`]. +//! [`bdk_chain`] including [`bdk_chain::TxGraph`], which includes `txid`s and full transactions. //! //! Refer to [`example_electrum`] for a complete example. //! -//! [`ElectrumApi::batch_transaction_get`]: electrum_client::ElectrumApi::batch_transaction_get //! [`example_electrum`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_electrum #![warn(missing_docs)] diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index 8f77209fc..1653f0bcd 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -64,11 +64,16 @@ fn scan_detects_confirmed_tx() -> Result<()> { env.wait_until_electrum_sees_block()?; let ElectrumUpdate { chain_update, - relevant_txids, - } = client.sync(recv_chain.tip(), [spk_to_track], None, None, 5)?; + graph_update, + } = client.sync::( + recv_chain.tip(), + [spk_to_track], + Some(recv_graph.graph()), + None, + None, + 5, + )?; - let missing = relevant_txids.missing_full_txs(recv_graph.graph()); - let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let _ = recv_chain .apply_update(chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; @@ -130,11 +135,16 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { env.wait_until_electrum_sees_block()?; let ElectrumUpdate { chain_update, - relevant_txids, - } = client.sync(recv_chain.tip(), [spk_to_track.clone()], None, None, 5)?; + graph_update, + } = client.sync::( + recv_chain.tip(), + [spk_to_track.clone()], + Some(recv_graph.graph()), + None, + None, + 5, + )?; - let missing = relevant_txids.missing_full_txs(recv_graph.graph()); - let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let _ = recv_chain .apply_update(chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; @@ -160,11 +170,16 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { env.wait_until_electrum_sees_block()?; let ElectrumUpdate { chain_update, - relevant_txids, - } = client.sync(recv_chain.tip(), [spk_to_track.clone()], None, None, 5)?; + graph_update, + } = client.sync::( + recv_chain.tip(), + [spk_to_track.clone()], + Some(recv_graph.graph()), + None, + None, + 5, + )?; - let missing = relevant_txids.missing_full_txs(recv_graph.graph()); - let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let _ = recv_chain .apply_update(chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index e3b758e74..2439709c0 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -181,7 +181,13 @@ fn main() -> anyhow::Result<()> { }; client - .full_scan(tip, keychain_spks, stop_gap, scan_options.batch_size) + .full_scan::<_, ConfirmationHeightAnchor>( + tip, + keychain_spks, + Some(graph.lock().unwrap().graph()), + stop_gap, + scan_options.batch_size, + ) .context("scanning the blockchain")? } ElectrumCommands::Sync { @@ -274,14 +280,20 @@ fn main() -> anyhow::Result<()> { })); } - let tip = chain.tip(); + let electrum_update = client + .sync::( + chain.tip(), + spks, + Some(graph.graph()), + txids, + outpoints, + scan_options.batch_size, + ) + .context("scanning the blockchain")?; // drop lock on graph and chain drop((graph, chain)); - let electrum_update = client - .sync(tip, spks, txids, outpoints, scan_options.batch_size) - .context("scanning the blockchain")?; (electrum_update, BTreeMap::new()) } }; @@ -289,17 +301,11 @@ fn main() -> anyhow::Result<()> { let ( ElectrumUpdate { chain_update, - relevant_txids, + mut graph_update, }, keychain_update, ) = response; - let missing_txids = { - let graph = &*graph.lock().unwrap(); - relevant_txids.missing_full_txs(graph.graph()) - }; - - let mut graph_update = relevant_txids.into_tx_graph(&client, missing_txids)?; let now = std::time::UNIX_EPOCH .elapsed() .expect("must get time") @@ -320,7 +326,12 @@ fn main() -> anyhow::Result<()> { indexer, ..Default::default() }); - changeset.append(graph.apply_update(graph_update)); + changeset.append(graph.apply_update(graph_update.map_anchors(|a| { + ConfirmationHeightAnchor { + anchor_block: a.anchor_block, + confirmation_height: a.confirmation_height, + } + }))); changeset }; diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index 4af9e71de..76034ef54 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -7,6 +7,7 @@ use std::io::Write; use std::str::FromStr; use bdk::bitcoin::Address; +use bdk::chain::ConfirmationTimeHeightAnchor; use bdk::wallet::Update; use bdk::{bitcoin::Network, Wallet}; use bdk::{KeychainKind, SignOptions}; @@ -58,15 +59,19 @@ fn main() -> Result<(), anyhow::Error> { let ( ElectrumUpdate { chain_update, - relevant_txids, + mut graph_update, }, keychain_update, - ) = client.full_scan(prev_tip, keychain_spks, STOP_GAP, BATCH_SIZE)?; + ) = client.full_scan::<_, ConfirmationTimeHeightAnchor>( + prev_tip, + keychain_spks, + Some(wallet.as_ref()), + STOP_GAP, + BATCH_SIZE, + )?; println!(); - let missing = relevant_txids.missing_full_txs(wallet.as_ref()); - let mut graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); let _ = graph_update.update_last_seen_unconfirmed(now); From 4aa60ce26afcf60f06bf312a7a53447d3c4871c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 30 Apr 2024 14:49:03 +0800 Subject: [PATCH 2/3] feat(chain): `TxGraph::insert_tx` reuses `Arc` When we insert a transaction that is already wrapped in `Arc`, we should reuse the `Arc`. --- crates/chain/src/tx_graph.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/chain/src/tx_graph.rs b/crates/chain/src/tx_graph.rs index f6144e7a2..565ef5d0d 100644 --- a/crates/chain/src/tx_graph.rs +++ b/crates/chain/src/tx_graph.rs @@ -516,12 +516,12 @@ impl TxGraph { /// Inserts the given transaction into [`TxGraph`]. /// /// The [`ChangeSet`] returned will be empty if `tx` already exists. - pub fn insert_tx(&mut self, tx: Transaction) -> ChangeSet { + pub fn insert_tx>>(&mut self, tx: T) -> ChangeSet { + let tx = tx.into(); let mut update = Self::default(); - update.txs.insert( - tx.txid(), - (TxNodeInternal::Whole(tx.into()), BTreeSet::new(), 0), - ); + update + .txs + .insert(tx.txid(), (TxNodeInternal::Whole(tx), BTreeSet::new(), 0)); self.apply_update(update) } From 1df673c575437f6db35bc8e9a0f469a2bcf34674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 30 Apr 2024 14:50:21 +0800 Subject: [PATCH 3/3] feat(electrum)!: evan's fixes [WIP] --- crates/electrum/src/electrum_ext.rs | 194 ++++++++++++-------- crates/electrum/tests/test_electrum.rs | 70 +++---- example-crates/example_electrum/src/main.rs | 11 +- example-crates/wallet_electrum/src/main.rs | 27 +-- 4 files changed, 164 insertions(+), 138 deletions(-) diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 7ad2ae270..baef58c5e 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -1,26 +1,85 @@ use bdk_chain::{ - bitcoin::{OutPoint, ScriptBuf, Txid}, + bitcoin::{OutPoint, ScriptBuf, Transaction, Txid}, collections::{HashMap, HashSet}, local_chain::CheckPoint, tx_graph::TxGraph, - Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, + BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, }; use electrum_client::{ElectrumApi, Error, HeaderNotification}; -use std::{collections::BTreeMap, fmt::Debug, str::FromStr}; +use std::{collections::BTreeMap, fmt::Debug, str::FromStr, sync::Arc}; /// We include a chain suffix of a certain length for the purpose of robustness. const CHAIN_SUFFIX_LENGTH: u32 = 8; +/// Type that maintains a cache of [`Arc`]-wrapped transactions. +pub type TxCache = HashMap>; + /// Combination of chain and transactions updates from electrum /// /// We have to update the chain and the txids at the same time since we anchor the txids to /// the same chain tip that we check before and after we gather the txids. #[derive(Debug)] -pub struct ElectrumUpdate { +pub struct ElectrumUpdate { /// Chain update pub chain_update: CheckPoint, /// Tracks electrum updates in TxGraph - pub graph_update: TxGraph, + pub graph_update: TxGraph, +} + +impl ElectrumUpdate { + /// Transform the [`ElectrumUpdate`] to have [`Anchor`]s of another type. + /// + /// Refer to [`TxGraph::map_anchors`]. + pub fn map_anchors(self, f: F) -> ElectrumUpdate + where + F: FnMut(A) -> A2, + { + ElectrumUpdate { + chain_update: self.chain_update, + graph_update: self.graph_update.map_anchors(f), + } + } +} + +impl ElectrumUpdate { + /// Transforms the [`TxGraph`]'s [`Anchor`] type to [`ConfirmationTimeHeightAnchor`]. + pub fn into_confirmation_time_update( + self, + client: &impl ElectrumApi, + ) -> Result, Error> { + let relevant_heights = self + .graph_update + .all_anchors() + .iter() + .map(|(a, _)| a.confirmation_height) + .collect::>(); + + let height_to_time = relevant_heights + .clone() + .into_iter() + .zip( + client + .batch_block_header(relevant_heights)? + .into_iter() + .map(|bh| bh.time as u64), + ) + .collect::>(); + + let chain_update = self.chain_update; + let graph_update = + self.graph_update + .clone() + .map_anchors(|a| ConfirmationTimeHeightAnchor { + anchor_block: a.anchor_block, + confirmation_height: a.confirmation_height, + confirmation_time: height_to_time[&a.confirmation_height], + }); + + Ok(ElectrumUpdate { + chain_update, + graph_update, + }) + } } /// Trait to extend [`electrum_client::Client`] functionality. @@ -35,11 +94,11 @@ pub trait ElectrumExt { /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a /// single batch request. - fn full_scan( + fn full_scan( &self, + tx_cache: &mut TxCache, prev_tip: CheckPoint, keychain_spks: BTreeMap>, - full_txs: Option<&TxGraph>, stop_gap: usize, batch_size: usize, ) -> Result<(ElectrumUpdate, BTreeMap), Error>; @@ -61,11 +120,11 @@ pub trait ElectrumExt { /// may include scripts that have been used, use [`full_scan`] with the keychain. /// /// [`full_scan`]: ElectrumExt::full_scan - fn sync( + fn sync( &self, + tx_cache: &mut TxCache, prev_tip: CheckPoint, misc_spks: impl IntoIterator, - full_txs: Option<&TxGraph>, txids: impl IntoIterator, outpoints: impl IntoIterator, batch_size: usize, @@ -73,11 +132,11 @@ pub trait ElectrumExt { } impl ElectrumExt for E { - fn full_scan( + fn full_scan( &self, + tx_cache: &mut TxCache, prev_tip: CheckPoint, keychain_spks: BTreeMap>, - full_txs: Option<&TxGraph>, stop_gap: usize, batch_size: usize, ) -> Result<(ElectrumUpdate, BTreeMap), Error> { @@ -85,18 +144,18 @@ impl ElectrumExt for E { .into_iter() .map(|(k, s)| (k, s.into_iter())) .collect::>(); + + // We keep track of already-scanned spks just in case a reorg happens and we need to do a + // rescan. We need to keep track of this as iterators in `keychain_spks` are "unbounded" so + // cannot be collected. In addition, we keep track of whether an spk has an active tx + // history for determining the `last_active_index`. + // * key: (keychain, spk_index) that identifies the spk. + // * val: (script_pubkey, has_tx_history). let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new(); let (electrum_update, keychain_update) = loop { let (tip, _) = construct_update_tip(self, prev_tip.clone())?; - let mut tx_graph = TxGraph::::default(); - if let Some(txs) = full_txs { - let _ = - tx_graph.apply_update(txs.clone().map_anchors(|a| ConfirmationHeightAnchor { - anchor_block: a.anchor_block(), - confirmation_height: a.confirmation_height_upper_bound(), - })); - } + let mut graph_update = TxGraph::::default(); let cps = tip .iter() .take(10) @@ -108,7 +167,8 @@ impl ElectrumExt for E { scanned_spks.append(&mut populate_with_spks( self, &cps, - &mut tx_graph, + tx_cache, + &mut graph_update, &mut scanned_spks .iter() .map(|(i, (spk, _))| (i.clone(), spk.clone())), @@ -121,7 +181,8 @@ impl ElectrumExt for E { populate_with_spks( self, &cps, - &mut tx_graph, + tx_cache, + &mut graph_update, keychain_spks, stop_gap, batch_size, @@ -140,8 +201,6 @@ impl ElectrumExt for E { let chain_update = tip; - let graph_update = into_confirmation_time_tx_graph(self, &tx_graph)?; - let keychain_update = request_spks .into_keys() .filter_map(|k| { @@ -165,11 +224,11 @@ impl ElectrumExt for E { Ok((electrum_update, keychain_update)) } - fn sync( + fn sync( &self, + tx_cache: &mut TxCache, prev_tip: CheckPoint, misc_spks: impl IntoIterator, - full_txs: Option<&TxGraph>, txids: impl IntoIterator, outpoints: impl IntoIterator, batch_size: usize, @@ -179,10 +238,10 @@ impl ElectrumExt for E { .enumerate() .map(|(i, spk)| (i as u32, spk)); - let (mut electrum_update, _) = self.full_scan( + let (electrum_update, _) = self.full_scan( + tx_cache, prev_tip.clone(), [((), spk_iter)].into(), - full_txs, usize::MAX, batch_size, )?; @@ -195,11 +254,8 @@ impl ElectrumExt for E { .collect::>(); let mut tx_graph = TxGraph::::default(); - populate_with_txids(self, &cps, &mut tx_graph, txids)?; + populate_with_txids(self, &cps, tx_cache, &mut tx_graph, txids)?; populate_with_outpoints(self, &cps, &mut tx_graph, outpoints)?; - let _ = electrum_update - .graph_update - .apply_update(into_confirmation_time_tx_graph(self, &tx_graph)?); Ok(electrum_update) } @@ -383,11 +439,12 @@ fn populate_with_outpoints( fn populate_with_txids( client: &impl ElectrumApi, cps: &BTreeMap, - tx_graph: &mut TxGraph, + tx_cache: &mut TxCache, + graph_update: &mut TxGraph, txids: impl IntoIterator, ) -> Result<(), Error> { for txid in txids { - let tx = match client.transaction_get(&txid) { + let tx = match fetch_tx(client, tx_cache, txid) { Ok(tx) => tx, Err(electrum_client::Error::Protocol(_)) => continue, Err(other_err) => return Err(other_err), @@ -408,20 +465,36 @@ fn populate_with_txids( None => continue, }; - if tx_graph.get_tx(txid).is_none() { - let _ = tx_graph.insert_tx(tx); + if graph_update.get_tx(txid).is_none() { + // TODO: We need to be able to insert an `Arc` of a transaction. + let _ = graph_update.insert_tx(tx); } if let Some(anchor) = anchor { - let _ = tx_graph.insert_anchor(txid, anchor); + let _ = graph_update.insert_anchor(txid, anchor); } } Ok(()) } +fn fetch_tx( + client: &C, + tx_cache: &mut TxCache, + txid: Txid, +) -> Result, Error> { + use std::collections::hash_map::Entry; + Ok(match tx_cache.entry(txid) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => entry + .insert(Arc::new(client.transaction_get(&txid)?)) + .clone(), + }) +} + fn populate_with_spks( client: &impl ElectrumApi, cps: &BTreeMap, - tx_graph: &mut TxGraph, + tx_cache: &mut TxCache, + graph_update: &mut TxGraph, spks: &mut impl Iterator, stop_gap: usize, batch_size: usize, @@ -453,51 +526,12 @@ fn populate_with_spks( unused_spk_count = 0; } - for tx in spk_history { - let mut update = TxGraph::::default(); - - if tx_graph.get_tx(tx.tx_hash).is_none() { - let full_tx = client.transaction_get(&tx.tx_hash)?; - update = TxGraph::::new([full_tx]); + for tx_res in spk_history { + let _ = graph_update.insert_tx(fetch_tx(client, tx_cache, tx_res.tx_hash)?); + if let Some(anchor) = determine_tx_anchor(cps, tx_res.height, tx_res.tx_hash) { + let _ = graph_update.insert_anchor(tx_res.tx_hash, anchor); } - - if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) { - let _ = update.insert_anchor(tx.tx_hash, anchor); - } - - let _ = tx_graph.apply_update(update); } } } } - -fn into_confirmation_time_tx_graph( - client: &impl ElectrumApi, - tx_graph: &TxGraph, -) -> Result, Error> { - let relevant_heights = tx_graph - .all_anchors() - .iter() - .map(|(a, _)| a.confirmation_height) - .collect::>(); - - let height_to_time = relevant_heights - .clone() - .into_iter() - .zip( - client - .batch_block_header(relevant_heights)? - .into_iter() - .map(|bh| bh.time as u64), - ) - .collect::>(); - - let new_graph = tx_graph - .clone() - .map_anchors(|a| ConfirmationTimeHeightAnchor { - anchor_block: a.anchor_block, - confirmation_height: a.confirmation_height, - confirmation_time: height_to_time[&a.confirmation_height], - }); - Ok(new_graph) -} diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index 1653f0bcd..ecd5de358 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -62,22 +62,21 @@ fn scan_detects_confirmed_tx() -> Result<()> { // Sync up to tip. env.wait_until_electrum_sees_block()?; - let ElectrumUpdate { - chain_update, - graph_update, - } = client.sync::( - recv_chain.tip(), - [spk_to_track], - Some(recv_graph.graph()), - None, - None, - 5, - )?; + let update = client + .sync( + &mut Default::default(), + recv_chain.tip(), + [spk_to_track], + None, + None, + 5, + )? + .into_confirmation_time_update(&client)?; let _ = recv_chain - .apply_update(chain_update) + .apply_update(update.chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; - let _ = recv_graph.apply_update(graph_update); + let _ = recv_graph.apply_update(update.graph_update); // Check to see if tx is confirmed. assert_eq!( @@ -133,25 +132,24 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { // Sync up to tip. env.wait_until_electrum_sees_block()?; - let ElectrumUpdate { - chain_update, - graph_update, - } = client.sync::( - recv_chain.tip(), - [spk_to_track.clone()], - Some(recv_graph.graph()), - None, - None, - 5, - )?; + let update = client + .sync( + &mut Default::default(), + recv_chain.tip(), + [spk_to_track.clone()], + None, + None, + 5, + )? + .into_confirmation_time_update(&client)?; let _ = recv_chain - .apply_update(chain_update) + .apply_update(update.chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; - let _ = recv_graph.apply_update(graph_update.clone()); + let _ = recv_graph.apply_update(update.graph_update.clone()); // Retain a snapshot of all anchors before reorg process. - let initial_anchors = graph_update.all_anchors(); + let initial_anchors = update.graph_update.all_anchors(); // Check if initial balance is correct. assert_eq!( @@ -171,14 +169,16 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { let ElectrumUpdate { chain_update, graph_update, - } = client.sync::( - recv_chain.tip(), - [spk_to_track.clone()], - Some(recv_graph.graph()), - None, - None, - 5, - )?; + } = client + .sync( + &mut Default::default(), + recv_chain.tip(), + [spk_to_track.clone()], + None, + None, + 5, + )? + .into_confirmation_time_update(&client)?; let _ = recv_chain .apply_update(chain_update) diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index 2439709c0..4a47daea2 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -13,7 +13,7 @@ use bdk_chain::{ }; use bdk_electrum::{ electrum_client::{self, Client, ElectrumApi}, - ElectrumExt, ElectrumUpdate, + ElectrumExt, ElectrumUpdate, TxCache, }; use example_cli::{ anyhow::{self, Context}, @@ -146,6 +146,7 @@ fn main() -> anyhow::Result<()> { }; let client = electrum_cmd.electrum_args().client(args.network)?; + let mut tx_cache = TxCache::new(); let response = match electrum_cmd.clone() { ElectrumCommands::Scan { @@ -181,10 +182,10 @@ fn main() -> anyhow::Result<()> { }; client - .full_scan::<_, ConfirmationHeightAnchor>( + .full_scan::<_>( + &mut tx_cache, tip, keychain_spks, - Some(graph.lock().unwrap().graph()), stop_gap, scan_options.batch_size, ) @@ -281,10 +282,10 @@ fn main() -> anyhow::Result<()> { } let electrum_update = client - .sync::( + .sync( + &mut tx_cache, chain.tip(), spks, - Some(graph.graph()), txids, outpoints, scan_options.batch_size, diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index 76034ef54..848dbe578 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -7,13 +7,13 @@ use std::io::Write; use std::str::FromStr; use bdk::bitcoin::Address; -use bdk::chain::ConfirmationTimeHeightAnchor; use bdk::wallet::Update; use bdk::{bitcoin::Network, Wallet}; use bdk::{KeychainKind, SignOptions}; +use bdk_electrum::TxCache; use bdk_electrum::{ electrum_client::{self, ElectrumApi}, - ElectrumExt, ElectrumUpdate, + ElectrumExt, }; use bdk_file_store::Store; @@ -38,6 +38,7 @@ fn main() -> Result<(), anyhow::Error> { print!("Syncing..."); let client = electrum_client::Client::new("ssl://electrum.blockstream.info:60002")?; + let mut tx_cache = TxCache::new(); let prev_tip = wallet.latest_checkpoint(); let keychain_spks = wallet @@ -56,29 +57,19 @@ fn main() -> Result<(), anyhow::Error> { }) .collect(); - let ( - ElectrumUpdate { - chain_update, - mut graph_update, - }, - keychain_update, - ) = client.full_scan::<_, ConfirmationTimeHeightAnchor>( - prev_tip, - keychain_spks, - Some(wallet.as_ref()), - STOP_GAP, - BATCH_SIZE, - )?; + let (update, keychain_update) = + client.full_scan(&mut tx_cache, prev_tip, keychain_spks, STOP_GAP, BATCH_SIZE)?; + let mut update = update.into_confirmation_time_update(&client)?; println!(); let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); - let _ = graph_update.update_last_seen_unconfirmed(now); + let _ = update.graph_update.update_last_seen_unconfirmed(now); let wallet_update = Update { last_active_indices: keychain_update, - graph: graph_update, - chain: Some(chain_update), + graph: update.graph_update, + chain: Some(update.chain_update), }; wallet.apply_update(wallet_update)?; wallet.commit()?;