Merge #1403: Update bdk_electrum crate to use sync/full-scan structs

b45897e feat(electrum): update docs and simplify logic of `ElectrumExt` (志宇)
92fb6cb chore(electrum): do not use `anyhow::Result` directly (志宇)
b2f3cac feat(electrum): include option for previous `TxOut`s for fee calculation (Wei Chen)
c0d7d60 feat(chain)!: use custom return types for `ElectrumExt` methods (志宇)
2945c6b fix(electrum): fixed `sync` functionality (Wei Chen)
9ed33c2 docs(electrum): fixed `full_scan`, `sync`, and crate documentation (Wei Chen)
b1f861b feat: update logging of electrum examples (志宇)
a6fdfb2 feat(electrum)!: use new sync/full-scan structs for `ElectrumExt` (志宇)
653e4fe feat(wallet): cache txs when constructing full-scan/sync requests (志宇)
58f27b3 feat(chain): introduce `TxCache` to `SyncRequest` and `FullScanRequest` (志宇)
721bb7f fix(chain): Make `Anchor` type in `FullScanResult` generic (志宇)
e3cfb84 feat(chain): `TxGraph::insert_tx` reuses `Arc` (志宇)
2ffb656 refactor(electrum): remove `RelevantTxids` and track txs in `TxGraph` (Wei Chen)

Pull request description:

  Fixes #1265
  Possibly fixes #1419

  ### Context

  Previous changes such as

  * Universal structures for full-scan/sync (PR #1413)
  * Making `CheckPoint` linked list query-able (PR #1369)
  * Making `Transaction`s cheaply-clonable (PR #1373)

  has allowed us to simplify the interaction between chain-source and receiving-structures (`bdk_chain`).

  The motivation is to accomplish something like this ([as mentioned here](#1153 (comment))):
  ```rust
  let things_I_am_interested_in = wallet.lock().unwrap().start_sync();
  let update = electrum_or_esplora.sync(things_i_am_interested_in)?;
  wallet.lock().unwrap().apply_update(update)?:
  ```

  ### Description

  This PR greatly simplifies the API of our Electrum chain-source (`bdk_electrum`) by making use of the aforementioned changes. Instead of referring back to the receiving `TxGraph` mid-sync/scan to determine which full transaction to fetch, we provide the Electrum chain-source already-fetched full transactions to start sync/scan (this is cheap, as transactions are wrapped in `Arc`s since #1373).

  In addition, an option has been added to include the previous `TxOut` for transactions received from an external wallet for fee calculation.

  ### Changelog notice

  * Change `TxGraph::insert_tx` to take in anything that satisfies `Into<Arc<Transaction>>`. This allows us to reuse the `Arc` pointer of what is being inserted.
  * Add `tx_cache` field to `SyncRequest` and `FullScanRequest`.
  * Make `Anchor` type in `FullScanResult` generic for more flexibility.
  * Change `ElectrumExt` methods to take in `SyncRequest`/`FullScanRequest` and return `SyncResult`/`FullScanResult`. Also update electrum examples accordingly.
  * Add `ElectrumResultExt` trait which allows us to convert the update `TxGraph` of `SyncResult`/`FullScanResult` for `bdk_electrum`.
  * Added an option for `full_scan` and `sync` to also fetch previous `TxOut`s to allow for fee calculation.

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo fmt` and `cargo clippy` before committing

  #### New Features:

  * [x] I've added tests for the new feature
  * [x] I've added docs for the new feature

ACKs for top commit:
  ValuedMammal:
    ACK b45897e
  notmandatory:
    ACK b45897e

Tree-SHA512: 1e274546015e7c7257965b36079ffe0cb3c2c0b7c2e0c322bcf32a06925a0c3e1119da1c8fd5318f1dbd82c2e952f6a07f227a9b023c48f506a62c93045d96d3

Author: notmandatory

crates/bdk/src/wallet/mod.rs

@@ -2565,6 +2565,7 @@ impl Wallet {
     /// start a blockchain sync with a spk based blockchain client.
     pub fn start_sync_with_revealed_spks(&self) -> SyncRequest {
         SyncRequest::from_chain_tip(self.chain.tip())
+            .cache_graph_txs(self.tx_graph())
             .populate_with_revealed_spks(&self.indexed_graph.index, ..)
     }
 
@@ -2578,6 +2579,7 @@ impl Wallet {
     /// in which the list of used scripts is not known.
     pub fn start_full_scan(&self) -> FullScanRequest<KeychainKind> {
         FullScanRequest::from_keychain_txout_index(self.chain.tip(), &self.indexed_graph.index)
+            .cache_graph_txs(self.tx_graph())
     }
 }
 

crates/chain/src/spk_client.rs

@@ -1,11 +1,18 @@
 //! Helper types for spk-based blockchain clients.
 
+use crate::{
+    collections::{BTreeMap, HashMap},
+    local_chain::CheckPoint,
+    ConfirmationTimeHeightAnchor, TxGraph,
+};
+use alloc::{boxed::Box, sync::Arc, vec::Vec};
+use bitcoin::{OutPoint, Script, ScriptBuf, Transaction, Txid};
 use core::{fmt::Debug, marker::PhantomData, ops::RangeBounds};
 
-use alloc::{boxed::Box, collections::BTreeMap, vec::Vec};
-use bitcoin::{OutPoint, Script, ScriptBuf, Txid};
-
-use crate::{local_chain::CheckPoint, ConfirmationTimeHeightAnchor, TxGraph};
+/// A cache of [`Arc`]-wrapped full transactions, identified by their [`Txid`]s.
+///
+/// This is used by the chain-source to avoid re-fetching full transactions.
+pub type TxCache = HashMap<Txid, Arc<Transaction>>;
 
 /// Data required to perform a spk-based blockchain client sync.
 ///
@@ -17,6 +24,8 @@ pub struct SyncRequest {
     ///
     /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip
     pub chain_tip: CheckPoint,
+    /// Cache of full transactions, so the chain-source can avoid re-fetching.
+    pub tx_cache: TxCache,
     /// Transactions that spend from or to these indexed script pubkeys.
     pub spks: Box<dyn ExactSizeIterator<Item = ScriptBuf> + Send>,
     /// Transactions with these txids.
@@ -30,12 +39,36 @@ impl SyncRequest {
     pub fn from_chain_tip(cp: CheckPoint) -> Self {
         Self {
             chain_tip: cp,
+            tx_cache: TxCache::new(),
             spks: Box::new(core::iter::empty()),
             txids: Box::new(core::iter::empty()),
             outpoints: Box::new(core::iter::empty()),
         }
     }
 
+    /// Add to the [`TxCache`] held by the request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn cache_txs<T>(mut self, full_txs: impl IntoIterator<Item = (Txid, T)>) -> Self
+    where
+        T: Into<Arc<Transaction>>,
+    {
+        self.tx_cache = full_txs
+            .into_iter()
+            .map(|(txid, tx)| (txid, tx.into()))
+            .collect();
+        self
+    }
+
+    /// Add all transactions from [`TxGraph`] into the [`TxCache`].
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn cache_graph_txs<A>(self, graph: &TxGraph<A>) -> Self {
+        self.cache_txs(graph.full_txs().map(|tx_node| (tx_node.txid, tx_node.tx)))
+    }
+
     /// Set the [`Script`]s that will be synced against.
     ///
     /// This consumes the [`SyncRequest`] and returns the updated one.
@@ -194,6 +227,8 @@ pub struct FullScanRequest<K> {
     ///
     /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip
     pub chain_tip: CheckPoint,
+    /// Cache of full transactions, so the chain-source can avoid re-fetching.
+    pub tx_cache: TxCache,
     /// Iterators of script pubkeys indexed by the keychain index.
     pub spks_by_keychain: BTreeMap<K, Box<dyn Iterator<Item = (u32, ScriptBuf)> + Send>>,
 }
@@ -204,10 +239,34 @@ impl<K: Ord + Clone> FullScanRequest<K> {
     pub fn from_chain_tip(chain_tip: CheckPoint) -> Self {
         Self {
             chain_tip,
+            tx_cache: TxCache::new(),
             spks_by_keychain: BTreeMap::new(),
         }
     }
 
+    /// Add to the [`TxCache`] held by the request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn cache_txs<T>(mut self, full_txs: impl IntoIterator<Item = (Txid, T)>) -> Self
+    where
+        T: Into<Arc<Transaction>>,
+    {
+        self.tx_cache = full_txs
+            .into_iter()
+            .map(|(txid, tx)| (txid, tx.into()))
+            .collect();
+        self
+    }
+
+    /// Add all transactions from [`TxGraph`] into the [`TxCache`].
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn cache_graph_txs<A>(self, graph: &TxGraph<A>) -> Self {
+        self.cache_txs(graph.full_txs().map(|tx_node| (tx_node.txid, tx_node.tx)))
+    }
+
     /// Construct a new [`FullScanRequest`] from a given `chain_tip` and `index`.
     ///
     /// Unbounded script pubkey iterators for each keychain (`K`) are extracted using
@@ -316,9 +375,9 @@ impl<K: Ord + Clone> FullScanRequest<K> {
 /// Data returned from a spk-based blockchain client full scan.
 ///
 /// See also [`FullScanRequest`].
-pub struct FullScanResult<K> {
+pub struct FullScanResult<K, A = ConfirmationTimeHeightAnchor> {
     /// The update to apply to the receiving [`LocalChain`](crate::local_chain::LocalChain).
-    pub graph_update: TxGraph<ConfirmationTimeHeightAnchor>,
+    pub graph_update: TxGraph<A>,
     /// The update to apply to the receiving [`TxGraph`].
     pub chain_update: CheckPoint,
     /// Last active indices for the corresponding keychains (`K`).

crates/chain/src/tx_graph.rs

@@ -516,12 +516,12 @@ impl<A: Clone + Ord> TxGraph<A> {
     /// Inserts the given transaction into [`TxGraph`].
     ///
     /// The [`ChangeSet`] returned will be empty if `tx` already exists.
-    pub fn insert_tx(&mut self, tx: Transaction) -> ChangeSet<A> {
+    pub fn insert_tx<T: Into<Arc<Transaction>>>(&mut self, tx: T) -> ChangeSet<A> {
+        let tx = tx.into();
         let mut update = Self::default();
-        update.txs.insert(
-            tx.txid(),
-            (TxNodeInternal::Whole(tx.into()), BTreeSet::new(), 0),
-        );
+        update
+            .txs
+            .insert(tx.txid(), (TxNodeInternal::Whole(tx), BTreeSet::new(), 0));
         self.apply_update(update)
     }
 

crates/electrum/Cargo.toml

@@ -12,7 +12,7 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.13.0", default-features = false }
+bdk_chain = { path = "../chain", version = "0.13.0" }
 electrum-client = { version = "0.19" }
 #rustls = { version = "=0.21.1", optional = true, features = ["dangerous_configuration"] }