Merge #247: deps: Update bdk_chain to 0.23.0

6aba330 fix: gracefully handle `reveal_to_target` (valued mammal)
741f2ae test: persist indexed script pubkeys (valued mammal)
db25555 deps!: Update `bdk_chain` to 0.23.0 (valued mammal)

Pull request description:

  ### Description

  The PR updates bdk_chain to 0.23.0. Additionally we introduce the ability to persist derived SPKs indexed by descriptor ID and derivation index.

  The `CreateParams::use_spk_cache` method enables or disables a persistent cache for script pubkeys (SPKs) derived by the wallet. When enabled, the wallet will store and reuse previously derived SPKs, avoiding redundant derivation on subsequent loads.

  When a wallet has many revealed addresses (i.e., many derived SPKs), loading the wallet can become slow if every address must be re-derived from scratch.

  By enabling the SPK cache:

  - **On wallet creation:** The wallet will persistently store each derived SPK as addresses are revealed.
  - **On wallet load:** If `use_spk_cache` is also set in the corresponding `LoadParams`, the wallet will load the cached SPKs directly from storage, skipping the need to re-derive them from the descriptor. This can dramatically reduce load times for wallets with hundreds or thousands of revealed addresses, as the expensive crypto operations are avoided.

  **Caveat:**
  Both creation and loading must have `use_spk_cache(true)` set for the cache to be used. If not, the wallet will fall back to deriving SPKs as usual.

  fixes #246
  fixes #237

  ### Changelog notice

  #### Added

  - `CreateParams::use_spk_cache`
  - `LoadParams::use_spk_cache`

  #### Changed

  - bump bdk_chain to 0.23.0
  - **Note:** This change extends the wallet `ChangeSet` type by adding `first_seen` to the tx_graph member, and adding `spk_cache` to the indexer.

  ### Checklists

  #### All Submissions:

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo +nightly fmt` and `cargo clippy` before committing
  * [x] I've added tests for the new feature
  * [x] I've added docs for the new feature
  * [x] This pull request breaks the existing API
  * [x] I'm linking the issue being fixed by this PR

ACKs for top commit:
  notmandatory:
    ACK 6aba330

Tree-SHA512: dd4d657e61836d12da388f233dcf463638da457fd1c982d13ee469dfdf8df77f3351a6edaa300156755217427da8296db03d2161c7baf3cb71cf5626b355680a

Author: ValuedMammal

examples/example_wallet_electrum/Cargo.toml

@@ -5,5 +5,5 @@ edition = "2021"
 
 [dependencies]
 bdk_wallet = { path = "../../wallet", features = ["file_store"] }
-bdk_electrum = { version = "0.22.0" }
+bdk_electrum = { version = "0.23.0" }
 anyhow = "1"

examples/example_wallet_esplora_async/Cargo.toml

@@ -7,6 +7,6 @@ edition = "2021"
 
 [dependencies]
 bdk_wallet = { path = "../../wallet", features = ["rusqlite"] }
-bdk_esplora = { version = "0.21.0", features = ["async-https", "tokio"] }
+bdk_esplora = { version = "0.22.0", features = ["async-https", "tokio"] }
 tokio = { version = "1.38.1", features = ["rt", "rt-multi-thread", "macros"] }
 anyhow = "1"

examples/example_wallet_esplora_blocking/Cargo.toml

@@ -8,5 +8,5 @@ publish = false
 
 [dependencies]
 bdk_wallet = { path = "../../wallet", features = ["file_store"] }
-bdk_esplora = { version = "0.21.0", features = ["blocking"] }
+bdk_esplora = { version = "0.22.0", features = ["blocking"] }
 anyhow = "1"

examples/example_wallet_rpc/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 bdk_wallet = { path = "../../wallet", features = ["file_store"] }
-bdk_bitcoind_rpc = { version = "0.19.0" }
+bdk_bitcoind_rpc = { version = "0.20.0" }
 
 anyhow = "1"
 clap = { version = "4.5.17", features = ["derive", "env"] }

wallet/Cargo.toml

@@ -21,11 +21,11 @@ miniscript = { version = "12.3.1", features = [ "serde" ], default-features = fa
 bitcoin = { version = "0.32.4", features = [ "serde", "base64" ], default-features = false }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
-bdk_chain = { version = "0.22.0", features = [ "miniscript", "serde" ], default-features = false }
+bdk_chain = { version = "0.23.0", features = [ "miniscript", "serde" ], default-features = false }
 
 # Optional dependencies
 bip39 = { version = "2.0", optional = true }
-bdk_file_store = { version = "0.20.0", optional = true }
+bdk_file_store = { version = "0.21.0", optional = true }
 
 [features]
 default = ["std"]
@@ -40,7 +40,7 @@ test-utils = ["std"]
 [dev-dependencies]
 assert_matches = "1.5.0"
 tempfile = "3"
-bdk_chain = { version = "0.22.0", features = ["rusqlite"] }
+bdk_chain = { version = "0.23.0", features = ["rusqlite"] }
 bdk_wallet = { path = ".", features = ["rusqlite", "file_store", "test-utils"] }
 anyhow = "1"
 rand = "^0.8"

wallet/src/wallet/coin_selection.rs

@@ -750,6 +750,7 @@ mod test {
             value,
             index,
             ChainPosition::Unconfirmed {
+                first_seen: Some(last_seen),
                 last_seen: Some(last_seen),
             },
         )
@@ -850,7 +851,10 @@ mod test {
                             transitively: None,
                         }
                     } else {
-                        ChainPosition::Unconfirmed { last_seen: Some(0) }
+                        ChainPosition::Unconfirmed {
+                            first_seen: Some(1),
+                            last_seen: Some(1),
+                        }
                     },
                 }),
             });
@@ -875,7 +879,10 @@ mod test {
                     keychain: KeychainKind::External,
                     is_spent: false,
                     derivation_index: 42,
-                    chain_position: ChainPosition::Unconfirmed { last_seen: Some(0) },
+                    chain_position: ChainPosition::Unconfirmed {
+                        first_seen: Some(1),
+                        last_seen: Some(1),
+                    },
                 }),
             })
             .collect()
@@ -1231,7 +1238,10 @@ mod test {
         optional.push(utxo(
             Amount::from_sat(500_000),
             3,
-            ChainPosition::<ConfirmationBlockTime>::Unconfirmed { last_seen: Some(0) },
+            ChainPosition::<ConfirmationBlockTime>::Unconfirmed {
+                first_seen: Some(1),
+                last_seen: Some(1),
+            },
         ));
 
         // Defensive assertions, for sanity and in case someone changes the test utxos vector.

wallet/src/wallet/mod.rs

@@ -438,7 +438,12 @@ impl Wallet {
             None => (None, Arc::new(SignersContainer::new())),
         };
 
-        let index = create_indexer(descriptor, change_descriptor, params.lookahead)?;
+        let index = create_indexer(
+            descriptor,
+            change_descriptor,
+            params.lookahead,
+            params.use_spk_cache,
+        )?;
 
         let descriptor = index.get_descriptor(KeychainKind::External).cloned();
         let change_descriptor = index.get_descriptor(KeychainKind::Internal).cloned();
@@ -637,8 +642,13 @@ impl Wallet {
             None => Arc::new(SignersContainer::new()),
         };
 
-        let index = create_indexer(descriptor, change_descriptor, params.lookahead)
-            .map_err(LoadError::Descriptor)?;
+        let index = create_indexer(
+            descriptor,
+            change_descriptor,
+            params.lookahead,
+            params.use_spk_cache,
+        )
+        .map_err(LoadError::Descriptor)?;
 
         let mut indexed_graph = IndexedTxGraph::new(index);
         indexed_graph.apply_changeset(changeset.indexer.into());
@@ -1118,9 +1128,9 @@ impl Wallet {
     ///         "tx is an ancestor of a tx anchored in {}:{}",
     ///         anchor.block_id.height, anchor.block_id.hash,
     ///     ),
-    ///     ChainPosition::Unconfirmed { last_seen } => println!(
-    ///         "tx is last seen at {:?}, it is unconfirmed as it is not anchored in the best chain",
-    ///         last_seen,
+    ///     ChainPosition::Unconfirmed { first_seen, last_seen } => println!(
+    ///         "tx is first seen at {:?}, last seen at {:?}, it is unconfirmed as it is not anchored in the best chain",
+    ///         first_seen, last_seen
     ///     ),
     /// }
     /// ```
@@ -1611,13 +1621,12 @@ impl Wallet {
 
         // recording changes to the change keychain
         if let (Excess::Change { .. }, Some((keychain, index))) = (excess, drain_index) {
-            let (_, index_changeset) = self
-                .indexed_graph
-                .index
-                .reveal_to_target(keychain, index)
-                .expect("must not be None");
-            self.stage.merge(index_changeset.into());
-            self.mark_used(keychain, index);
+            if let Some((_, index_changeset)) =
+                self.indexed_graph.index.reveal_to_target(keychain, index)
+            {
+                self.stage.merge(index_changeset.into());
+                self.mark_used(keychain, index);
+            }
         }
 
         Ok(psbt)
@@ -2619,17 +2628,14 @@ fn create_indexer(
     descriptor: ExtendedDescriptor,
     change_descriptor: Option<ExtendedDescriptor>,
     lookahead: u32,
+    use_spk_cache: bool,
 ) -> Result<KeychainTxOutIndex<KeychainKind>, DescriptorError> {
-    let mut indexer = KeychainTxOutIndex::<KeychainKind>::new(lookahead);
+    let mut indexer = KeychainTxOutIndex::<KeychainKind>::new(lookahead, use_spk_cache);
 
-    // let (descriptor, keymap) = descriptor;
-    // let signers = Arc::new(SignersContainer::build(keymap, &descriptor, secp));
     assert!(indexer
         .insert_descriptor(KeychainKind::External, descriptor)
         .expect("first descriptor introduced must succeed"));
 
-    // let (descriptor, keymap) = change_descriptor;
-    // let change_signers = Arc::new(SignersContainer::build(keymap, &descriptor, secp));
     if let Some(change_descriptor) = change_descriptor {
         assert!(indexer
             .insert_descriptor(KeychainKind::Internal, change_descriptor)

wallet/src/wallet/params.rs

@@ -39,6 +39,7 @@ pub struct CreateParams {
     pub(crate) network: Network,
     pub(crate) genesis_hash: Option<BlockHash>,
     pub(crate) lookahead: u32,
+    pub(crate) use_spk_cache: bool,
 }
 
 impl CreateParams {
@@ -61,6 +62,7 @@ impl CreateParams {
             network: Network::Bitcoin,
             genesis_hash: None,
             lookahead: DEFAULT_LOOKAHEAD,
+            use_spk_cache: false,
         }
     }
 
@@ -82,6 +84,7 @@ impl CreateParams {
             network: Network::Bitcoin,
             genesis_hash: None,
             lookahead: DEFAULT_LOOKAHEAD,
+            use_spk_cache: false,
         }
     }
 
@@ -118,6 +121,15 @@ impl CreateParams {
         self
     }
 
+    /// Use a persistent cache of indexed script pubkeys (SPKs).
+    ///
+    /// **Note:** To persist across restarts, this option must also be set at load time with
+    /// [`LoadParams`](LoadParams::use_spk_cache).
+    pub fn use_spk_cache(mut self, use_spk_cache: bool) -> Self {
+        self.use_spk_cache = use_spk_cache;
+        self
+    }
+
     /// Create [`PersistedWallet`] with the given [`WalletPersister`].
     pub fn create_wallet<P>(
         self,
@@ -157,6 +169,7 @@ pub struct LoadParams {
     pub(crate) check_descriptor: Option<Option<DescriptorToExtract>>,
     pub(crate) check_change_descriptor: Option<Option<DescriptorToExtract>>,
     pub(crate) extract_keys: bool,
+    pub(crate) use_spk_cache: bool,
 }
 
 impl LoadParams {
@@ -173,6 +186,7 @@ impl LoadParams {
             check_descriptor: None,
             check_change_descriptor: None,
             extract_keys: false,
+            use_spk_cache: false,
         }
     }
 
@@ -234,6 +248,15 @@ impl LoadParams {
         self
     }
 
+    /// Use a persistent cache of indexed script pubkeys (SPKs).
+    ///
+    /// **Note:** This should only be used if you have previously persisted a cache of script
+    /// pubkeys using [`CreateParams::use_spk_cache`].
+    pub fn use_spk_cache(mut self, use_spk_cache: bool) -> Self {
+        self.use_spk_cache = use_spk_cache;
+        self
+    }
+
     /// Load [`PersistedWallet`] with the given [`WalletPersister`].
     pub fn load_wallet<P>(
         self,

wallet/src/wallet/tx_builder.rs

@@ -1024,7 +1024,10 @@ mod test {
                 txout: TxOut::NULL,
                 keychain: KeychainKind::External,
                 is_spent: false,
-                chain_position: chain::ChainPosition::Unconfirmed { last_seen: Some(0) },
+                chain_position: chain::ChainPosition::Unconfirmed {
+                    first_seen: Some(1),
+                    last_seen: Some(1),
+                },
                 derivation_index: 0,
             },
             LocalOutput {

wallet/tests/wallet.rs

@@ -1,10 +1,14 @@
+use std::collections::BTreeMap;
 use std::path::Path;
 use std::str::FromStr;
 use std::sync::Arc;
 
 use anyhow::Context;
 use assert_matches::assert_matches;
-use bdk_chain::{BlockId, CanonicalizationParams, ChainPosition, ConfirmationBlockTime};
+use bdk_chain::{
+    keychain_txout::DEFAULT_LOOKAHEAD, BlockId, CanonicalizationParams, ChainPosition,
+    ConfirmationBlockTime, DescriptorExt,
+};
 use bdk_wallet::coin_selection::{self, LargestFirstCoinSelection};
 use bdk_wallet::descriptor::{calc_checksum, DescriptorError, IntoWalletDescriptor};
 use bdk_wallet::error::CreateTxError;
@@ -70,6 +74,7 @@ fn wallet_is_persisted() -> anyhow::Result<()> {
             let mut db = create_db(&file_path)?;
             let mut wallet = Wallet::create(external_desc, internal_desc)
                 .network(Network::Testnet)
+                .use_spk_cache(true)
                 .create_wallet(&mut db)?;
             wallet.reveal_next_address(KeychainKind::External);
 
@@ -79,13 +84,6 @@ fn wallet_is_persisted() -> anyhow::Result<()> {
         };
 
         // recover wallet
-        {
-            let mut db = open_db(&file_path).context("failed to recover db")?;
-            let _ = Wallet::load()
-                .check_network(Network::Testnet)
-                .load_wallet(&mut db)?
-                .expect("wallet must exist");
-        }
         {
             let mut db = open_db(&file_path).context("failed to recover db")?;
             let wallet = Wallet::load()
@@ -113,6 +111,67 @@ fn wallet_is_persisted() -> anyhow::Result<()> {
                     .0
             );
         }
+        // Test SPK cache
+        {
+            let mut db = open_db(&file_path).context("failed to recover db")?;
+            let mut wallet = Wallet::load()
+                .check_network(Network::Testnet)
+                .use_spk_cache(true)
+                .load_wallet(&mut db)?
+                .expect("wallet must exist");
+
+            let external_did = wallet
+                .public_descriptor(KeychainKind::External)
+                .descriptor_id();
+            let internal_did = wallet
+                .public_descriptor(KeychainKind::Internal)
+                .descriptor_id();
+
+            assert!(wallet.staged().is_none());
+
+            let _addr = wallet.reveal_next_address(KeychainKind::External);
+            let cs = wallet.staged().expect("we should have staged a changeset");
+            assert!(!cs.indexer.spk_cache.is_empty(), "failed to cache spks");
+            assert_eq!(cs.indexer.spk_cache.len(), 2, "we persisted two keychains");
+            let spk_cache: &BTreeMap<u32, ScriptBuf> =
+                cs.indexer.spk_cache.get(&external_did).unwrap();
+            assert_eq!(spk_cache.len() as u32, 1 + 1 + DEFAULT_LOOKAHEAD);
+            assert_eq!(spk_cache.keys().last(), Some(&26));
+            let spk_cache = cs.indexer.spk_cache.get(&internal_did).unwrap();
+            assert_eq!(spk_cache.len() as u32, DEFAULT_LOOKAHEAD);
+            assert_eq!(spk_cache.keys().last(), Some(&24));
+            // Clear the stage
+            let _ = wallet.take_staged();
+            let _addr = wallet.reveal_next_address(KeychainKind::Internal);
+            let cs = wallet.staged().unwrap();
+            assert_eq!(cs.indexer.spk_cache.len(), 1);
+            let spk_cache = cs.indexer.spk_cache.get(&internal_did).unwrap();
+            assert_eq!(spk_cache.len(), 1);
+            assert_eq!(spk_cache.keys().next(), Some(&25));
+        }
+        // SPK cache requires load params
+        {
+            let mut db = open_db(&file_path).context("failed to recover db")?;
+            let mut wallet = Wallet::load()
+                .check_network(Network::Testnet)
+                // .use_spk_cache(false)
+                .load_wallet(&mut db)?
+                .expect("wallet must exist");
+
+            let internal_did = wallet
+                .public_descriptor(KeychainKind::Internal)
+                .descriptor_id();
+
+            assert!(wallet.staged().is_none());
+
+            let _addr = wallet.reveal_next_address(KeychainKind::Internal);
+            let cs = wallet.staged().expect("we should have staged a changeset");
+            assert_eq!(cs.indexer.last_revealed.get(&internal_did), Some(&0));
+            assert!(
+                cs.indexer.spk_cache.is_empty(),
+                "we didn't set `use_spk_cache`"
+            );
+        }
 
         Ok(())
     }
@@ -4406,7 +4465,7 @@ fn test_taproot_load_descriptor_duplicated_keys() {
 #[test]
 #[cfg(debug_assertions)]
 #[should_panic(
-    expected = "replenish lookahead: must not have existing spk: keychain=Internal, lookahead=25, next_store_index=0, next_reveal_index=0"
+    expected = "replenish lookahead: must not have existing spk: keychain=Internal, lookahead=25, next_index=0"
 )]
 fn test_keychains_with_overlapping_spks() {
     // this can happen if a non-wildcard descriptor keychain derives an spk that a