fix(esplora)!: Change sync, full_scan to take a timestamp parameter

This is used for setting the time a transaction was last
seen in mempool. When doing a sync or full scan, the caller
should specify the current time, for example as a UNIX
timestamp.

Author: ValuedMammal

crates/esplora/src/async_ext.rs

@@ -51,7 +51,8 @@ pub trait EsploraAsyncExt {
     ///
     /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
     /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in
-    /// parallel.
+    /// parallel. `time` is the current time, typically a UNIX timestamp, used only when setting
+    /// the time a transaction was last seen unconfirmed.
     async fn full_scan<K: Ord + Clone + Send>(
         &self,
         keychain_spks: BTreeMap<
@@ -60,6 +61,7 @@ pub trait EsploraAsyncExt {
         >,
         stop_gap: usize,
         parallel_requests: usize,
+        time: Option<u64>,
     ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error>;
 
     /// Sync a set of scripts with the blockchain (via an Esplora client) for the data
@@ -69,6 +71,7 @@ pub trait EsploraAsyncExt {
     /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s
     /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we
     ///     want to include in the update
+    /// * `time`: UNIX timestamp used to set the time a transaction was last seen unconfirmed
     ///
     /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
     /// may include scripts that have been used, use [`full_scan`] with the keychain.
@@ -80,6 +83,7 @@ pub trait EsploraAsyncExt {
         txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
         outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
         parallel_requests: usize,
+        time: Option<u64>,
     ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error>;
 }
 
@@ -157,6 +161,7 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
         >,
         stop_gap: usize,
         parallel_requests: usize,
+        time: Option<u64>,
     ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error> {
         type TxsOfSpkIndex = (u32, Vec<esplora_client::Tx>);
         let parallel_requests = Ord::max(parallel_requests, 1);
@@ -204,6 +209,11 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
                         if let Some(anchor) = anchor_from_status(&tx.status) {
                             let _ = graph.insert_anchor(tx.txid, anchor);
                         }
+                        if !tx.status.confirmed {
+                            if let Some(seen_at) = time {
+                                let _ = graph.insert_seen_at(tx.txid, seen_at);
+                            }
+                        }
 
                         let previous_outputs = tx.vin.iter().filter_map(|vin| {
                             let prevout = vin.prevout.as_ref()?;
@@ -250,6 +260,7 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
         txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
         outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
         parallel_requests: usize,
+        time: Option<u64>,
     ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
         let mut graph = self
             .full_scan(
@@ -263,6 +274,7 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
                 .into(),
                 usize::MAX,
                 parallel_requests,
+                time,
             )
             .await
             .map(|(g, _)| g)?;
@@ -287,10 +299,16 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
                 if let Some(anchor) = anchor_from_status(&status) {
                     let _ = graph.insert_anchor(txid, anchor);
                 }
+                if !status.confirmed {
+                    if let Some(seen_at) = time {
+                        let _ = graph.insert_seen_at(txid, seen_at);
+                    }
+                }
             }
         }
 
         for op in outpoints.into_iter() {
+            // get tx for this outpoint
             if graph.get_tx(op.txid).is_none() {
                 if let Some(tx) = self.get_tx(&op.txid).await? {
                     let _ = graph.insert_tx(tx);
@@ -299,8 +317,14 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
                 if let Some(anchor) = anchor_from_status(&status) {
                     let _ = graph.insert_anchor(op.txid, anchor);
                 }
+                if !status.confirmed {
+                    if let Some(seen_at) = time {
+                        let _ = graph.insert_seen_at(op.txid, seen_at);
+                    }
+                }
             }
 
+            // get spending status of this outpoint
             if let Some(op_status) = self.get_output_status(&op.txid, op.vout as _).await? {
                 if let Some(txid) = op_status.txid {
                     if graph.get_tx(txid).is_none() {
@@ -311,6 +335,11 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
                         if let Some(anchor) = anchor_from_status(&status) {
                             let _ = graph.insert_anchor(txid, anchor);
                         }
+                        if !status.confirmed {
+                            if let Some(seen_at) = time {
+                                let _ = graph.insert_seen_at(txid, seen_at);
+                            }
+                        }
                     }
                 }
             }

crates/esplora/src/blocking_ext.rs

@@ -49,12 +49,14 @@ pub trait EsploraExt {
     ///
     /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
     /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in
-    /// parallel.
+    /// parallel. `time` is the current time, typically a UNIX time, used only when setting
+    /// the time a transaction was last seen unconfirmed.
     fn full_scan<K: Ord + Clone>(
         &self,
         keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
         stop_gap: usize,
         parallel_requests: usize,
+        time: Option<u64>,
     ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error>;
 
     /// Sync a set of scripts with the blockchain (via an Esplora client) for the data
@@ -64,6 +66,7 @@ pub trait EsploraExt {
     /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s
     /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we
     ///     want to include in the update
+    /// * `time`: UNIX timestamp used to set the time a transaction was last seen unconfirmed
     ///
     /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
     /// may include scripts that have been used, use [`full_scan`] with the keychain.
@@ -75,6 +78,7 @@ pub trait EsploraExt {
         txids: impl IntoIterator<Item = Txid>,
         outpoints: impl IntoIterator<Item = OutPoint>,
         parallel_requests: usize,
+        time: Option<u64>,
     ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error>;
 }
 
@@ -144,6 +148,7 @@ impl EsploraExt for esplora_client::BlockingClient {
         keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
         stop_gap: usize,
         parallel_requests: usize,
+        time: Option<u64>,
     ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error> {
         type TxsOfSpkIndex = (u32, Vec<esplora_client::Tx>);
         let parallel_requests = Ord::max(parallel_requests, 1);
@@ -194,6 +199,11 @@ impl EsploraExt for esplora_client::BlockingClient {
                         if let Some(anchor) = anchor_from_status(&tx.status) {
                             let _ = graph.insert_anchor(tx.txid, anchor);
                         }
+                        if !tx.status.confirmed {
+                            if let Some(seen_at) = time {
+                                let _ = graph.insert_seen_at(tx.txid, seen_at);
+                            }
+                        }
 
                         let previous_outputs = tx.vin.iter().filter_map(|vin| {
                             let prevout = vin.prevout.as_ref()?;
@@ -240,6 +250,7 @@ impl EsploraExt for esplora_client::BlockingClient {
         txids: impl IntoIterator<Item = Txid>,
         outpoints: impl IntoIterator<Item = OutPoint>,
         parallel_requests: usize,
+        time: Option<u64>,
     ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
         let mut graph = self
             .full_scan(
@@ -253,6 +264,7 @@ impl EsploraExt for esplora_client::BlockingClient {
                 .into(),
                 usize::MAX,
                 parallel_requests,
+                time,
             )
             .map(|(g, _)| g)?;
 
@@ -284,10 +296,16 @@ impl EsploraExt for esplora_client::BlockingClient {
                 if let Some(anchor) = anchor_from_status(&status) {
                     let _ = graph.insert_anchor(txid, anchor);
                 }
+                if !status.confirmed {
+                    if let Some(seen_at) = time {
+                        let _ = graph.insert_seen_at(txid, seen_at);
+                    }
+                }
             }
         }
 
         for op in outpoints {
+            // get tx for this outpoint
             if graph.get_tx(op.txid).is_none() {
                 if let Some(tx) = self.get_tx(&op.txid)? {
                     let _ = graph.insert_tx(tx);
@@ -296,8 +314,14 @@ impl EsploraExt for esplora_client::BlockingClient {
                 if let Some(anchor) = anchor_from_status(&status) {
                     let _ = graph.insert_anchor(op.txid, anchor);
                 }
+                if !status.confirmed {
+                    if let Some(seen_at) = time {
+                        let _ = graph.insert_seen_at(op.txid, seen_at);
+                    }
+                }
             }
 
+            // get spending status of this outpoint
             if let Some(op_status) = self.get_output_status(&op.txid, op.vout as _)? {
                 if let Some(txid) = op_status.txid {
                     if graph.get_tx(txid).is_none() {
@@ -308,6 +332,11 @@ impl EsploraExt for esplora_client::BlockingClient {
                         if let Some(anchor) = anchor_from_status(&status) {
                             let _ = graph.insert_anchor(txid, anchor);
                         }
+                        if !status.confirmed {
+                            if let Some(seen_at) = time {
+                                let _ = graph.insert_seen_at(txid, seen_at);
+                            }
+                        }
                     }
                 }
             }

crates/esplora/tests/async_ext.rs

@@ -106,6 +106,7 @@ pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
             vec![].into_iter(),
             vec![].into_iter(),
             1,
+            None,
         )
         .await?;
 
@@ -188,10 +189,12 @@ pub async fn test_async_update_tx_graph_gap_limit() -> anyhow::Result<()> {
 
     // A scan with a gap limit of 2 won't find the transaction, but a scan with a gap limit of 3
     // will.
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 2, 1).await?;
+    let (graph_update, active_indices) =
+        env.client.full_scan(keychains.clone(), 2, 1, None).await?;
     assert!(graph_update.full_txs().next().is_none());
     assert!(active_indices.is_empty());
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 3, 1).await?;
+    let (graph_update, active_indices) =
+        env.client.full_scan(keychains.clone(), 3, 1, None).await?;
     assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr);
     assert_eq!(active_indices[&0], 3);
 
@@ -213,12 +216,13 @@ pub async fn test_async_update_tx_graph_gap_limit() -> anyhow::Result<()> {
 
     // A scan with gap limit 4 won't find the second transaction, but a scan with gap limit 5 will.
     // The last active indice won't be updated in the first case but will in the second one.
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 4, 1).await?;
+    let (graph_update, active_indices) =
+        env.client.full_scan(keychains.clone(), 4, 1, None).await?;
     let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
     assert_eq!(txs.len(), 1);
     assert!(txs.contains(&txid_4th_addr));
     assert_eq!(active_indices[&0], 3);
-    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1).await?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1, None).await?;
     let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
     assert_eq!(txs.len(), 2);
     assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));

crates/esplora/tests/blocking_ext.rs

@@ -134,6 +134,7 @@ pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
         vec![].into_iter(),
         vec![].into_iter(),
         1,
+        None,
     )?;
 
     // Check to see if we have the floating txouts available from our two created transactions'
@@ -216,10 +217,10 @@ pub fn test_update_tx_graph_gap_limit() -> anyhow::Result<()> {
 
     // A scan with a gap limit of 2 won't find the transaction, but a scan with a gap limit of 3
     // will.
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 2, 1)?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 2, 1, None)?;
     assert!(graph_update.full_txs().next().is_none());
     assert!(active_indices.is_empty());
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 3, 1)?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 3, 1, None)?;
     assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr);
     assert_eq!(active_indices[&0], 3);
 
@@ -241,12 +242,12 @@ pub fn test_update_tx_graph_gap_limit() -> anyhow::Result<()> {
 
     // A scan with gap limit 4 won't find the second transaction, but a scan with gap limit 5 will.
     // The last active indice won't be updated in the first case but will in the second one.
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 4, 1)?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 4, 1, None)?;
     let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
     assert_eq!(txs.len(), 1);
     assert!(txs.contains(&txid_4th_addr));
     assert_eq!(active_indices[&0], 3);
-    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1)?;
+    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1, None)?;
     let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
     assert_eq!(txs.len(), 2);
     assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));

example-crates/example_esplora/src/main.rs

@@ -2,6 +2,7 @@ use std::{
     collections::{BTreeMap, BTreeSet},
     io::{self, Write},
     sync::Mutex,
+    time,
 };
 
 use bdk_chain::{
@@ -189,8 +190,16 @@ fn main() -> anyhow::Result<()> {
             // is reached. It returns a `TxGraph` update (`graph_update`) and a structure that
             // represents the last active spk derivation indices of keychains
             // (`keychain_indices_update`).
+            let now = time::SystemTime::now()
+                .duration_since(time::UNIX_EPOCH)?
+                .as_secs();
             let (graph_update, last_active_indices) = client
-                .full_scan(keychain_spks, *stop_gap, scan_options.parallel_requests)
+                .full_scan(
+                    keychain_spks,
+                    *stop_gap,
+                    scan_options.parallel_requests,
+                    Some(now),
+                )
                 .context("scanning for transactions")?;
 
             let mut graph = graph.lock().expect("mutex must not be poisoned");
@@ -307,8 +316,16 @@ fn main() -> anyhow::Result<()> {
                 }
             }
 
-            let graph_update =
-                client.sync(spks, txids, outpoints, scan_options.parallel_requests)?;
+            let now = time::SystemTime::now()
+                .duration_since(time::UNIX_EPOCH)?
+                .as_secs();
+            let graph_update = client.sync(
+                spks,
+                txids,
+                outpoints,
+                scan_options.parallel_requests,
+                Some(now),
+            )?;
 
             graph.lock().unwrap().apply_update(graph_update)
         }

example-crates/wallet_esplora_async/src/main.rs

@@ -1,4 +1,4 @@
-use std::{io::Write, str::FromStr};
+use std::{io::Write, str::FromStr, time};
 
 use bdk::{
     bitcoin::{Address, Network},
@@ -53,8 +53,11 @@ async fn main() -> Result<(), anyhow::Error> {
             (k, k_spks)
         })
         .collect();
+    let now = time::SystemTime::now()
+        .duration_since(time::UNIX_EPOCH)?
+        .as_secs();
     let (update_graph, last_active_indices) = client
-        .full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)
+        .full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS, Some(now))
         .await?;
     let missing_heights = update_graph.missing_heights(wallet.local_chain());
     let chain_update = client.update_local_chain(prev_tip, missing_heights).await?;

example-crates/wallet_esplora_blocking/src/main.rs

@@ -3,7 +3,7 @@ const SEND_AMOUNT: u64 = 1000;
 const STOP_GAP: usize = 5;
 const PARALLEL_REQUESTS: usize = 1;
 
-use std::{io::Write, str::FromStr};
+use std::{io::Write, str::FromStr, time};
 
 use bdk::{
     bitcoin::{Address, Network},
@@ -53,8 +53,11 @@ fn main() -> Result<(), anyhow::Error> {
         })
         .collect();
 
+    let now = time::SystemTime::now()
+        .duration_since(time::UNIX_EPOCH)?
+        .as_secs();
     let (update_graph, last_active_indices) =
-        client.full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)?;
+        client.full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS, Some(now))?;
     let missing_heights = update_graph.missing_heights(wallet.local_chain());
     let chain_update = client.update_local_chain(prev_tip, missing_heights)?;
     let update = Update {