More effienct esplora syncing (#690)

Author: benthecarman

mutiny-core/src/lib.rs

@@ -47,7 +47,7 @@ pub use crate::ldkstorage::{CHANNEL_MANAGER_KEY, MONITORS_PREFIX_KEY};
 use crate::auth::MutinyAuthClient;
 use crate::labels::{Contact, LabelStorage};
 use crate::nostr::nwc::SpendingConditions;
-use crate::storage::MutinyStorage;
+use crate::storage::{MutinyStorage, NEED_FULL_SYNC_KEY};
 use crate::{error::MutinyError, nostr::ReservedProfile};
 use crate::{nodemanager::NodeManager, nostr::ProfileType};
 use crate::{nostr::NostrManager, utils::sleep};
@@ -162,6 +162,15 @@ impl<S: MutinyStorage> MutinyWallet<S> {
         // start the nostr wallet connect background process
         mw.start_nostr_wallet_connect(first_node).await;
 
+        #[cfg(not(test))]
+        {
+            // if we need a full sync from a restore
+            if mw.storage.get(NEED_FULL_SYNC_KEY)?.unwrap_or_default() {
+                mw.node_manager.wallet.full_sync().await?;
+                mw.storage.delete(&[NEED_FULL_SYNC_KEY])?;
+            }
+        }
+
         Ok(mw)
     }
 
@@ -416,6 +425,8 @@ impl<S: MutinyStorage> MutinyWallet<S> {
 
         self.start().await?;
 
+        self.node_manager.wallet.full_sync().await?;
+
         Ok(())
     }
 
@@ -428,6 +439,7 @@ impl<S: MutinyStorage> MutinyWallet<S> {
         S::clear().await?;
         storage.start().await?;
         storage.insert_mnemonic(m)?;
+        storage.set_data(NEED_FULL_SYNC_KEY, true, None)?;
         Ok(())
     }
 }

mutiny-core/src/nodemanager.rs

@@ -11,7 +11,7 @@ use crate::scb::{
     EncryptedSCB, StaticChannelBackup, StaticChannelBackupStorage,
     SCB_ENCRYPTION_KEY_DERIVATION_PATH,
 };
-use crate::storage::{MutinyStorage, DEVICE_ID_KEY, KEYCHAIN_STORE_KEY};
+use crate::storage::{MutinyStorage, DEVICE_ID_KEY, KEYCHAIN_STORE_KEY, NEED_FULL_SYNC_KEY};
 use crate::utils::sleep;
 use crate::MutinyWalletConfig;
 use crate::{
@@ -521,7 +521,7 @@ pub struct NodeManager<S: MutinyStorage> {
     #[cfg(target_arch = "wasm32")]
     websocket_proxy_addr: String,
     esplora: Arc<AsyncClient>,
-    wallet: Arc<OnChainWallet<S>>,
+    pub(crate) wallet: Arc<OnChainWallet<S>>,
     gossip_sync: Arc<RapidGossipSync>,
     scorer: Arc<utils::Mutex<ProbScorer>>,
     chain: Arc<MutinyChain<S>>,
@@ -2278,6 +2278,7 @@ impl<S: MutinyStorage> NodeManager<S> {
 
         // delete the bdk keychain store
         self.storage.delete(&[KEYCHAIN_STORE_KEY])?;
+        self.storage.set_data(NEED_FULL_SYNC_KEY, true, None)?;
 
         // shut back down after reading if it was already closed
         if needs_db_connection {

mutiny-core/src/onchain.rs

@@ -1,5 +1,5 @@
 use anyhow::anyhow;
-use std::collections::HashSet;
+use std::collections::{BTreeMap, HashSet};
 use std::str::FromStr;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, RwLock};
@@ -91,6 +91,83 @@ impl<S: MutinyStorage> OnChainWallet<S> {
     }
 
     pub async fn sync(&self) -> Result<(), MutinyError> {
+        // get first wallet lock that only needs to read
+        let (checkpoints, spks, txids) = {
+            if let Ok(wallet) = self.wallet.try_read() {
+                let checkpoints = wallet.checkpoints();
+
+                let spk_vec = wallet
+                    .spk_index()
+                    .unused_spks(..)
+                    .map(|(k, v)| (*k, v.clone()))
+                    .collect::<Vec<_>>();
+
+                let mut spk_map = BTreeMap::new();
+                for ((a, b), c) in spk_vec {
+                    spk_map.entry(a).or_insert_with(Vec::new).push((b, c));
+                }
+
+                let chain = wallet.local_chain();
+                let chain_tip = chain.tip().unwrap_or_default();
+
+                let unconfirmed_txids = wallet
+                    .tx_graph()
+                    .list_chain_txs(chain, chain_tip)
+                    .filter(|canonical_tx| !canonical_tx.observed_as.is_confirmed())
+                    .map(|canonical_tx| canonical_tx.node.txid)
+                    .collect::<Vec<Txid>>();
+
+                (checkpoints.clone(), spk_map, unconfirmed_txids)
+            } else {
+                log_error!(self.logger, "Could not get wallet lock to sync");
+                return Err(MutinyError::WalletOperationFailed);
+            }
+        };
+
+        let update = self
+            .blockchain
+            .scan(&checkpoints, spks, txids, core::iter::empty(), 20, 5)
+            .await?;
+
+        // get new wallet lock for writing and apply the update
+        for _ in 0..10 {
+            match self.wallet.try_write() {
+                Ok(mut wallet) => match wallet.apply_update(update) {
+                    Ok(changed) => {
+                        // commit the changes if there were any
+                        if changed {
+                            wallet.commit()?;
+                        }
+
+                        return Ok(());
+                    }
+                    Err(e) => {
+                        // failed to apply wallet update
+                        log_error!(self.logger, "Could not apply wallet update: {e}");
+                        return Err(MutinyError::Other(anyhow!("Could not apply update: {e}")));
+                    }
+                },
+                Err(e) => {
+                    // if we can't get the lock, we just return and try again later
+                    log_error!(
+                        self.logger,
+                        "Could not get wallet lock: {e}, retrying in 250ms"
+                    );
+
+                    if self.stop.load(Ordering::Relaxed) {
+                        return Err(MutinyError::NotRunning);
+                    };
+
+                    sleep(250).await;
+                }
+            }
+        }
+
+        log_error!(self.logger, "Could not get wallet lock after 10 retries");
+        Err(MutinyError::WalletOperationFailed)
+    }
+
+    pub async fn full_sync(&self) -> Result<(), MutinyError> {
         // get first wallet lock that only needs to read
         let (checkpoints, spks) = {
             if let Ok(wallet) = self.wallet.try_read() {

mutiny-core/src/storage.rs

@@ -16,6 +16,7 @@ use uuid::Uuid;
 
 pub const KEYCHAIN_STORE_KEY: &str = "bdk_keychain";
 pub(crate) const MNEMONIC_KEY: &str = "mnemonic";
+pub(crate) const NEED_FULL_SYNC_KEY: &str = "needs_full_sync";
 pub const NODES_KEY: &str = "nodes";
 const FEE_ESTIMATES_KEY: &str = "fee_estimates";
 const FIRST_SYNC_KEY: &str = "first_sync";