feat(sync): implement exponential backoff for transient sync errors with retry delay multiplier

Author: tosynthegeek

src/chain/mod.rs

@@ -36,7 +36,7 @@ use lightning_transaction_sync::EsploraSyncClient;
 use lightning_block_sync::gossip::UtxoSource;
 use lightning_block_sync::init::{synchronize_listeners, validate_best_block_header};
 use lightning_block_sync::poll::{ChainPoller, ChainTip, ValidatedBlockHeader};
-use lightning_block_sync::SpvClient;
+use lightning_block_sync::{BlockSourceErrorKind, SpvClient};
 
 use bdk_esplora::EsploraAsyncExt;
 use bdk_wallet::Update as BdkUpdate;
@@ -425,6 +425,8 @@ impl ChainSource {
 					"Starting initial synchronization of chain listeners. This might take a while..",
 				);
 
+				let mut backoff_delay_multiplier = 0;
+
 				loop {
 					let channel_manager_best_block_hash =
 						channel_manager.current_best_block().block_hash;
@@ -504,8 +506,23 @@ impl ChainSource {
 
 						Err(e) => {
 							log_error!(logger, "Failed to synchronize chain listeners: {:?}", e);
-							tokio::time::sleep(Duration::from_secs(CHAIN_POLLING_INTERVAL_SECS))
-								.await;
+							backoff_delay_multiplier += 1;
+							let backoff_delay =
+								CHAIN_POLLING_INTERVAL_SECS * backoff_delay_multiplier;
+							if e.kind() == BlockSourceErrorKind::Transient {
+								log_info!(
+									logger,
+									"Transient error. Retrying sync in {} seconds (backoff multiplier: {}).",
+									backoff_delay,
+									backoff_delay_multiplier
+								);
+								tokio::time::sleep(Duration::from_secs(backoff_delay)).await;
+							} else {
+								log_error!(
+									logger,
+									"Failed to synchronize chain listeners: {:?}, e"
+								);
+							}
 						},
 					}
 				}