fix(cardano-chain-follower): update forever task thread stat

Signed-off-by: bkioshn <[email protected]>

Author: bkioshn

rust/cardano-chain-follower/src/chain_sync.rs

@@ -533,10 +533,6 @@ async fn live_sync_backfill_and_purge(
 ///
 /// This does not return, it is a background task.
 pub(crate) async fn chain_sync(cfg: ChainSyncConfig, rx: mpsc::Receiver<MithrilUpdateMessage>) {
-    /// Thread name for stats.
-    const THREAD_NAME: &str = "ChainSync";
-
-    stats::start_thread(cfg.chain, THREAD_NAME, true);
     debug!(
         "Chain Sync for: {} from {} : Starting",
         cfg.chain, cfg.relay_address,

rust/cardano-chain-follower/src/chain_sync_config.rs

@@ -4,7 +4,7 @@
 //! each network. Chain Followers use the data supplied by the Chain-Sync.
 //! This module configures the chain sync processes.
 
-use std::sync::LazyLock;
+use std::{panic, sync::LazyLock};
 
 use cardano_blockchain_types::Network;
 use dashmap::DashMap;
@@ -125,6 +125,7 @@ impl ChainSyncConfig {
     ///
     /// `Error`: On error.
     pub async fn run(self) -> Result<()> {
+        const THREAD_NAME: &str = "ChainSync";
         debug!(
             chain = self.chain.to_string(),
             "Chain Synchronization Starting"
@@ -150,8 +151,23 @@ impl ChainSyncConfig {
         // Start the Mithril Snapshot Follower
         let rx = self.mithril_cfg.run().await?;
 
-        // Start Chain Sync
-        *locked_handle = Some(tokio::spawn(chain_sync(self.clone(), rx)));
+        // Wrap inside a panic catcher to detect if the task panics.
+        let result = panic::catch_unwind(|| {
+            stats::start_thread(self.chain, THREAD_NAME, true);
+            // Start Chain Sync
+            tokio::spawn(chain_sync(self.clone(), rx))
+        });
+
+        if let Ok(handle) = result {
+            *locked_handle = Some(handle);
+        } else {
+            // Chain sync panic, stop the thread and log.
+            error!(
+                chain = self.chain.to_string(),
+                "Chain Sync for {} : PANICKED", self.chain
+            );
+            stats::stop_thread(self.chain, THREAD_NAME);
+        }
 
         // sync_map.insert(chain, handle);
         debug!("Chain Sync for {} : Started", self.chain);

rust/cardano-chain-follower/src/mithril_snapshot_config.rs

@@ -1,9 +1,7 @@
 //! Configuration for the Mithril Snapshot used by the follower.
 
 use std::{
-    path::{Path, PathBuf},
-    str::FromStr,
-    sync::LazyLock,
+    panic, path::{Path, PathBuf}, str::FromStr, sync::LazyLock
 };
 
 use anyhow::bail;
@@ -20,11 +18,7 @@ use tokio::{
 use tracing::{debug, error};
 
 use crate::{
-    error::{Error, Result},
-    mithril_snapshot_data::{latest_mithril_snapshot_id, SnapshotData},
-    mithril_snapshot_sync::background_mithril_update,
-    snapshot_id::SnapshotId,
-    turbo_downloader::DlConfig,
+    error::{Error, Result}, mithril_snapshot_data::{latest_mithril_snapshot_id, SnapshotData}, mithril_snapshot_sync::background_mithril_update, snapshot_id::SnapshotId, stats, turbo_downloader::DlConfig
 };
 
 /// Type we use to manage the Sync Task handle map.
@@ -384,6 +378,9 @@ impl MithrilSnapshotConfig {
 
     /// Run a Mithril Follower for the given network and configuration.
     pub(crate) async fn run(&self) -> Result<mpsc::Receiver<MithrilUpdateMessage>> {
+        /// Thread name for stats.
+        const THREAD_NAME: &str = "MithrilSnapshotUpdater";
+
         debug!(
             chain = self.chain.to_string(),
             "Mithril Auto-update : Starting"
@@ -413,7 +410,23 @@ impl MithrilSnapshotConfig {
         let (tx, rx) = mpsc::channel::<MithrilUpdateMessage>(2);
 
         // let handle = tokio::spawn(background_mithril_update(chain, self.clone(), tx));
-        *locked_handle = Some(tokio::spawn(background_mithril_update(self.clone(), tx)));
+
+        // Wrap inside a panic catcher to detect if the task panics.
+        let result = panic::catch_unwind(|| {
+            stats::start_thread(self.chain, THREAD_NAME, true);
+            tokio::spawn(background_mithril_update(self.clone(), tx))
+        });
+
+        if let Ok(handle) = result {
+            *locked_handle = Some(handle);
+        } else {
+            // Mithril update panic, stop the thread and log.
+            error!(
+                chain = self.chain.to_string(),
+                "Background Mithril Update for {} : PANICKED", self.chain
+            );
+            stats::stop_thread(self.chain, THREAD_NAME);
+        }
 
         // sync_map.insert(chain, handle);
         debug!(

rust/cardano-chain-follower/src/mithril_snapshot_sync.rs

@@ -694,10 +694,6 @@ macro_rules! next_iteration {
 pub(crate) async fn background_mithril_update(
     cfg: MithrilSnapshotConfig, tx: Sender<MithrilUpdateMessage>,
 ) {
-    /// Thread name for stats.
-    const THREAD_NAME: &str = "MithrilSnapshotUpdater";
-
-    stats::start_thread(cfg.chain, THREAD_NAME, true);
     debug!(
         "Mithril Snapshot background updater for: {} from {} to {} : Starting",
         cfg.chain,