Merge branch 'main' into feat/cat-kid-to-cat-id

Author: stevenj

.config/dictionaries/project.dic

@@ -143,6 +143,7 @@ maindbname
 mapref
 mdlint
 mdns
+MEMMAP
 memx
 Metadatum
 mgrybyk

rust/cardano-blockchain-types/src/network.rs

@@ -37,13 +37,15 @@ pub(crate) const ENVVAR_MITHRIL_EXE_NAME: &str = "MITHRIL_EXE_NAME";
     strum::Display,
 )]
 #[strum(ascii_case_insensitive)]
+#[strum(serialize_all = "snake_case")]
+#[repr(usize)]
 pub enum Network {
     /// Cardano mainnet network.
-    Mainnet,
+    Mainnet = 0,
     /// Cardano pre-production network.
-    Preprod,
+    Preprod = 1,
     /// Cardano preview network.
-    Preview,
+    Preview = 2,
 }
 
 // Mainnet Defaults.
@@ -224,6 +226,7 @@ mod tests {
 
     use anyhow::Ok;
     use chrono::{TimeZone, Utc};
+    use strum::VariantNames;
 
     use super::*;
 
@@ -248,6 +251,23 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn test_variant_to_usize() {
+        assert_eq!(Network::Mainnet as usize, 0);
+        assert_eq!(Network::Preprod as usize, 1);
+        assert_eq!(Network::Preview as usize, 2);
+    }
+
+    #[test]
+    fn test_variant_to_string() {
+        assert_eq!(Network::Mainnet.to_string(), "mainnet");
+        assert_eq!(Network::Preprod.to_string(), "preprod");
+        assert_eq!(Network::Preview.to_string(), "preview");
+
+        let expected_names = ["mainnet", "preprod", "preview"];
+        assert_eq!(Network::VARIANTS, expected_names);
+    }
+
     #[test]
     fn test_time_to_slot_before_blockchain() {
         let network = Network::Mainnet;

rust/cardano-chain-follower/Cargo.toml

@@ -20,7 +20,7 @@ mithril-client = { version = "0.10.4", default-features = false, features = [
     "num-integer-backend",
 ] }
 cardano-blockchain-types = { version = "0.0.1", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "r20250114-00" }
-catalyst-types = { version = "0.0.1", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "r20250108-00" }
+catalyst-types = { version = "0.0.1", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "r20250121-00" }
 
 thiserror = "1.0.69"
 tokio = { version = "1.42.0", features = [
@@ -31,7 +31,7 @@ tokio = { version = "1.42.0", features = [
 ] }
 tracing = "0.1.41"
 tracing-log = "0.2.0"
-dashmap = "6.1.0"
+dashmap = { version = "6.1.0", features = ["serde"] }
 url = "2.5.4"
 anyhow = "1.0.95"
 chrono = "0.4.39"
@@ -48,14 +48,14 @@ serde = "1.0.217"
 serde_json = "1.0.134"
 mimalloc = { version = "0.1.43", optional = true }
 memx = "0.1.32"
-fmmap = { version = "0.3.3", features = ["sync", "tokio-async"] }
 zstd = "0.13.2"
 logcall = "0.1.11"
 tar = "0.4.43"
 ureq = { version = "2.12.1", features = ["native-certs"] }
 http = "1.2.0"
 hickory-resolver = { version = "0.24.2", features = ["dns-over-rustls"] }
 moka = { version = "0.12.9", features = ["sync"] }
+cpu-time = "1.0.0"
 
 [dev-dependencies]
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }

rust/cardano-chain-follower/src/chain_sync.rs

@@ -528,7 +528,13 @@ pub(crate) async fn chain_sync(cfg: ChainSyncConfig, rx: mpsc::Receiver<MithrilU
 
     // Start the Live chain backfill task.
     let _backfill_join_handle = spawn(async move {
+        stats::start_thread(
+            cfg.chain,
+            stats::thread::name::LIVE_SYNC_BACKFILL_AND_PURGE,
+            true,
+        );
         live_sync_backfill_and_purge(backfill_cfg.clone(), rx, sync_waiter).await;
+        stats::stop_thread(cfg.chain, stats::thread::name::LIVE_SYNC_BACKFILL_AND_PURGE);
     });
 
     // Live Fill data starts at fork 1.
@@ -539,7 +545,6 @@ pub(crate) async fn chain_sync(cfg: ChainSyncConfig, rx: mpsc::Receiver<MithrilU
     loop {
         // We never have a connection if we end up around the loop, so make a new one.
         let mut peer = persistent_reconnect(&cfg.relay_address, cfg.chain).await;
-
         match resync_live_tip(&mut peer, cfg.chain).await {
             Ok(tip) => debug!("Tip Resynchronized to {tip}"),
             Err(error) => {

rust/cardano-chain-follower/src/chain_sync_config.rs

@@ -150,8 +150,13 @@ impl ChainSyncConfig {
         // Start the Mithril Snapshot Follower
         let rx = self.mithril_cfg.run().await?;
 
+        let config = self.clone();
         // Start Chain Sync
-        *locked_handle = Some(tokio::spawn(chain_sync(self.clone(), rx)));
+        *locked_handle = Some(tokio::spawn(async move {
+            stats::start_thread(config.chain, stats::thread::name::CHAIN_SYNC, true);
+            chain_sync(config.clone(), rx).await;
+            stats::stop_thread(config.chain, stats::thread::name::CHAIN_SYNC);
+        }));
 
         // sync_map.insert(chain, handle);
         debug!("Chain Sync for {} : Started", self.chain);

rust/cardano-chain-follower/src/chain_sync_ready.rs

@@ -12,7 +12,7 @@ use tokio::{
 };
 use tracing::error;
 
-use crate::chain_update;
+use crate::{chain_update, stats};
 
 /// Data we hold related to sync being ready or not.
 struct SyncReady {
@@ -88,6 +88,7 @@ pub(crate) fn wait_for_sync_ready(chain: Network) -> SyncReadyWaiter {
     let (tx, rx) = oneshot::channel::<()>();
 
     tokio::spawn(async move {
+        stats::start_thread(chain, stats::thread::name::WAIT_FOR_SYNC_READY, true);
         // We are safe to use `expect` here because the SYNC_READY list is exhaustively
         // initialized. Its a Serious BUG if that not True, so panic is OK.
         #[allow(clippy::expect_used)]
@@ -101,7 +102,7 @@ pub(crate) fn wait_for_sync_ready(chain: Network) -> SyncReadyWaiter {
         if let Ok(()) = rx.await {
             status.ready = true;
         }
-
+        stats::stop_thread(chain, stats::thread::name::WAIT_FOR_SYNC_READY);
         // If the channel closes early, we can NEVER use the Blockchain data.
     });
 

rust/cardano-chain-follower/src/mithril_query.rs

@@ -2,25 +2,32 @@
 
 use std::path::Path;
 
-use cardano_blockchain_types::Point;
+use cardano_blockchain_types::{Network, Point};
 use pallas_hardano::storage::immutable::FallibleBlock;
 use tokio::task;
 
-use crate::error::{Error, Result};
+use crate::{
+    error::{Error, Result},
+    stats,
+};
 
 /// Synchronous Immutable block iterator.
 pub(crate) type ImmutableBlockIterator = Box<dyn Iterator<Item = FallibleBlock> + Send + Sync>;
 
 /// Get a mithril snapshot iterator.
 pub(crate) async fn make_mithril_iterator(
-    path: &Path, start: &Point,
+    path: &Path, start: &Point, chain: Network,
 ) -> Result<ImmutableBlockIterator> {
     let path = path.to_path_buf();
     let start = start.clone();
     // Initial input
     let res = task::spawn_blocking(move || {
-        pallas_hardano::storage::immutable::read_blocks_from_point(&path, start.clone().into())
-            .map_err(|error| Error::MithrilSnapshot(Some(error)))
+        stats::start_thread(chain, stats::thread::name::MITHRIL_ITERATOR, false);
+        let result =
+            pallas_hardano::storage::immutable::read_blocks_from_point(&path, start.clone().into())
+                .map_err(|error| Error::MithrilSnapshot(Some(error)));
+        stats::stop_thread(chain, stats::thread::name::MITHRIL_ITERATOR);
+        result
     })
     .await;
 

rust/cardano-chain-follower/src/mithril_snapshot_config.rs

@@ -24,6 +24,7 @@ use crate::{
     mithril_snapshot_data::{latest_mithril_snapshot_id, SnapshotData},
     mithril_snapshot_sync::background_mithril_update,
     snapshot_id::SnapshotId,
+    stats,
     turbo_downloader::DlConfig,
 };
 
@@ -413,7 +414,16 @@ impl MithrilSnapshotConfig {
         let (tx, rx) = mpsc::channel::<MithrilUpdateMessage>(2);
 
         // let handle = tokio::spawn(background_mithril_update(chain, self.clone(), tx));
-        *locked_handle = Some(tokio::spawn(background_mithril_update(self.clone(), tx)));
+        let config = self.clone();
+        *locked_handle = Some(tokio::spawn(async move {
+            stats::start_thread(
+                config.chain,
+                stats::thread::name::MITHRIL_SNAPSHOT_UPDATER,
+                true,
+            );
+            background_mithril_update(config.clone(), tx).await;
+            stats::stop_thread(config.chain, stats::thread::name::MITHRIL_SNAPSHOT_UPDATER);
+        }));
 
         // sync_map.insert(chain, handle);
         debug!(

rust/cardano-chain-follower/src/mithril_snapshot_iterator.rs

@@ -73,7 +73,7 @@ impl MithrilSnapshotIterator {
         chain: Network, path: &Path, from: &Point, search_interval: u64,
     ) -> Option<MithrilSnapshotIterator> {
         let point = probe_point(from, search_interval);
-        let Ok(mut iterator) = make_mithril_iterator(path, &point).await else {
+        let Ok(mut iterator) = make_mithril_iterator(path, &point, chain).await else {
             return None;
         };
 
@@ -116,7 +116,7 @@ impl MithrilSnapshotIterator {
         let this = this?;
 
         // Remake the iterator, based on the new known point.
-        let Ok(iterator) = make_mithril_iterator(path, &this).await else {
+        let Ok(iterator) = make_mithril_iterator(path, &this, chain).await else {
             return None;
         };
 
@@ -176,7 +176,7 @@ impl MithrilSnapshotIterator {
 
         debug!("Actual Mithril Iterator Start: {}", from);
 
-        let iterator = make_mithril_iterator(path, from).await?;
+        let iterator = make_mithril_iterator(path, from, chain).await?;
 
         Ok(MithrilSnapshotIterator {
             inner: Arc::new(Mutex::new(MithrilSnapshotIteratorInner {

rust/cardano-chain-follower/src/mithril_snapshot_sync.rs

@@ -300,9 +300,12 @@ async fn validate_mithril_snapshot(
     match tokio::spawn(async move {
         // This can be long running and CPU Intensive.
         // So we spawn it off to a background task.
-        MessageBuilder::new()
+        stats::start_thread(chain, stats::thread::name::COMPUTE_SNAPSHOT_MSG, true);
+        let result = MessageBuilder::new()
             .compute_snapshot_message(&cert, &mithril_path)
-            .await
+            .await;
+        stats::stop_thread(chain, stats::thread::name::COMPUTE_SNAPSHOT_MSG);
+        result
     })
     .await
     {
@@ -517,6 +520,7 @@ fn background_validate_mithril_snapshot(
     chain: Network, certificate: MithrilCertificate, tmp_path: PathBuf,
 ) -> tokio::task::JoinHandle<bool> {
     tokio::spawn(async move {
+        stats::start_thread(chain, stats::thread::name::VALIDATE_MITHRIL_SNAPSHOT, true);
         debug!(
             "Mithril Snapshot background updater for: {} : Check Certificate.",
             chain
@@ -540,6 +544,7 @@ fn background_validate_mithril_snapshot(
             chain
         );
 
+        stats::stop_thread(chain, stats::thread::name::VALIDATE_MITHRIL_SNAPSHOT);
         true
     })
 }