fix(cardano-chain-follower): move mmap file to cat-types

Signed-off-by: bkioshn <[email protected]>

Author: bkioshn

.config/dictionaries/project.dic

@@ -141,6 +141,7 @@ maindbname
 mapref
 mdlint
 mdns
+MEMMAP
 memx
 Metadatum
 mgrybyk

rust/cardano-chain-follower/Cargo.toml

@@ -20,7 +20,7 @@ mithril-client = { version = "0.10.4", default-features = false, features = [
     "num-integer-backend",
 ] }
 cardano-blockchain-types = { version = "0.0.1", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "r20250114-00" }
-catalyst-types = { version = "0.0.1", git = "https://github.com/input-output-hk/catalyst-libs.git", tag = "r20250108-00" }
+catalyst-types = { version = "0.0.1", path = "../catalyst-types" }
 
 thiserror = "1.0.69"
 tokio = { version = "1.42.0", features = [

rust/cardano-chain-follower/src/lib.rs

@@ -14,7 +14,6 @@ mod mithril_snapshot_data;
 mod mithril_snapshot_iterator;
 mod mithril_snapshot_sync;
 mod mithril_turbo_downloader;
-mod mmap_file;
 mod snapshot_id;
 mod stats;
 pub mod turbo_downloader;

rust/cardano-chain-follower/src/mithril_turbo_downloader.rs

@@ -14,7 +14,7 @@ use std::{
 
 use anyhow::{anyhow, bail};
 use async_trait::async_trait;
-use catalyst_types::conversion::from_saturating;
+use catalyst_types::{conversion::from_saturating, mmap_file::MemoryMapFile};
 use dashmap::DashSet;
 use memx::memcmp;
 use mithril_client::{
@@ -28,7 +28,6 @@ use zstd::Decoder;
 use crate::{
     mithril_snapshot_config::MithrilSnapshotConfig,
     mithril_snapshot_data::latest_mithril_snapshot_data,
-    mmap_file::MemoryMapFile,
     stats::{self},
     turbo_downloader::ParallelDownloadProcessor,
 };
@@ -134,7 +133,8 @@ impl Inner {
             self.ext_size.fetch_add(entry_size, Ordering::SeqCst);
 
             // Try and deduplicate the file if we can, otherwise just extract it.
-            if let Ok(prev_mmap) = self.can_deduplicate(&rel_file, entry_size, prev_file.as_ref()) {
+            if let Ok(prev_mmap) = Self::can_deduplicate(&rel_file, entry_size, prev_file.as_ref())
+            {
                 let expected_file_size = from_saturating(entry_size);
                 let mut buf: Vec<u8> = Vec::with_capacity(expected_file_size);
                 if entry.read_to_end(&mut buf)? != expected_file_size {
@@ -145,31 +145,20 @@ impl Inner {
                         buf.len()
                     );
                 }
+                // Got the full file and its the expected size.  Is it different?
+                if memcmp(prev_mmap.file_as_slice(), buf.as_slice()) == cmp::Ordering::Equal {
+                    // Same so lets Hardlink it, and throw away the temp buffer.
 
-                let mut prev_mmap_option = Some(prev_mmap);
-                // Take the value inside the option out
-                if let Some(prev_mmap) = prev_mmap_option.take() {
-                    if memcmp(prev_mmap.file_as_slice(), buf.as_slice()) == cmp::Ordering::Equal {
-                        // Same so let's hardlink it, and throw away the temp buffer.
-                        // Make sure our big mmap get dropped.
-                        drop(prev_mmap);
-                        stats::set_mmap_drop(self.cfg.chain);
-
-                        // File is the same, so dedup it.
-                        if self.cfg.dedup_tmp(&abs_file, &latest_snapshot).is_ok() {
-                            self.dedup_size.fetch_add(entry_size, Ordering::SeqCst);
-                            changed_file!(self, rel_file, abs_file, entry_size);
-                            drop(buf);
-                            continue;
-                        }
-                    }
-                }
-
-                // If the `prev_mmap` is not yet drop, drop it now.
-                // Need to do this way because drop is moved into the if block.
-                if let Some(prev_mmap) = prev_mmap_option {
+                    // Make sure our big mmap get dropped.
                     drop(prev_mmap);
-                    stats::set_mmap_drop(self.cfg.chain);
+
+                    // File is the same, so dedup it.
+                    if self.cfg.dedup_tmp(&abs_file, &latest_snapshot).is_ok() {
+                        self.dedup_size.fetch_add(entry_size, Ordering::SeqCst);
+                        changed_file!(self, rel_file, abs_file, entry_size);
+                        drop(buf);
+                        continue;
+                    }
                 }
 
                 if let Err(error) = std::fs::write(&abs_file, buf) {
@@ -233,7 +222,7 @@ impl Inner {
 
     /// Check if a given path from the archive is able to be deduplicated.
     fn can_deduplicate(
-        &self, rel_file: &Path, file_size: u64, prev_file: Option<&PathBuf>,
+        rel_file: &Path, file_size: u64, prev_file: Option<&PathBuf>,
     ) -> MithrilResult<MemoryMapFile> {
         // Can't dedup if the current file is not de-dupable (must be immutable)
         if rel_file.starts_with("immutable") {
@@ -243,7 +232,7 @@ impl Inner {
                     // If the current file is not exactly the same as the previous file size, we
                     // can't dedup.
                     if file_size == current_size {
-                        if let Ok(pref_file_loaded) = self.mmap_open_sync(prev_file) {
+                        if let Ok(pref_file_loaded) = Self::mmap_open_sync(prev_file) {
                             if pref_file_loaded.size() == file_size {
                                 return Ok(pref_file_loaded);
                             }
@@ -256,12 +245,9 @@ impl Inner {
     }
 
     /// Open a file using mmap for performance.
-    fn mmap_open_sync(&self, path: &Path) -> MithrilResult<MemoryMapFile> {
+    fn mmap_open_sync(path: &Path) -> MithrilResult<MemoryMapFile> {
         match MemoryMapFile::try_from(path) {
-            Ok(mmap_file) => {
-                stats::update_mmap_count_and_size(self.cfg.chain, mmap_file.size());
-                Ok(mmap_file)
-            },
+            Ok(mmap_file) => Ok(mmap_file),
             Err(error) => {
                 error!(error=%error, file=%path.to_string_lossy(), "Failed to open file");
                 Err(error.into())

rust/cardano-chain-follower/src/mmap_file.rs

@@ -3,16 +3,15 @@
 pub(crate) mod follower;
 pub(crate) mod live_chain;
 pub(crate) mod mithril;
-pub(crate) mod mmap_file;
 pub(crate) mod rollback;
 pub(crate) mod thread;
 
 use std::sync::{Arc, LazyLock, RwLock};
 
 use cardano_blockchain_types::{Network, Slot};
+use catalyst_types::mmap_file::MemMapFileStat;
 use chrono::Utc;
 use dashmap::DashMap;
-use mmap_file::MemMapFileStat;
 use rollback::{rollbacks, rollbacks_reset, RollbackType};
 use serde::Serialize;
 use strum::IntoEnumIterator;
@@ -575,56 +574,6 @@ pub fn thread_stat_name(chain: Network) -> Vec<String> {
         .collect()
 }
 
-// ----------------- MMap File STATISTICS-------------------
-
-/// Update the mmap file counter and total size.
-pub(crate) fn update_mmap_count_and_size(chain: Network, size: u64) {
-    // This will actually always succeed.
-    let Some(stats) = lookup_stats(chain) else {
-        return;
-    };
-
-    let Ok(chain_stats) = stats.write() else {
-        // Worst case if this fails (it never should) is we stop updating stats.
-        error!("Stats RwLock should never be able to error.");
-        return;
-    };
-
-    chain_stats.mmap_file_stat.incr_file_counter();
-    chain_stats.mmap_file_stat.update_total_size(size);
-}
-
-/// Set the mmap file to be dropped.
-pub(crate) fn set_mmap_drop(chain: Network) {
-    // This will actually always succeed.
-    let Some(stats) = lookup_stats(chain) else {
-        return;
-    };
-
-    let Ok(chain_stats) = stats.write() else {
-        // Worst case if this fails (it never should) is we stop updating stats.
-        error!("Stats RwLock should never be able to error.");
-        return;
-    };
-
-    chain_stats.mmap_file_stat.set_is_drop();
-}
-
-/// Get the mmap file statistic.
-#[allow(dead_code)]
-pub(crate) fn mmap_file_stat(chain: Network) -> Option<MemMapFileStat> {
-    // This will actually always succeed.
-    let stats = lookup_stats(chain)?;
-
-    let Ok(chain_stats) = stats.write() else {
-        // Worst case if this fails (it never should) is we stop updating stats.
-        error!("Stats RwLock should never be able to error.");
-        return None;
-    };
-
-    Some(chain_stats.mmap_file_stat.clone())
-}
-
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
 mod tests {

rust/cardano-chain-follower/src/stats/mmap_file.rs

@@ -30,6 +30,8 @@ serde = { version = "1.0.217", features = ["derive"] }
 thiserror = "2.0.9"
 base64-url = "3.0.0"
 uuid = { version = "1.11.0", features = ["v4", "v7", "serde"] }
+fmmap = { version = "0.3.3", features = ["sync", "tokio-async"] }
+once_cell = "1.20.2"
 
 [dev-dependencies]
 ed25519-dalek = { version = "2.1.1", features = ["rand_core"] }

rust/cardano-chain-follower/src/stats/mod.rs

@@ -5,3 +5,4 @@ pub mod hashes;
 pub mod kid_uri;
 pub mod problem_report;
 pub mod uuid;
+pub mod mmap_file;