refactor(aggregator): add snapshot_ancillary to snapshotter

moving the responsability to identify the files to include in the
archive from the ancillary artifact builder to the snapshotter.

Co-authored-by: Damien Lachaume <[email protected]>

Author: Alenar

mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs

@@ -146,17 +146,14 @@ impl AncillaryArtifactBuilder {
             beacon.immutable_file_number
         );
 
-        let paths_to_include =
-            Self::get_files_and_directories_to_snapshot(beacon.immutable_file_number);
-
         let archive_name = format!(
             "{}-e{}-i{}.ancillary",
             self.cardano_network, *beacon.epoch, beacon.immutable_file_number,
         );
 
         let snapshot = self
             .snapshotter
-            .snapshot_subset(&archive_name, paths_to_include)
+            .snapshot_ancillary(beacon.immutable_file_number, &archive_name)
             .with_context(|| {
                 format!(
                     "Failed to create ancillary archive for immutable file number: {}",
@@ -236,19 +233,13 @@ impl AncillaryArtifactBuilder {
 
 #[cfg(test)]
 mod tests {
-    use std::fs::File;
-
-    use flate2::read::GzDecoder;
-    use tar::Archive;
-
     use mithril_common::{
-        digesters::{DummyCardanoDbBuilder, IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR},
+        digesters::DummyCardanoDbBuilder,
         test_utils::{assert_equivalent, TempDir},
     };
 
-    use crate::services::{CompressedArchiveSnapshotter, DumbSnapshotter, MockSnapshotter};
+    use crate::services::{DumbSnapshotter, MockSnapshotter};
     use crate::test_tools::TestLogger;
-    use crate::tools::file_archiver::FileArchiver;
 
     use super::*;
 
@@ -511,79 +502,11 @@ mod tests {
         assert!(!archive_path.exists());
     }
 
-    #[tokio::test]
-    async fn create_archive_should_embed_ledger_volatile_directories_and_last_immutables() {
-        let test_dir = "create_archive/cardano_database";
-        let cardano_db = DummyCardanoDbBuilder::new(test_dir)
-            .with_immutables(&[1, 2, 3])
-            .with_ledger_files(&["437", "537", "637", "737"])
-            .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
-            .build();
-        std::fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
-
-        let db_directory = cardano_db.get_dir().to_path_buf();
-        let snapshotter = CompressedArchiveSnapshotter::new(
-            db_directory.clone(),
-            db_directory.parent().unwrap().join("snapshot_dest"),
-            CompressionAlgorithm::Gzip,
-            Arc::new(FileArchiver::new_for_test(
-                cardano_db.get_dir().join("verification"),
-            )),
-            TestLogger::stdout(),
-        )
-        .unwrap();
-
-        let builder = AncillaryArtifactBuilder::new(
-            vec![Arc::new(MockAncillaryFileUploader::new())],
-            Arc::new(snapshotter),
-            CardanoNetwork::DevNet(123),
-            TestLogger::stdout(),
-        )
-        .unwrap();
-
-        let snapshot = builder
-            .create_ancillary_archive(&CardanoDbBeacon::new(99, 2))
-            .unwrap();
-
-        let mut archive = {
-            let file_tar_gz = File::open(snapshot.get_file_path()).unwrap();
-            let file_tar_gz_decoder = GzDecoder::new(file_tar_gz);
-            Archive::new(file_tar_gz_decoder)
-        };
-
-        let dst = cardano_db.get_dir().join("unpack_dir");
-        archive.unpack(dst.clone()).unwrap();
-
-        let expected_immutable_path = dst.join(IMMUTABLE_DIR);
-        assert!(expected_immutable_path.join("00003.chunk").exists());
-        assert!(expected_immutable_path.join("00003.primary").exists());
-        assert!(expected_immutable_path.join("00003.secondary").exists());
-        let immutables_nb = std::fs::read_dir(expected_immutable_path).unwrap().count();
-        assert_eq!(3, immutables_nb);
-
-        let expected_ledger_path = dst.join(LEDGER_DIR);
-        assert!(expected_ledger_path.join("437").exists());
-        assert!(expected_ledger_path.join("537").exists());
-        assert!(expected_ledger_path.join("637").exists());
-        assert!(expected_ledger_path.join("737").exists());
-        let volatile_nb = std::fs::read_dir(expected_ledger_path).unwrap().count();
-        assert_eq!(4, volatile_nb);
-
-        let expected_volatile_path = dst.join(VOLATILE_DIR);
-        assert!(expected_volatile_path.join("blocks-0.dat").exists());
-        assert!(expected_volatile_path.join("blocks-1.dat").exists());
-        assert!(expected_volatile_path.join("blocks-2.dat").exists());
-        let ledger_nb = std::fs::read_dir(expected_volatile_path).unwrap().count();
-        assert_eq!(3, ledger_nb);
-
-        assert!(!dst.join("whatever").exists());
-    }
-
     #[tokio::test]
     async fn upload_should_return_error_and_not_upload_when_archive_creation_fails() {
         let mut snapshotter = MockSnapshotter::new();
         snapshotter
-            .expect_snapshot_subset()
+            .expect_snapshot_ancillary()
             .returning(|_, _| Err(anyhow!("Failed to create archive")));
 
         let mut uploader = MockAncillaryFileUploader::new();

mithril-aggregator/src/services/snapshotter/compressed_archive_snapshotter.rs

@@ -4,7 +4,7 @@ use std::fs;
 use std::path::PathBuf;
 use std::sync::Arc;
 
-use mithril_common::digesters::{immutable_trio_names, IMMUTABLE_DIR};
+use mithril_common::digesters::{immutable_trio_names, LedgerFile, IMMUTABLE_DIR, LEDGER_DIR};
 use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber};
 use mithril_common::logging::LoggerExtensions;
 use mithril_common::StdResult;
@@ -51,6 +51,22 @@ impl Snapshotter for CompressedArchiveSnapshotter {
         self.snapshot(archive_name_without_extension, appender)
     }
 
+    fn snapshot_ancillary(
+        &self,
+        immutable_file_number: ImmutableFileNumber,
+        archive_name_without_extension: &str,
+    ) -> StdResult<FileArchive> {
+        debug!(
+            self.logger,
+            "Snapshotting ancillary archive: '{archive_name_without_extension}'"
+        );
+
+        let paths_to_include =
+            self.get_files_and_directories_for_ancillary_snapshot(immutable_file_number)?;
+        let appender = AppenderEntries::new(paths_to_include, self.db_directory.clone())?;
+        self.snapshot(archive_name_without_extension, appender)
+    }
+
     fn snapshot_immutable_trio(
         &self,
         immutable_file_number: ImmutableFileNumber,
@@ -126,6 +142,36 @@ impl CompressedArchiveSnapshotter {
             appender,
         )
     }
+
+    /// Returns the list of files and directories to include in ancillary snapshot.
+    ///
+    /// The immutable file included in the ancillary archive corresponds to the last one (and not finalized yet)
+    /// when the immutable file number given to the function corresponds to the penultimate.
+    fn get_files_and_directories_for_ancillary_snapshot(
+        &self,
+        immutable_file_number: u64,
+    ) -> StdResult<Vec<PathBuf>> {
+        let mut files_to_snapshot = Vec::with_capacity(4);
+
+        let next_immutable_file_number = immutable_file_number + 1;
+        files_to_snapshot.extend_from_slice(
+            immutable_trio_names(next_immutable_file_number)
+                .iter()
+                .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
+                .collect::<Vec<_>>()
+                .as_slice(),
+        );
+
+        let db_ledger_dir = self.db_directory.join(LEDGER_DIR);
+        let ledger_files = LedgerFile::list_all_in_dir(&db_ledger_dir)?;
+        let last_ledger = ledger_files.last().ok_or(anyhow!(
+            "No ledger file found in directory: `{}`",
+            db_ledger_dir.display()
+        ))?;
+        files_to_snapshot.push(PathBuf::from(LEDGER_DIR).join(&last_ledger.filename));
+
+        Ok(files_to_snapshot)
+    }
 }
 
 #[cfg(test)]
@@ -282,4 +328,49 @@ mod tests {
             );
         }
     }
+
+    mod snapshot_ancillary {
+        use mithril_common::digesters::VOLATILE_DIR;
+
+        use super::*;
+
+        #[tokio::test]
+        async fn create_archive_should_embed_only_last_ledger_and_last_immutables() {
+            let test_dir = get_test_directory(
+                "create_archive_should_embed_only_last_ledger_and_last_immutables",
+            );
+            let cardano_db = DummyCardanoDbBuilder::new(
+                "create_archive_should_embed_only_last_ledger_and_last_immutables",
+            )
+            .with_immutables(&[1, 2, 3])
+            .with_ledger_files(&["437", "537", "637", "737", "9not_included"])
+            .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
+            .build();
+            fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
+
+            let db_directory = cardano_db.get_dir();
+            let snapshotter =
+                snapshotter_for_test(&test_dir, db_directory, CompressionAlgorithm::Gzip);
+
+            let snapshot = snapshotter.snapshot_ancillary(2, "ancillary").unwrap();
+
+            let unpack_dir = unpack_gz_decoder(test_dir, snapshot);
+
+            let expected_immutable_path = unpack_dir.join(IMMUTABLE_DIR);
+            assert!(expected_immutable_path.join("00003.chunk").exists());
+            assert!(expected_immutable_path.join("00003.primary").exists());
+            assert!(expected_immutable_path.join("00003.secondary").exists());
+            assert_eq!(3, list_files(&expected_immutable_path).len());
+
+            // Only the last ledger file should be included
+            let expected_ledger_path = unpack_dir.join(LEDGER_DIR);
+            assert!(expected_ledger_path.join("737").exists());
+            assert_eq!(1, list_files(&expected_ledger_path).len());
+
+            let expected_volatile_path = unpack_dir.join(VOLATILE_DIR);
+            assert!(!expected_volatile_path.exists());
+
+            assert!(!unpack_dir.join("whatever").exists());
+        }
+    }
 }

mithril-aggregator/src/services/snapshotter/interface.rs

@@ -18,6 +18,15 @@ pub trait Snapshotter: Sync + Send {
         files: Vec<PathBuf>,
     ) -> StdResult<FileArchive>;
 
+    /// Create a new snapshot of ancillary files.
+    ///
+    /// Ancillary files include the last, uncompleted, immutable trio and the last ledger file.
+    fn snapshot_ancillary(
+        &self,
+        immutable_file_number: ImmutableFileNumber,
+        archive_name_without_extension: &str,
+    ) -> StdResult<FileArchive>;
+
     /// Create a new snapshot of an immutable trio.
     fn snapshot_immutable_trio(
         &self,

mithril-aggregator/src/services/snapshotter/test_doubles.rs

@@ -67,6 +67,14 @@ impl Snapshotter for DumbSnapshotter {
         self.snapshot_all(archive_name_without_extension)
     }
 
+    fn snapshot_ancillary(
+        &self,
+        _immutable_file_number: ImmutableFileNumber,
+        archive_name_without_extension: &str,
+    ) -> StdResult<FileArchive> {
+        self.snapshot_all(archive_name_without_extension)
+    }
+
     fn snapshot_immutable_trio(
         &self,
         _immutable_file_number: ImmutableFileNumber,
@@ -131,6 +139,14 @@ impl Snapshotter for FakeSnapshotter {
         self.snapshot_all(archive_name_without_extension)
     }
 
+    fn snapshot_ancillary(
+        &self,
+        _immutable_file_number: ImmutableFileNumber,
+        archive_name_without_extension: &str,
+    ) -> StdResult<FileArchive> {
+        self.snapshot_all(archive_name_without_extension)
+    }
+
     fn snapshot_immutable_trio(
         &self,
         _immutable_file_number: ImmutableFileNumber,
@@ -199,6 +215,17 @@ mod tests {
                     )
                 );
             }
+            {
+                let ancillary_snapshot = snapshotter
+                    .snapshot_ancillary(3, "whatever")
+                    .expect("Dumb snapshotter::snapshot_ancillary should not fail.");
+                assert_eq!(
+                    Some(ancillary_snapshot),
+                    snapshotter.get_last_snapshot().expect(
+                        "Dumb snapshotter::get_last_snapshot should not fail when some last snapshot."
+                    )
+                );
+            }
             {
                 let immutable_snapshot = snapshotter
                     .snapshot_immutable_trio(4, "whatever")
@@ -257,6 +284,16 @@ mod tests {
                     );
                     assert!(full_snapshot.get_file_path().is_file());
                 }
+                {
+                    let ancillary_snapshot =
+                        fake_snapshotter.snapshot_ancillary(3, filename).unwrap();
+
+                    assert_eq!(
+                        ancillary_snapshot.get_file_path(),
+                        &test_dir.join(filename).with_extension("tar.gz")
+                    );
+                    assert!(ancillary_snapshot.get_file_path().is_file());
+                }
                 {
                     let immutable_snapshot = fake_snapshotter
                         .snapshot_immutable_trio(5, filename)

mithril-aggregator/src/tools/file_archiver/entities.rs

@@ -29,6 +29,7 @@ impl ArchiveParameters {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct FileArchive {
     pub(super) filepath: PathBuf,
+    // Todo: rename to `archive_size` and add `uncompressed_size` field.
     pub(super) filesize: u64,
     pub(super) compression_algorithm: CompressionAlgorithm,
 }