feat(aggregator): file archiver now compute and return size of uncompressed data

Co-authored-by: Damien Lachaume <[email protected]>

Author: Alenar

mithril-aggregator/src/artifact_builder/cardano_database_artifacts/ancillary.rs

@@ -377,6 +377,7 @@ mod tests {
             .upload_ancillary_archive(FileArchive::new(
                 PathBuf::from("archive_path"),
                 0,
+                0,
                 CompressionAlgorithm::Gzip,
             ))
             .await
@@ -416,6 +417,7 @@ mod tests {
             .upload_ancillary_archive(FileArchive::new(
                 PathBuf::from("archive_path"),
                 0,
+                0,
                 CompressionAlgorithm::Gzip,
             ))
             .await
@@ -443,7 +445,7 @@ mod tests {
             "upload_ancillary_archive_should_remove_archive_after_upload",
         );
         let archive_path = create_fake_archive(&source_dir, "ancillary.tar.gz");
-        let archive = FileArchive::new(archive_path.clone(), 0, CompressionAlgorithm::Gzip);
+        let archive = FileArchive::new(archive_path.clone(), 0, 0, CompressionAlgorithm::Gzip);
         let uploader = fake_uploader(
             archive_path.as_os_str().to_str().unwrap(),
             "an_uri",
@@ -472,7 +474,7 @@ mod tests {
             "upload_ancillary_archive_should_remove_archive_when_no_uploader_succeed",
         );
         let archive_path = create_fake_archive(&source_dir, "ancillary.tar.gz");
-        let archive = FileArchive::new(archive_path.clone(), 0, CompressionAlgorithm::Gzip);
+        let archive = FileArchive::new(archive_path.clone(), 0, 0, CompressionAlgorithm::Gzip);
         let uploader = fake_uploader_returning_error();
 
         let builder = AncillaryArtifactBuilder::new(

mithril-aggregator/src/artifact_builder/cardano_immutable_files_full.rs

@@ -107,7 +107,7 @@ impl CardanoImmutableFilesFullArtifactBuilder {
             snapshot_digest,
             self.cardano_network,
             beacon,
-            ongoing_snapshot.get_file_size(),
+            ongoing_snapshot.get_archive_size(),
             remote_locations,
             ongoing_snapshot.get_compression_algorithm(),
             &self.cardano_node_version,
@@ -211,7 +211,7 @@ mod tests {
             snapshot_digest.to_owned(),
             fake_data::network(),
             beacon,
-            last_ongoing_snapshot.get_file_size(),
+            last_ongoing_snapshot.get_archive_size(),
             remote_locations,
             CompressionAlgorithm::Zstandard,
             &Version::parse("1.0.0").unwrap(),
@@ -226,6 +226,7 @@ mod tests {
         let snapshot = FileArchive::new(
             file_path.to_path_buf(),
             7331,
+            7331,
             CompressionAlgorithm::default(),
         );
 
@@ -321,6 +322,7 @@ mod tests {
         let snapshot = FileArchive::new(
             file_path.to_path_buf(),
             7331,
+            7331,
             CompressionAlgorithm::default(),
         );
         let mut snapshot_uploader = MockFileUploader::new();

mithril-aggregator/src/services/snapshotter/test_doubles.rs

@@ -52,6 +52,7 @@ impl Snapshotter for DumbSnapshotter {
                 self.compression_algorithm.tar_file_extension()
             )),
             0,
+            0,
             self.compression_algorithm,
         );
         *value = Some(snapshot.clone());
@@ -127,6 +128,7 @@ impl Snapshotter for FakeSnapshotter {
         Ok(FileArchive::new(
             fake_archive_path,
             0,
+            0,
             self.compression_algorithm,
         ))
     }
@@ -185,13 +187,13 @@ mod tests {
             let snapshot = snapshotter.snapshot_all("archive").unwrap();
 
             assert_eq!(PathBuf::from("archive.tar.gz"), *snapshot.get_file_path());
-            assert_eq!(0, snapshot.get_file_size());
+            assert_eq!(0, snapshot.get_archive_size());
 
             let snapshot = snapshotter
                 .snapshot_subset("archive", vec![PathBuf::from("whatever")])
                 .unwrap();
             assert_eq!(PathBuf::from("archive.tar.gz"), *snapshot.get_file_path());
-            assert_eq!(0, snapshot.get_file_size());
+            assert_eq!(0, snapshot.get_archive_size());
         }
 
         #[test]

mithril-aggregator/src/tools/file_archiver/api.rs

@@ -14,6 +14,7 @@ use mithril_common::entities::CompressionAlgorithm;
 use mithril_common::logging::LoggerExtensions;
 use mithril_common::StdResult;
 
+use crate::tools::file_size;
 use crate::ZstandardCompressionParameters;
 
 use super::appender::TarAppender;
@@ -175,16 +176,24 @@ impl FileArchiver {
             }
         }
 
-        let filesize = Self::get_file_size(archive_path).with_context(|| {
+        let uncompressed_size = appender.compute_uncompressed_data_size().with_context(|| {
             format!(
-                "FileArchiver can not get file size of archive with path: '{}'",
+                "FileArchiver can not get the size of the uncompressed data to archive: '{}'",
                 archive_path.display()
             )
         })?;
+        let archive_filesize =
+            file_size::compute_size_of_path(archive_path).with_context(|| {
+                format!(
+                    "FileArchiver can not get file size of archive with path: '{}'",
+                    archive_path.display()
+                )
+            })?;
 
         Ok(FileArchive {
             filepath: archive_path.to_path_buf(),
-            filesize,
+            archive_filesize,
+            uncompressed_size,
             compression_algorithm,
         })
     }
@@ -260,18 +269,6 @@ impl FileArchiver {
         verify_result
     }
 
-    fn get_file_size(filepath: &Path) -> StdResult<u64> {
-        let res = fs::metadata(filepath)
-            .with_context(|| {
-                format!(
-                    "FileArchiver can not get metadata of file: '{}'",
-                    filepath.display()
-                )
-            })?
-            .len();
-        Ok(res)
-    }
-
     // Helper to unpack and delete a file from en entry, for archive verification purpose
     fn unpack_and_delete_file_from_entry<R: Read>(
         entry: Entry<R>,
@@ -309,7 +306,7 @@ mod tests {
     use mithril_common::test_utils::assert_equivalent;
 
     use crate::test_tools::TestLogger;
-    use crate::tools::file_archiver::appender::AppenderDirAll;
+    use crate::tools::file_archiver::appender::{AppenderDirAll, AppenderFile};
     use crate::tools::file_archiver::test_tools::*;
     use crate::ZstandardCompressionParameters;
 
@@ -446,14 +443,14 @@ mod tests {
                 AppenderDirAll::new(archived_directory.clone()),
             )
             .unwrap();
-        let first_snapshot_size = first_snapshot.get_file_size();
+        let first_snapshot_size = first_snapshot.get_archive_size();
 
         create_file(&archived_directory, "another_file_to_archive.txt");
 
         let second_snapshot = file_archiver
             .archive(archive_params, AppenderDirAll::new(archived_directory))
             .unwrap();
-        let second_snapshot_size = second_snapshot.get_file_size();
+        let second_snapshot_size = second_snapshot.get_archive_size();
 
         assert_ne!(first_snapshot_size, second_snapshot_size);
 
@@ -472,4 +469,31 @@ mod tests {
         file_archiver.set_verification_temp_dir("sub_dir");
         file_archiver.set_verification_temp_dir("sub_dir".to_string());
     }
+
+    #[test]
+    fn compute_size_of_uncompressed_data_and_archive() {
+        let test_dir = get_test_directory("compute_size_of_uncompressed_data_and_archive");
+
+        let file_path = test_dir.join("file.txt");
+        let file = File::create(&file_path).unwrap();
+        file.set_len(777).unwrap();
+
+        let file_archiver = FileArchiver::new_for_test(test_dir.join("verification"));
+
+        let archive_params = ArchiveParameters {
+            archive_name_without_extension: "archive".to_string(),
+            target_directory: test_dir.clone(),
+            compression_algorithm: CompressionAlgorithm::Gzip,
+        };
+        let snapshot = file_archiver
+            .archive(
+                archive_params.clone(),
+                AppenderFile::append_at_archive_root(file_path.clone()).unwrap(),
+            )
+            .unwrap();
+
+        let expected_archive_size = file_size::compute_size_of_path(&snapshot.filepath).unwrap();
+        assert_eq!(expected_archive_size, snapshot.get_archive_size(),);
+        assert_eq!(777, snapshot.get_uncompressed_size());
+    }
 }

mithril-aggregator/src/tools/file_archiver/entities.rs

@@ -29,21 +29,23 @@ impl ArchiveParameters {
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct FileArchive {
     pub(super) filepath: PathBuf,
-    // Todo: rename to `archive_size` and add `uncompressed_size` field.
-    pub(super) filesize: u64,
+    pub(super) archive_filesize: u64,
+    pub(super) uncompressed_size: u64,
     pub(super) compression_algorithm: CompressionAlgorithm,
 }
 
 impl FileArchive {
     /// Create a new instance of FileArchive.
     pub fn new(
         filepath: PathBuf,
-        filesize: u64,
+        archive_filesize: u64,
+        uncompressed_size: u64,
         compression_algorithm: CompressionAlgorithm,
     ) -> Self {
         Self {
             filepath,
-            filesize,
+            archive_filesize,
+            uncompressed_size,
             compression_algorithm,
         }
     }
@@ -53,6 +55,7 @@ impl FileArchive {
         Self {
             filepath: PathBuf::from("archive.tar.gz"),
             archive_filesize: 10,
+            uncompressed_size: 789,
             compression_algorithm: CompressionAlgorithm::Gzip,
         }
     }
@@ -63,8 +66,13 @@ impl FileArchive {
     }
 
     /// Get the size of the archive.
-    pub fn get_file_size(&self) -> u64 {
-        self.filesize
+    pub fn get_archive_size(&self) -> u64 {
+        self.archive_filesize
+    }
+
+    /// Get the size of the data before compression.
+    pub fn get_uncompressed_size(&self) -> u64 {
+        self.uncompressed_size
     }
 
     /// Get the compression algorithm used to create the archive.