Merge pull request #1600 from input-output-hk/ensemble/1594/aggregate-fake-aggregator-json-data

Aggregate fake aggregator reference json files

Author: Alenar

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-build-script"
-version = "0.1.4"
+version = "0.2.0"
 description = "A toolbox for Mithril crates build scripts"
 authors = { workspace = true }
 edition = { workspace = true }
@@ -11,4 +11,5 @@ include = ["**/*.rs", "Cargo.toml", "README.md", ".gitignore"]
 
 [dependencies]
 semver = "1.0.21"
+serde_json = "1.0.115"
 serde_yaml = "0.9.31"

internal/mithril-build-script/Cargo.toml

@@ -2,8 +2,11 @@ use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::fmt::Write as _;
 use std::fs;
+use std::fs::File;
 use std::path::Path;
 
+use serde_json;
+
 pub type ArtifactId = String;
 pub type FileContent = String;
 
@@ -29,11 +32,6 @@ pub struct FakeAggregatorData {
 
 impl FakeAggregatorData {
     pub fn load_from_folder(folder: &Path) -> Self {
-        fn extract_artifact_id(filename: &str, prefix: &str) -> String {
-            let id_with_extension = filename.strip_prefix(prefix).unwrap();
-            id_with_extension.strip_suffix(".json").unwrap().to_string()
-        }
-
         let mut data = FakeAggregatorData::default();
 
         for entry in list_json_files_in_folder(folder) {
@@ -49,41 +47,32 @@ impl FakeAggregatorData {
                 "epoch-settings.json" => {
                     data.epoch_settings = file_content;
                 }
-                "mithril-stake-distributions.json" => {
+                "mithril-stake-distributions-list.json" => {
                     data.msds_list = file_content;
                 }
-                "snapshots.json" => {
+                "snapshots-list.json" => {
                     data.snapshots_list = file_content;
                 }
-                "certificates.json" => {
+                "certificates-list.json" => {
                     data.certificates_list = file_content;
                 }
-                "ctx-snapshots.json" => {
+                "ctx-snapshots-list.json" => {
                     data.ctx_snapshots_list = file_content;
                 }
-                _ if filename.starts_with("mithril-stake-distribution-") => {
-                    data.individual_msds.insert(
-                        extract_artifact_id(&filename, "mithril-stake-distribution-"),
-                        file_content,
-                    );
+                "mithril-stake-distributions.json" => {
+                    data.individual_msds = Self::read_artifacts_json_file(&entry.path());
                 }
-                _ if filename.starts_with("snapshot-") => {
-                    data.individual_snapshots
-                        .insert(extract_artifact_id(&filename, "snapshot-"), file_content);
+                "snapshots.json" => {
+                    data.individual_snapshots = Self::read_artifacts_json_file(&entry.path());
                 }
-                _ if filename.starts_with("certificate-") => {
-                    data.individual_certificates
-                        .insert(extract_artifact_id(&filename, "certificate-"), file_content);
+                "certificates.json" => {
+                    data.individual_certificates = Self::read_artifacts_json_file(&entry.path());
                 }
-                _ if filename.starts_with("ctx-snapshot-") => {
-                    data.individual_ctx_snapshots.insert(
-                        extract_artifact_id(&filename, "ctx-snapshot-"),
-                        file_content,
-                    );
+                "ctx-snapshots.json" => {
+                    data.individual_ctx_snapshots = Self::read_artifacts_json_file(&entry.path());
                 }
-                _ if filename.starts_with("ctx-proof-") => {
-                    data.ctx_proofs
-                        .insert(extract_artifact_id(&filename, "ctx-proof-"), file_content);
+                "ctx-proofs.json" => {
+                    data.ctx_proofs = Self::read_artifacts_json_file(&entry.path());
                 }
                 // unknown file
                 _ => {}
@@ -177,9 +166,30 @@ impl FakeAggregatorData {
             )
         )
     }
+
+    fn read_artifacts_json_file(json_file: &Path) -> BTreeMap<ArtifactId, FileContent> {
+        let file = File::open(json_file).unwrap();
+        let parsed_json: serde_json::Value = serde_json::from_reader(&file).unwrap();
+
+        let json_object = parsed_json.as_object().unwrap();
+        let res: Result<Vec<_>, _> = json_object
+            .iter()
+            .map(|(key, value)| extract_artifact_id_and_content(key, value))
+            .collect();
+
+        BTreeMap::from_iter(res.unwrap())
+    }
 }
 
-pub fn list_json_files_in_folder(folder: &Path) -> impl Iterator<Item = std::fs::DirEntry> + '_ {
+fn extract_artifact_id_and_content(
+    key: &String,
+    value: &serde_json::Value,
+) -> Result<(ArtifactId, FileContent), String> {
+    let json_content = serde_json::to_string_pretty(value).map_err(|e| e.to_string())?;
+    Ok((key.to_owned(), json_content))
+}
+
+pub fn list_json_files_in_folder(folder: &Path) -> impl Iterator<Item = fs::DirEntry> + '_ {
     crate::list_files_in_folder(folder)
         .filter(|e| e.file_name().to_string_lossy().ends_with(".json"))
 }
@@ -253,6 +263,8 @@ pub fn generate_ids_array(array_name: &str, ids: BTreeSet<ArtifactId>) -> String
 
 #[cfg(test)]
 mod tests {
+    use crate::get_temp_dir;
+
     use super::*;
 
     #[test]
@@ -312,4 +324,38 @@ fn b() {}
             )
         )
     }
+
+    #[test]
+    fn parse_artifacts_json_into_btree_of_key_and_pretty_sub_json() {
+        let dir = get_temp_dir("read_artifacts_json_file");
+        let file = dir.join("test.json");
+        fs::write(
+            &file,
+            r#"{
+    "hash1": { "name": "artifact1" },
+    "hash2": { "name": "artifact2" }
+}"#,
+        )
+        .unwrap();
+
+        let id_per_json = FakeAggregatorData::read_artifacts_json_file(&file);
+
+        let expected = BTreeMap::from([
+            (
+                "hash1".to_string(),
+                r#"{
+  "name": "artifact1"
+}"#
+                .to_string(),
+            ),
+            (
+                "hash2".to_string(),
+                r#"{
+  "name": "artifact2"
+}"#
+                .to_string(),
+            ),
+        ]);
+        assert_eq!(expected, id_per_json);
+    }
 }

internal/mithril-build-script/src/fake_aggregator.rs

@@ -62,6 +62,23 @@ pub(crate) fn list_files_in_folder(folder: &Path) -> impl Iterator<Item = fs::Di
         })
 }
 
+#[cfg(test)]
+// Note: adding `mithril-common` as a dev-dependency would lead to a circular dependency, so
+// we can't use its `TempDir` api.
+pub(crate) fn get_temp_dir(dir_name: &str) -> PathBuf {
+    let dir = std::env::temp_dir()
+        .join("mithril_test")
+        .join("build_script")
+        .join(dir_name);
+
+    if dir.exists() {
+        fs::remove_dir_all(&dir).unwrap();
+    }
+    fs::create_dir_all(&dir).unwrap();
+
+    dir
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

internal/mithril-build-script/src/lib.rs

@@ -72,24 +72,9 @@ pub fn get_open_api_versions_mapping() -> HashMap<OpenAPIFileName, semver::Versi
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::get_temp_dir;
     use std::path::Path;
 
-    // Note: adding `mithril-common` as a dev-dependency would lead to a circular dependency, so
-    // we can't use its `TempDir` api.
-    fn get_temp_dir(dir_name: &str) -> PathBuf {
-        let dir = std::env::temp_dir()
-            .join("mithril_test")
-            .join("build_script")
-            .join(dir_name);
-
-        if dir.exists() {
-            fs::remove_dir_all(&dir).unwrap();
-        }
-        fs::create_dir_all(&dir).unwrap();
-
-        dir
-    }
-
     fn write_minimal_open_api_file(version: &str, path: &Path) {
         fs::write(
             path,

internal/mithril-build-script/src/open_api.rs

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-common"
-version = "0.3.23"
+version = "0.3.24"
 description = "Common types, interfaces, and utilities for Mithril nodes."
 authors = { workspace = true }
 edition = { workspace = true }
@@ -98,7 +98,7 @@ slog-term = "2.9.0"
 tokio = { version = "1.35.1", features = ["macros", "rt-multi-thread", "time"] }
 
 [build-dependencies]
-mithril-build-script = { path = "../internal/mithril-build-script", version = "=0.1.4" }
+mithril-build-script = { path = "../internal/mithril-build-script", version = "=0.2.0" }
 
 [features]
 default = []

mithril-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mithril-aggregator-fake"
-version = "0.2.5"
+version = "0.3.0"
 description = "Mithril Fake Aggregator for client testing"
 authors = { workspace = true }
 documentation = { workspace = true }