feat(build-script): add generation of cdb list per epoch & csd per epoch

+ add epoch lists for cardano database (cdb), indexed by epoch number
+ refactor cardano stake distribution (csd) epoch list gen:
  - use a common mechanism with cdb
  - change index from str to epoch number

Author: Alenar

internal/mithril-build-script/src/fake_aggregator.rs

@@ -110,6 +110,11 @@ impl FakeAggregatorData {
     }
 
     pub fn generate_code_for_ids(self) -> String {
+        let cardano_stake_distributions_per_epoch =
+            extract_item_by_epoch(&self.individual_cardano_stake_distributions, "/epoch");
+        let cardano_database_snapshots_per_epoch =
+            extract_item_list_per_epoch(&self.cardano_database_snapshots_list, "/beacon/epoch");
+
         Self::assemble_code(
             &[
                 generate_ids_array(
@@ -128,16 +133,18 @@ impl FakeAggregatorData {
                         self.individual_cardano_stake_distributions.keys().cloned(),
                     ),
                 ),
-                generate_ids_array(
+                generate_epoch_array(
                     "cardano_stake_distribution_epochs",
-                    BTreeSet::from_iter(extract_cardano_stake_distribution_epochs(
-                        &self.individual_cardano_stake_distributions,
-                    )),
+                    BTreeSet::from_iter(cardano_stake_distributions_per_epoch.keys().cloned()),
                 ),
                 generate_ids_array(
                     "cardano_database_snapshot_hashes",
                     BTreeSet::from_iter(self.individual_cardano_database_snapshots.keys().cloned()),
                 ),
+                generate_epoch_array(
+                    "cardano_database_snapshot_epochs",
+                    BTreeSet::from_iter(cardano_database_snapshots_per_epoch.keys().cloned()),
+                ),
                 generate_ids_array(
                     "certificate_hashes",
                     BTreeSet::from_iter(self.individual_certificates.keys().cloned()),
@@ -158,6 +165,11 @@ impl FakeAggregatorData {
     }
 
     pub fn generate_code_for_all_data(self) -> String {
+        let cardano_stake_distributions_per_epoch =
+            extract_item_by_epoch(&self.individual_cardano_stake_distributions, "/epoch");
+        let cardano_database_snapshots_per_epoch =
+            extract_item_list_per_epoch(&self.cardano_database_snapshots_list, "/beacon/epoch");
+
         Self::assemble_code(
             &[
                 generate_list_getter("status", self.status),
@@ -188,11 +200,13 @@ impl FakeAggregatorData {
                         self.individual_cardano_stake_distributions.keys().cloned(),
                     ),
                 ),
-                generate_ids_array(
+                generate_epoch_array(
                     "cardano_stake_distribution_epochs",
-                    BTreeSet::from_iter(extract_cardano_stake_distribution_epochs(
-                        &self.individual_cardano_stake_distributions,
-                    )),
+                    BTreeSet::from_iter(cardano_stake_distributions_per_epoch.keys().cloned()),
+                ),
+                generate_artifact_per_epoch_getter(
+                    "cardano_stake_distributions_per_epoch",
+                    extract_item_by_epoch(&self.individual_cardano_stake_distributions, "/epoch"),
                 ),
                 generate_artifact_getter(
                     "cardano_stake_distributions",
@@ -210,6 +224,10 @@ impl FakeAggregatorData {
                     "cardano_database_snapshot_hashes",
                     BTreeSet::from_iter(self.individual_cardano_database_snapshots.keys().cloned()),
                 ),
+                generate_epoch_array(
+                    "cardano_database_snapshot_epochs",
+                    BTreeSet::from_iter(cardano_database_snapshots_per_epoch.keys().cloned()),
+                ),
                 generate_artifact_getter(
                     "cardano_database_snapshots",
                     self.individual_cardano_database_snapshots,
@@ -218,6 +236,10 @@ impl FakeAggregatorData {
                     "cardano_database_snapshot_list",
                     self.cardano_database_snapshots_list,
                 ),
+                generate_artifact_per_epoch_getter(
+                    "cardano_database_snapshot_list_per_epoch",
+                    cardano_database_snapshots_per_epoch,
+                ),
                 generate_artifact_getter("certificates", self.individual_certificates),
                 generate_list_getter("certificate_list", self.certificates_list),
                 generate_ids_array(
@@ -280,27 +302,6 @@ impl FakeAggregatorData {
     }
 }
 
-pub fn extract_cardano_stake_distribution_epochs(
-    individual_csds: &BTreeMap<ArtifactId, FileContent>,
-) -> Vec<String> {
-    individual_csds
-        .values()
-        .map(|content| {
-            let json_value: serde_json::Value =
-                serde_json::from_str(content).unwrap_or_else(|err| {
-                    panic!("Failed to parse JSON in csd content: {content}\nError: {err}");
-                });
-
-            json_value
-                .get("epoch")
-                .and_then(|epoch| epoch.as_u64().map(|s| s.to_string()))
-                .unwrap_or_else(|| {
-                    panic!("Epoch not found or invalid in csd content: {content}");
-                })
-        })
-        .collect()
-}
-
 fn extract_artifact_id_and_content(
     key: &String,
     value: &serde_json::Value,
@@ -309,6 +310,57 @@ fn extract_artifact_id_and_content(
     Ok((key.to_owned(), json_content))
 }
 
+/// Takes a map of json string indexed by hashes and re-indexes them using their epoch
+///
+/// Each item in the map must contain an epoch value at the specified JSON pointer location.
+pub fn extract_item_by_epoch(
+    items_per_hash: &BTreeMap<String, String>,
+    json_pointer_for_epoch: &str,
+) -> BTreeMap<u64, String> {
+    let mut res = BTreeMap::new();
+
+    for (key, value) in items_per_hash {
+        let parsed_json: serde_json::Value = serde_json::from_str(value)
+            .unwrap_or_else(|_| panic!("Could not parse JSON entity '{key}'"));
+        let epoch = parsed_json
+            .pointer(json_pointer_for_epoch)
+            .unwrap_or_else(|| panic!("missing `{json_pointer_for_epoch}` for JSON entity '{key}'"))
+            .as_u64()
+            .unwrap_or_else(|| {
+                panic!("`{json_pointer_for_epoch}` is not a number for JSON entity '{key}'")
+            });
+        res.insert(epoch, value.clone());
+    }
+
+    res
+}
+
+/// Takes a JSON string containing a list of items and extracts them into a map keyed by epoch.
+///
+/// Each item in the list must contain an epoch value at the specified JSON pointer location.
+pub fn extract_item_list_per_epoch(
+    source: &str,
+    json_pointer_for_epoch: &str,
+) -> BTreeMap<u64, String> {
+    let parsed_json: Vec<serde_json::Value> =
+        serde_json::from_str(source).expect("Failed to parse JSON list");
+    let mut list_per_epoch = BTreeMap::<u64, Vec<serde_json::Value>>::new();
+
+    for item in parsed_json {
+        let epoch = item
+            .pointer(json_pointer_for_epoch)
+            .unwrap_or_else(|| panic!("missing `{json_pointer_for_epoch}` for a json value"))
+            .as_u64()
+            .unwrap_or_else(|| panic!("`{json_pointer_for_epoch}` is not a number"));
+        list_per_epoch.entry(epoch).or_default().push(item);
+    }
+
+    list_per_epoch
+        .into_iter()
+        .map(|(k, v)| (k, serde_json::to_string(&v).unwrap()))
+        .collect()
+}
+
 pub fn list_json_files_in_folder(folder: &Path) -> impl Iterator<Item = fs::DirEntry> + '_ {
     crate::list_files_in_folder(folder)
         .filter(|e| e.file_name().to_string_lossy().ends_with(".json"))
@@ -344,6 +396,36 @@ pub fn generate_artifact_getter(
     )
 }
 
+// pub(crate) fn $fun_name()() -> BTreeMap<u64, String>
+pub fn generate_artifact_per_epoch_getter(
+    fun_name: &str,
+    source_jsons: BTreeMap<u64, FileContent>,
+) -> String {
+    let mut artifacts_list = String::new();
+
+    for (artifact_id, file_content) in source_jsons {
+        write!(
+            artifacts_list,
+            r###"
+        (
+            {artifact_id},
+            r#"{file_content}"#
+        ),"###
+        )
+        .unwrap();
+    }
+
+    format!(
+        r###"pub(crate) fn {fun_name}() -> BTreeMap<u64, String> {{
+    [{artifacts_list}
+    ]
+    .into_iter()
+    .map(|(k, v)| (k.to_owned(), v.to_owned()))
+    .collect()
+}}"###
+    )
+}
+
 /// pub(crate) fn $fun_name() -> &'static str
 pub fn generate_list_getter(fun_name: &str, source_json: FileContent) -> String {
     format!(
@@ -377,6 +459,30 @@ pub fn generate_ids_array(array_name: &str, ids: BTreeSet<ArtifactId>) -> String
     )
 }
 
+/// pub(crate) fn $array_name() -> [u64; $epoch.len]
+pub fn generate_epoch_array(array_name: &str, epoch: BTreeSet<u64>) -> String {
+    let mut ids_list = String::new();
+
+    for id in &epoch {
+        write!(
+            ids_list,
+            r#"
+        {id},"#
+        )
+        .unwrap();
+    }
+
+    format!(
+        r###"pub(crate) const fn {}() -> [u64; {}] {{
+    [{}
+    ]
+}}"###,
+        array_name,
+        epoch.len(),
+        ids_list,
+    )
+}
+
 #[cfg(test)]
 mod tests {
     use crate::get_temp_dir;
@@ -476,48 +582,105 @@ fn b() {}
     }
 
     #[test]
-    fn extract_csd_epochs_with_valid_data() {
-        let mut csds = BTreeMap::new();
-        csds.insert(
-            "csd-123".to_string(),
-            r#"{"hash": "csd-123", "epoch": 123}"#.to_string(),
-        );
-        csds.insert(
-            "csd-456".to_string(),
-            r#"{"hash": "csd-456", "epoch": 456}"#.to_string(),
-        );
-
-        let epochs = extract_cardano_stake_distribution_epochs(&csds);
+    fn test_extract_item_by_epoch_by_epoch_with_valid_data() {
+        let items_per_hash = BTreeMap::from([
+            (
+                "hash1".to_string(),
+                r#"{"bar":4,"epoch":3,"foo":"...","hash":"2"}"#.to_string(),
+            ),
+            (
+                "hash2".to_string(),
+                r#"{"bar":7,"epoch":2,"foo":"...","hash":"1"}"#.to_string(),
+            ),
+        ]);
 
-        assert_eq!(epochs, vec![123.to_string(), 456.to_string()]);
+        // note: values are not re-serialized, so they are kept as is
+        let item_per_epoch = extract_item_by_epoch(&items_per_hash, "/epoch");
+        assert_eq!(
+            BTreeMap::from([
+                (3, items_per_hash.get("hash1").unwrap().to_string()),
+                (2, items_per_hash.get("hash2").unwrap().to_string())
+            ]),
+            item_per_epoch
+        )
     }
 
     #[test]
-    #[should_panic(expected = "Failed to parse JSON in csd content")]
-    fn extract_csd_epochs_with_invalid_json() {
-        let mut csds = BTreeMap::new();
-        csds.insert(
+    #[should_panic(expected = "Could not parse JSON entity 'csd-123'")]
+    fn test_extract_item_by_epoch_by_epoch_with_invalid_json() {
+        let mut items_per_hash = BTreeMap::new();
+        items_per_hash.insert(
             "csd-123".to_string(),
             r#""hash": "csd-123", "epoch": "123"#.to_string(),
         );
 
-        extract_cardano_stake_distribution_epochs(&csds);
+        extract_item_by_epoch(&items_per_hash, "/epoch");
+    }
+
+    #[test]
+    #[should_panic(expected = "missing `/epoch` for JSON entity 'csd-123'")]
+    fn test_extract_item_by_epoch_with_missing_epoch() {
+        let mut items_per_hash = BTreeMap::new();
+        items_per_hash.insert("csd-123".to_string(), r#"{"hash": "csd-123"}"#.to_string());
+
+        extract_item_by_epoch(&items_per_hash, "/epoch");
+    }
+
+    #[test]
+    fn test_extract_item_by_epoch_with_empty_map() {
+        let items_per_hash = BTreeMap::new();
+
+        let epochs = extract_item_by_epoch(&items_per_hash, "/epoch");
+
+        assert!(epochs.is_empty());
+    }
+
+    #[test]
+    fn test_extract_item_list_per_epoch_for_epoch() {
+        let list_per_epoch_json = r#"[
+                { "beacon": { "epoch": 1, "bar": 4 }, "hash":"3","foo":"..." },
+                { "beacon": { "epoch": 2}, "hash":"2","foo":"..." },
+                { "beacon": { "epoch": 1}, "hash":"1","foo":"..." }
+            ]"#;
+
+        // note: values are re-serialized, so serde_json reorders the keys
+        let map_per_epoch = extract_item_list_per_epoch(list_per_epoch_json, "/beacon/epoch");
+        assert_eq!(
+            BTreeMap::from([
+                (
+                    1,
+                    r#"[{"beacon":{"bar":4,"epoch":1},"foo":"...","hash":"3"},{"beacon":{"epoch":1},"foo":"...","hash":"1"}]"#
+                        .to_string()
+                ),
+                (2, r#"[{"beacon":{"epoch":2},"foo":"...","hash":"2"}]"#.to_string()),
+            ]),
+            map_per_epoch
+        )
+    }
+
+    #[test]
+    #[should_panic(expected = "Failed to parse JSON list")]
+    fn test_extract_item_list_per_epoch_with_invalid_json() {
+        // invalid because of the trailing comma
+        let list_per_epoch_json =
+            r#"[ { "beacon": { "epoch": 1, "bar": 4 }, "hash":"3","foo":"..." }, ]"#;
+
+        extract_item_list_per_epoch(list_per_epoch_json, "/epoch");
     }
 
     #[test]
-    #[should_panic(expected = "Epoch not found or invalid in csd content")]
-    fn test_extract_csd_epochs_with_missing_epoch() {
-        let mut csds = BTreeMap::new();
-        csds.insert("csd-123".to_string(), r#"{"hash": "csd-123"}"#.to_string());
+    #[should_panic(expected = "missing `/epoch` for a json value")]
+    fn test_extract_item_list_per_epoch_with_missing_epoch() {
+        let list_per_epoch_json = r#"[ { "beacon": { "bar": 4 }, "hash":"3","foo":"..." } ]"#;
 
-        extract_cardano_stake_distribution_epochs(&csds);
+        extract_item_list_per_epoch(list_per_epoch_json, "/epoch");
     }
 
     #[test]
-    fn test_extract_csd_epochs_with_empty_map() {
-        let csds = BTreeMap::new();
+    fn test_extract_item_list_per_epoch_with_list() {
+        let list_per_epoch_json = "[]";
 
-        let epochs = extract_cardano_stake_distribution_epochs(&csds);
+        let epochs = extract_item_list_per_epoch(list_per_epoch_json, "/epoch");
 
         assert!(epochs.is_empty());
     }

mithril-client-wasm/src/client_wasm.rs

@@ -895,7 +895,7 @@ mod tests {
     #[wasm_bindgen_test]
     async fn get_cardano_stake_distribution_by_epoch_should_return_value_convertible_in_rust_type()
     {
-        let epoch: u64 = test_data::cardano_stake_distribution_epochs()[0].parse().unwrap();
+        let epoch = test_data::cardano_stake_distribution_epochs()[0];
         let csd_js_value = get_mithril_client_stable()
             .get_cardano_stake_distribution_by_epoch(epoch)
             .await

mithril-test-lab/mithril-aggregator-fake/src/application.rs

@@ -585,8 +585,8 @@ mod tests {
             yield_now().await;
 
             let path = "/artifact/cardano-stake-distribution/epoch/{epoch}";
-            let epoch = default_values::cardano_stake_distribution_epochs()[0];
-            let response = http_request(PORT, &path.replace("{epoch}", epoch)).await;
+            let epoch = default_values::cardano_stake_distribution_epochs()[0].to_string();
+            let response = http_request(PORT, &path.replace("{epoch}", &epoch)).await;
 
             APISpec::verify_conformity(
                 get_spec_file(),