simulators/portal: update history tests to use new test data format (#1283)

Author: KolbyML

simulators/portal/Dockerfile

@@ -34,13 +34,14 @@ ARG PORTAL_CONSENSUS_AUTH
 ENV PORTAL_CONSENSUS_URL=$PORTAL_CONSENSUS_URL
 ENV PORTAL_CONSENSUS_AUTH=$PORTAL_CONSENSUS_AUTH
 
-RUN apt-get update && apt-get install -y wget
+RUN apt-get update && apt-get install -y wget git ca-certificates
 
 # copy build artifacts from build stage
 COPY --from=builder /portal/target/release/portal .
-ADD https://raw.githubusercontent.com/ethereum/portal-spec-tests/master/tests/mainnet/history/hive/test_data_collection_of_forks_blocks.yaml ./test-data/test_data_collection_of_forks_blocks.yaml
-ADD https://raw.githubusercontent.com/ethereum/portal-spec-tests/master/tests/mainnet/state/hive/test_data.yaml ./test-data/state_test_data.yaml
-ADD https://raw.githubusercontent.com/ethereum/portal-spec-tests/master/tests/mainnet/beacon_chain/hive/test_data.yaml ./test-data/beacon_test_data.yaml
+
+# clone portal-spec-tests repo. The add command is used to reset docker's cache so new data is pulled
+ADD "https://api.github.com/repos/ethereum/portal-spec-tests/commits?per_page=1" latest_commit
+RUN git clone https://github.com/ethereum/portal-spec-tests.git ./portal-spec-tests
 
 ENV RUST_LOG=debug,hyper_util=info,hyper=info,reqwest=info
 

simulators/portal/src/suites/beacon/constants.rs

@@ -3,7 +3,8 @@ use ethportal_api::{BeaconContentKey, BeaconContentValue, ContentValue};
 use serde_yaml::Value;
 use std::str::FromStr;
 
-pub const TEST_DATA_FILE_PATH: &str = "./test-data/beacon_test_data.yaml";
+pub const TEST_DATA_FILE_PATH: &str =
+    "./portal-spec-tests/tests/mainnet/beacon_chain/hive/test_data.yaml";
 
 pub fn get_test_data() -> anyhow::Result<Vec<(BeaconContentKey, BeaconContentValue)>> {
     let values = std::fs::read_to_string(TEST_DATA_FILE_PATH)?;

simulators/portal/src/suites/history/constants.rs

@@ -1,33 +1,51 @@
 use alloy_primitives::Bytes;
+use anyhow::bail;
 use ethportal_api::{ContentValue, HistoryContentKey, HistoryContentValue};
 use serde_yaml::Value;
 use std::str::FromStr;
 
-pub const TEST_DATA_FILE_PATH: &str = "./test-data/test_data_collection_of_forks_blocks.yaml";
-
 // trin-bridge constants
 pub const TRIN_BRIDGE_CLIENT_TYPE: &str = "trin-bridge";
 
+pub const TEST_DATA_FILE_PATH: &str = "./portal-spec-tests/tests/mainnet/history/hive/success";
+
 pub fn get_test_data() -> anyhow::Result<Vec<(HistoryContentKey, HistoryContentValue)>> {
-    let values = std::fs::read_to_string(TEST_DATA_FILE_PATH)?;
-    let values: Value = serde_yaml::from_str(&values)?;
-    values
-        .as_sequence()
-        .expect("unable to convert test data to sequence")
-        .iter()
-        .map(|value| {
-            let content_key: HistoryContentKey =
-                serde_yaml::from_value(value["content_key"].clone())?;
-            let raw_content_value = Bytes::from_str(
-                value["content_value"]
-                    .as_str()
-                    .expect("to find content value"),
-            )?;
-            let content_value =
-                HistoryContentValue::decode(&content_key, raw_content_value.as_ref())?;
-            Ok((content_key, content_value))
-        })
-        .collect()
+    let mut content = vec![];
+
+    for entry in std::fs::read_dir(TEST_DATA_FILE_PATH)? {
+        let entry = entry?;
+        let test_path = entry.path();
+        if !test_path.is_file() {
+            bail!("Expected a file, but found a directory: {test_path:?}");
+        }
+
+        let values = std::fs::read_to_string(test_path)?;
+        let values: Value = serde_yaml::from_str(&values)?;
+        let test_case = values
+            .as_sequence()
+            .expect("unable to convert test data to sequence")
+            .iter()
+            .map(|value| {
+                let content_key: HistoryContentKey =
+                    serde_yaml::from_value(value["content_key"].clone())
+                        .expect("to find content key");
+                let raw_content_value = Bytes::from_str(
+                    value["content_value"]
+                        .as_str()
+                        .expect("to find content value"),
+                )
+                .expect("unable to convert content value to bytes");
+                let content_value =
+                    HistoryContentValue::decode(&content_key, raw_content_value.as_ref())
+                        .expect("unable to decode content value");
+                (content_key, content_value)
+            })
+            .collect::<Vec<_>>();
+
+        content.extend(test_case);
+    }
+
+    Ok(content)
 }
 
 lazy_static::lazy_static! {

simulators/portal/src/suites/state/constants.rs

@@ -2,7 +2,7 @@ use alloy_primitives::Bytes;
 use ethportal_api::StateContentKey;
 use std::str::FromStr;
 
-pub const TEST_DATA_FILE_PATH: &str = "./test-data/state_test_data.yaml";
+pub const TEST_DATA_FILE_PATH: &str = "./portal-spec-tests/tests/mainnet/state/hive/test_data.yaml";
 
 // trin-bridge constants
 pub const TRIN_BRIDGE_CLIENT_TYPE: &str = "trin-bridge";