simulators/portal: initialize clients during history tests with HistoricalSummaries (#1284)

Author: KolbyML

simulators/portal/Cargo.lock

@@ -10,14 +10,16 @@ alloy-rlp = "0.3.11"
 alloy-primitives = "1.1"
 anyhow = "1.0"
 chrono = "0.4"
-ethportal-api = { git = "https://github.com/ethereum/trin", rev = "743475f5ebac00c1359a7b61c45125b22efd841c" }
+ethereum_ssz = "0.9"
+ethportal-api = { git = "https://github.com/ethereum/trin", rev = "fd0b3c19c20c68059370a51d9b433ed34daa0591" }
 futures = "0.3.25"
 hivesim = { git = "https://github.com/ethereum/hive", rev = "4408fc1de3fee3ac23acd25a812c117756af2f39" }
 itertools = "0.14"
 lazy_static = "1.4.0"
 reqwest = { version = "0.12.7", features = ["json"] }
 serde_json = "1.0.95"
 serde_yaml = "0.9.33"
+snap = "1.1"
 ssz_types = "0.11"
 tokio = { version = "1.14.0", features = ["full"] }
 tracing = "0.1.37"

simulators/portal/Cargo.toml

@@ -2,8 +2,8 @@ use alloy_primitives::B256;
 use anyhow::{anyhow, Result};
 use ethportal_api::{
     light_client::{
-        bootstrap::LightClientBootstrapDeneb, finality_update::LightClientFinalityUpdateDeneb,
-        optimistic_update::LightClientOptimisticUpdateDeneb,
+        bootstrap::LightClientBootstrapElectra, finality_update::LightClientFinalityUpdateElectra,
+        optimistic_update::LightClientOptimisticUpdateElectra,
     },
     types::content_key::beacon::{
         LightClientBootstrapKey, LightClientFinalityUpdateKey, LightClientOptimisticUpdateKey,
@@ -82,7 +82,7 @@ impl ConsensusProvider {
         let content_key = BeaconContentKey::LightClientBootstrap(LightClientBootstrapKey {
             block_hash: block_hash.into(),
         });
-        let bootstrap: LightClientBootstrapDeneb = serde_json::from_value(data)?;
+        let bootstrap: LightClientBootstrapElectra = serde_json::from_value(data)?;
         let content_value = BeaconContentValue::LightClientBootstrap(bootstrap.into());
 
         Ok((content_key, content_value))
@@ -97,7 +97,7 @@ impl ConsensusProvider {
             self.base_url
         );
         let data = make_request(&self.client, &url).await?;
-        let update: LightClientFinalityUpdateDeneb = serde_json::from_value(data)?;
+        let update: LightClientFinalityUpdateElectra = serde_json::from_value(data)?;
         let new_finalized_slot = update.finalized_header.beacon.slot;
         let content_key = BeaconContentKey::LightClientFinalityUpdate(
             LightClientFinalityUpdateKey::new(new_finalized_slot),
@@ -116,7 +116,7 @@ impl ConsensusProvider {
             self.base_url
         );
         let data = make_request(&self.client, &url).await?;
-        let update: LightClientOptimisticUpdateDeneb = serde_json::from_value(data)?;
+        let update: LightClientOptimisticUpdateElectra = serde_json::from_value(data)?;
         let content_key = BeaconContentKey::LightClientOptimisticUpdate(
             LightClientOptimisticUpdateKey::new(update.signature_slot),
         );

simulators/portal/src/suites/beacon/bridge/provider.rs

@@ -1,18 +1,111 @@
 use alloy_primitives::Bytes;
-use anyhow::bail;
-use ethportal_api::{ContentValue, HistoryContentKey, HistoryContentValue};
+use anyhow::{anyhow, bail, ensure};
+use ethportal_api::{
+    types::{
+        content_key::beacon::HistoricalSummariesWithProofKey,
+        content_value::beacon::ForkVersionedHistoricalSummariesWithProof,
+    },
+    BeaconContentKey, BeaconContentValue, ContentValue, HistoryContentKey, HistoryContentValue,
+};
 use serde_yaml::Value;
-use std::str::FromStr;
+use ssz::Decode;
+use std::{
+    fs::{self, File},
+    io::Read,
+    path::PathBuf,
+    str::FromStr,
+    sync::{Arc, LazyLock},
+};
 
 // trin-bridge constants
 pub const TRIN_BRIDGE_CLIENT_TYPE: &str = "trin-bridge";
 
-pub const TEST_DATA_FILE_PATH: &str = "./portal-spec-tests/tests/mainnet/history/hive/success";
+pub const BASE_TEST_PATH: &str = "./portal-spec-tests/tests/mainnet/history/hive";
+const HISTORICAL_SUMMARIES_FILE_PREFIX: &str = "fork_digest_historical_summaries_";
+const HISTORICAL_SUMMARIES_FILE_SUFFIX: &str = ".ssz_snappy";
 
-pub fn get_test_data() -> anyhow::Result<Vec<(HistoryContentKey, HistoryContentValue)>> {
+/// The content key and value for the latest historical summaries
+///
+/// This is a static variable that is initialized once and can be reused
+pub static LATEST_HISTORICAL_SUMMARIES: LazyLock<Arc<(BeaconContentKey, BeaconContentValue)>> =
+    LazyLock::new(|| {
+        let (content_key, content_value) =
+            get_latest_historical_summaries().expect("Failed to load latest historical summaries");
+        Arc::new((content_key, content_value))
+    });
+
+/// Get the latest historical summaries
+///
+/// This should only be used in LATEST_HISTORICAL_SUMMARIES
+fn get_latest_historical_summaries() -> anyhow::Result<(BeaconContentKey, BeaconContentValue)> {
+    let mut latest_epoch = 0;
+    let mut latest_file: Option<PathBuf> = None;
+
+    for entry in fs::read_dir(BASE_TEST_PATH)? {
+        let entry = entry?;
+        if entry.path().is_dir() {
+            continue;
+        }
+        let file_name = entry.file_name();
+        let file_name_str = file_name.to_string_lossy();
+
+        if file_name_str.starts_with(HISTORICAL_SUMMARIES_FILE_PREFIX)
+            && file_name_str.ends_with(HISTORICAL_SUMMARIES_FILE_SUFFIX)
+        {
+            let epoch_str = &file_name_str[HISTORICAL_SUMMARIES_FILE_PREFIX.len()
+                ..file_name_str.len() - HISTORICAL_SUMMARIES_FILE_SUFFIX.len()];
+            if let Ok(epoch) = epoch_str.parse::<u64>() {
+                if epoch > latest_epoch {
+                    latest_epoch = epoch;
+                    latest_file = Some(entry.path());
+                }
+            } else {
+                bail!("Unable to parse epoch from file name: {file_name_str}");
+            }
+        }
+    }
+
+    let Some(path) = latest_file else {
+        bail!("No historical summaries files found in the directory");
+    };
+
+    let mut encoded = vec![];
+    let historical_summaries = File::open(path)?.read_to_end(&mut encoded)?;
+    ensure!(
+        historical_summaries > 0,
+        "Historical summaries file is empty",
+    );
+
+    let mut decoder = snap::read::FrameDecoder::new(&*encoded);
+    let mut decoded: Vec<u8> = vec![];
+    decoder.read_to_end(&mut decoded)?;
+
+    let historical_summaries_with_proof =
+        ForkVersionedHistoricalSummariesWithProof::from_ssz_bytes(&decoded).map_err(|err| {
+            anyhow!("Failed to decoded ForkVersionedHistoricalSummariesWithProof {err:?}")
+        })?;
+
+    let content_key =
+        BeaconContentKey::HistoricalSummariesWithProof(HistoricalSummariesWithProofKey {
+            epoch: historical_summaries_with_proof
+                .historical_summaries_with_proof
+                .epoch,
+        });
+    let content_value =
+        BeaconContentValue::HistoricalSummariesWithProof(historical_summaries_with_proof);
+
+    Ok((content_key, content_value))
+}
+
+pub fn get_success_test_data() -> anyhow::Result<Vec<(HistoryContentKey, HistoryContentValue)>> {
+    get_test_data("success")
+}
+
+fn get_test_data(test_case: &str) -> anyhow::Result<Vec<(HistoryContentKey, HistoryContentValue)>> {
     let mut content = vec![];
 
-    for entry in std::fs::read_dir(TEST_DATA_FILE_PATH)? {
+    let success_test_path = format!("{BASE_TEST_PATH}/{test_case}");
+    for entry in std::fs::read_dir(success_test_path)? {
         let entry = entry?;
         let test_path = entry.path();
         if !test_path.is_file() {