load da_committee from genesis file

Author: pls148

crates/builder/src/lib.rs

@@ -120,6 +120,7 @@ pub mod testing {
             let config: HotShotConfig<SeqTypes> = HotShotConfig {
                 num_nodes_with_stake: NonZeroUsize::new(num_nodes_with_stake).unwrap(),
                 known_da_nodes: known_nodes_with_stake.clone(),
+                da_committees: Default::default(),
                 known_nodes_with_stake: known_nodes_with_stake.clone(),
                 next_view_timeout: Duration::from_secs(5).as_millis() as u64,
                 num_bootstrap: 1usize,

crates/builder/src/non_permissioned.rs

@@ -66,7 +66,7 @@ pub fn build_instance_state<V: Versions>(
     let coordinator = EpochMembershipCoordinator::new(
         Arc::new(RwLock::new(EpochCommittees::new_stake(
             vec![],
-            vec![],
+            Default::default(),
             None,
             fetcher,
             genesis.epoch_height.unwrap_or_default(),

crates/hotshot/testing/src/test_builder.rs

@@ -70,6 +70,7 @@ pub fn default_hotshot_config<TYPES: NodeType>(
         start_threshold: (1, 1),
         num_nodes_with_stake: NonZeroUsize::new(known_nodes_with_stake.len()).unwrap(),
         known_da_nodes: known_da_nodes.clone(),
+        da_committees: Default::default(),
         num_bootstrap: num_bootstrap_nodes,
         known_nodes_with_stake: known_nodes_with_stake.clone(),
         da_staked_committee_size: known_da_nodes.len(),

crates/hotshot/types/src/hotshot_config_file.rs

@@ -4,7 +4,7 @@
 // You should have received a copy of the MIT License
 // along with the HotShot repository. If not, see <https://mit-license.org/>.
 
-use std::{num::NonZeroUsize, time::Duration};
+use std::{collections::BTreeMap, num::NonZeroUsize, time::Duration};
 
 use alloy::primitives::U256;
 use url::Url;
@@ -45,6 +45,8 @@ pub struct HotShotConfigFile<TYPES: NodeType> {
     #[serde(skip)]
     /// The known DA nodes' public key and stake values
     pub known_da_nodes: Vec<PeerConfig<TYPES>>,
+    /// The known DA nodes' public keys and stake values, by start epoch
+    pub da_committees: BTreeMap<u64, Vec<PeerConfig<TYPES>>>,
     /// Number of staking DA nodes
     pub staked_da_nodes: usize,
     /// Number of fixed leaders for GPU VID
@@ -89,6 +91,7 @@ impl<TYPES: NodeType> From<HotShotConfigFile<TYPES>> for HotShotConfig<TYPES> {
             start_threshold: val.start_threshold,
             num_nodes_with_stake: val.num_nodes_with_stake,
             known_da_nodes: val.known_da_nodes,
+            da_committees: val.da_committees,
             known_nodes_with_stake: val.known_nodes_with_stake,
             da_staked_committee_size: val.staked_da_nodes,
             fixed_leader_for_gpuvid: val.fixed_leader_for_gpuvid,
@@ -154,6 +157,7 @@ impl<TYPES: NodeType> HotShotConfigFile<TYPES> {
             known_nodes_with_stake: gen_known_nodes_with_stake,
             staked_da_nodes,
             known_da_nodes,
+            da_committees: Default::default(),
             fixed_leader_for_gpuvid: 1,
             next_view_timeout: 10000,
             view_sync_timeout: Duration::from_millis(1000),

crates/hotshot/types/src/lib.rs

@@ -5,7 +5,9 @@
 // along with the HotShot repository. If not, see <https://mit-license.org/>.
 
 //! Types and Traits for the `HotShot` consensus module
-use std::{fmt::Debug, future::Future, num::NonZeroUsize, pin::Pin, time::Duration};
+use std::{
+    collections::BTreeMap, fmt::Debug, future::Future, num::NonZeroUsize, pin::Pin, time::Duration,
+};
 
 use alloy::primitives::U256;
 use bincode::Options;
@@ -19,7 +21,7 @@ use traits::{
 use url::Url;
 use vec1::Vec1;
 
-use crate::utils::bincode_opts;
+use crate::{traits::node_implementation::ConsensusTime, utils::bincode_opts};
 pub mod bundle;
 pub mod consensus;
 pub mod constants;
@@ -195,6 +197,8 @@ pub struct HotShotConfig<TYPES: NodeType> {
     pub known_nodes_with_stake: Vec<PeerConfig<TYPES>>,
     /// All public keys known to be DA nodes
     pub known_da_nodes: Vec<PeerConfig<TYPES>>,
+    /// All public keys known to be DA nodes, by start epoch
+    pub da_committees: BTreeMap<u64, Vec<PeerConfig<TYPES>>>,
     /// List of DA committee (staking)nodes for static DA committee
     pub da_staked_committee_size: usize,
     /// Number of fixed leaders for GPU VID, normally it will be 0, it's only used when running GPU VID
@@ -266,4 +270,26 @@ impl<TYPES: NodeType> HotShotConfig<TYPES> {
     pub fn hotshot_stake_table(&self) -> HSStakeTable<TYPES> {
         self.known_nodes_with_stake.clone().into()
     }
+
+    pub fn build_da_committees(&self) -> BTreeMap<TYPES::Epoch, Vec<PeerConfig<TYPES>>> {
+        if self.da_committees.is_empty() {
+            tracing::warn!(
+                "da_committees is not set, falling back to known_da_nodes, which is deprecated."
+            );
+
+            [(TYPES::Epoch::new(0), self.known_da_nodes.clone())].into()
+        } else {
+            if !self.known_da_nodes.is_empty() {
+                tracing::warn!(
+                    "Both known_da_nodes and da_committees are set, known_da_nodes is deprecated \
+                     and will be ignored."
+                );
+            }
+
+            self.da_committees
+                .iter()
+                .map(|(k, v)| (TYPES::Epoch::new(*k), v.clone()))
+                .collect()
+        }
+    }
 }

hotshot-query-service/src/testing/consensus.rs

@@ -141,6 +141,7 @@ impl<D: DataSourceLifeCycle + UpdateStatusData, V: Versions> MockNetwork<D, V> {
             num_bootstrap: 0,
             da_staked_committee_size: pub_keys.len(),
             known_da_nodes: known_nodes_with_stake.clone(),
+            da_committees: Default::default(),
             data_request_delay: Duration::from_millis(200),
             view_sync_timeout: Duration::from_millis(250),
             start_threshold: (

sequencer/src/genesis.rs

@@ -6,9 +6,10 @@ use std::{
 use alloy::primitives::Address;
 use anyhow::{Context, Ok};
 use espresso_types::{
-    v0_3::ChainConfig, FeeAccount, FeeAmount, GenesisHeader, L1BlockInfo, L1Client, Timestamp,
-    Upgrade,
+    v0_3::ChainConfig, FeeAccount, FeeAmount, GenesisHeader, L1BlockInfo, L1Client, SeqTypes,
+    Timestamp, Upgrade,
 };
+use hotshot_types::traits::node_implementation::NodeType;
 use serde::{Deserialize, Serialize};
 use vbs::version::Version;
 
@@ -44,6 +45,48 @@ pub enum L1Finalized {
     Timestamp { timestamp: Timestamp },
 }
 
+/// Helper type to deal with TOML keys that are u64 but represented as strings
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct TomlKeyU64(u64);
+
+impl<'de> Deserialize<'de> for TomlKeyU64 {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        tracing::warn!("Using TomlKeyU64::deserialize");
+        let s = String::deserialize(deserializer)?;
+
+        let n = s
+            .parse::<u64>()
+            .map_err(|_| serde::de::Error::custom("invalid epoch"))?;
+
+        std::result::Result::Ok(TomlKeyU64(n))
+    }
+}
+
+impl Serialize for TomlKeyU64 {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        serializer.serialize_str(&self.0.to_string())
+    }
+}
+
+impl Into<u64> for &TomlKeyU64 {
+    fn into(self) -> u64 {
+        self.0
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct PeerConfigData {
+    pub stake_table_key: <SeqTypes as NodeType>::SignatureKey,
+    pub state_ver_key: <SeqTypes as NodeType>::StateSignatureKey,
+    pub stake: u64,
+}
+
 /// Genesis of an Espresso chain.
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct Genesis {
@@ -67,6 +110,7 @@ pub struct Genesis {
     #[serde(rename = "upgrade", with = "upgrade_ser")]
     #[serde(default)]
     pub upgrades: BTreeMap<Version, Upgrade>,
+    pub da_committees: Option<BTreeMap<TomlKeyU64, Vec<PeerConfigData>>>,
 }
 
 impl Genesis {

sequencer/src/lib.rs

@@ -57,6 +57,7 @@ use hotshot_types::{
     epoch_membership::EpochMembershipCoordinator,
     light_client::{StateKeyPair, StateSignKey},
     signature_key::{BLSPrivKey, BLSPubKey},
+    stake_table::StakeTableEntry,
     traits::{
         metrics::{Metrics, NoMetrics},
         network::ConnectedNetwork,
@@ -390,6 +391,27 @@ where
     network_config.config.epoch_start_block = epoch_start_block;
     network_config.config.stake_table_capacity = stake_table_capacity;
 
+    if let Some(da_committees) = &genesis.da_committees {
+        tracing::warn!("setting da_committees from genesis");
+        network_config.config.da_committees = da_committees
+            .iter()
+            .map(|(k, v)| {
+                (
+                    k.into(),
+                    v.iter()
+                        .map(|pcd| hotshot_types::PeerConfig {
+                            stake_table_entry: StakeTableEntry {
+                                stake_key: pcd.stake_table_key,
+                                stake_amount: U256::from(pcd.stake),
+                            },
+                            state_ver_key: pcd.state_ver_key.clone(),
+                        })
+                        .collect(),
+                )
+            })
+            .collect();
+    }
+
     // If the `Libp2p` bootstrap nodes were supplied via the command line, override those
     // present in the config file.
     if let Some(bootstrap_nodes) = network_params.libp2p_bootstrap_nodes {
@@ -529,7 +551,7 @@ where
     // Create the HotShot membership
     let mut membership = EpochCommittees::new_stake(
         network_config.config.known_nodes_with_stake.clone(),
-        network_config.config.known_da_nodes.clone(),
+        network_config.config.build_da_committees(),
         block_reward,
         fetcher,
         epoch_height,

types/src/v0/config.rs

@@ -1,4 +1,4 @@
-use std::{num::NonZeroUsize, time::Duration};
+use std::{collections::BTreeMap, num::NonZeroUsize, time::Duration};
 
 use hotshot_types::{
     network::{
@@ -55,6 +55,7 @@ pub struct PublicHotShotConfig {
     num_nodes_with_stake: NonZeroUsize,
     known_nodes_with_stake: Vec<PeerConfig<SeqTypes>>,
     known_da_nodes: Vec<PeerConfig<SeqTypes>>,
+    da_committees: BTreeMap<u64, Vec<PeerConfig<SeqTypes>>>,
     da_staked_committee_size: usize,
     fixed_leader_for_gpuvid: usize,
     next_view_timeout: u64,
@@ -105,6 +106,7 @@ impl From<HotShotConfig<SeqTypes>> for PublicHotShotConfig {
             num_nodes_with_stake,
             known_nodes_with_stake,
             known_da_nodes,
+            da_committees,
             da_staked_committee_size,
             fixed_leader_for_gpuvid,
             next_view_timeout,
@@ -133,6 +135,7 @@ impl From<HotShotConfig<SeqTypes>> for PublicHotShotConfig {
             num_nodes_with_stake,
             known_nodes_with_stake,
             known_da_nodes,
+            da_committees,
             da_staked_committee_size,
             fixed_leader_for_gpuvid,
             next_view_timeout,
@@ -165,6 +168,7 @@ impl PublicHotShotConfig {
             num_nodes_with_stake: self.num_nodes_with_stake,
             known_nodes_with_stake: self.known_nodes_with_stake,
             known_da_nodes: self.known_da_nodes,
+            da_committees: self.da_committees,
             da_staked_committee_size: self.da_staked_committee_size,
             fixed_leader_for_gpuvid: self.fixed_leader_for_gpuvid,
             next_view_timeout: self.next_view_timeout,
@@ -196,9 +200,15 @@ impl PublicHotShotConfig {
     pub fn known_da_nodes(&self) -> Vec<PeerConfig<SeqTypes>> {
         self.known_da_nodes.clone()
     }
+
+    pub fn da_committees(&self) -> BTreeMap<u64, Vec<PeerConfig<SeqTypes>>> {
+        self.da_committees.clone()
+    }
+
     pub fn blocks_per_epoch(&self) -> u64 {
         self.epoch_height
     }
+
     pub fn epoch_start_block(&self) -> u64 {
         self.epoch_start_block
     }

types/src/v0/impls/stake_table.rs

@@ -1689,11 +1689,18 @@ impl EpochCommittees {
         // TODO remove `new` from trait and rename this to `new`.
         // https://github.com/EspressoSystems/HotShot/commit/fcb7d54a4443e29d643b3bbc53761856aef4de8b
         committee_members: Vec<PeerConfig<SeqTypes>>,
-        da_members: Vec<PeerConfig<SeqTypes>>,
+        da_committees: BTreeMap<Epoch, Vec<PeerConfig<SeqTypes>>>,
         fixed_block_reward: Option<RewardAmount>,
         fetcher: Fetcher,
         epoch_height: u64,
     ) -> Self {
+        // TODO: Store da_committees in EpochCommittees. Currently we just pull da_members back out, but we
+        // will need to figure out how to integrate da_committees into EpochCommittee and the like.
+        let da_members = da_committees
+            .get(&Epoch::new(0))
+            .cloned()
+            .expect("Should always have an epoch 0 DA committee");
+
         // For each member, get the stake table entry
         let stake_table: Vec<_> = committee_members
             .iter()