Bump SSZ version for larger bitfield SmallVec (#6915)

NA


  Bumps the `ethereum_ssz` version, along with other crates that share the dep.

Primarily, this give us bitfields which can store 128 bytes on the stack before allocating, rather than 32 bytes (sigp/ethereum_ssz#38). The validator count has increase massively since we set it at 32 bytes, so aggregation bitfields (et al) now require a heap allocation. This new value of 128 should get us to ~2m active validators.

Author: paulhauner

Cargo.lock

@@ -137,8 +137,8 @@ discv5 = { version = "0.9", features = ["libp2p"] }
 env_logger = "0.9"
 ethereum_hashing = "0.7.0"
 ethereum_serde_utils = "0.7"
-ethereum_ssz = "0.7"
-ethereum_ssz_derive = "0.7"
+ethereum_ssz = "0.8.2"
+ethereum_ssz_derive = "0.8.2"
 ethers-core = "1"
 ethers-providers = { version = "1", default-features = false }
 exit-future = "0.2"
@@ -154,7 +154,7 @@ libsecp256k1 = "0.7"
 log = "0.4"
 lru = "0.12"
 maplit = "1"
-milhouse = "0.3"
+milhouse = "0.5"
 mockito = "1.5.0"
 num_cpus = "1"
 parking_lot = "0.12"
@@ -192,7 +192,7 @@ slog-term = "2"
 sloggers = { version = "2", features = ["json"] }
 smallvec = { version = "1.11.2", features = ["arbitrary"] }
 snap = "1"
-ssz_types = "0.8"
+ssz_types = "0.10"
 strum = { version = "0.24", features = ["derive"] }
 superstruct = "0.8"
 syn = "1"
@@ -211,8 +211,8 @@ tracing-appender = "0.2"
 tracing-core = "0.1"
 tracing-log = "0.2"
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
-tree_hash = "0.8"
-tree_hash_derive = "0.8"
+tree_hash = "0.9"
+tree_hash_derive = "0.9"
 url = "2"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 warp = { version = "0.3.7", default-features = false, features = ["tls"] }

Cargo.toml

@@ -2713,16 +2713,16 @@ where
         let mut block_hash_from_slot: HashMap<Slot, SignedBeaconBlockHash> = HashMap::new();
         let mut state_hash_from_slot: HashMap<Slot, BeaconStateHash> = HashMap::new();
         for slot in slots {
-            let (block_hash, new_state) = self
-                .add_attested_block_at_slot_with_sync(
-                    *slot,
-                    state,
-                    state_root,
-                    validators,
-                    sync_committee_strategy,
-                )
-                .await
-                .unwrap();
+            // Using a `Box::pin` to reduce the stack size. Clippy was raising a lints.
+            let (block_hash, new_state) = Box::pin(self.add_attested_block_at_slot_with_sync(
+                *slot,
+                state,
+                state_root,
+                validators,
+                sync_committee_strategy,
+            ))
+            .await
+            .unwrap();
 
             state = new_state;
 

beacon_node/beacon_chain/src/test_utils.rs

@@ -328,8 +328,7 @@ async fn test_rewards_base_multi_inclusion() {
         .extend_slots(E::slots_per_epoch() as usize * 2 - 4)
         .await;
 
-    // pin to reduce stack size for clippy
-    Box::pin(check_all_base_rewards(&harness, initial_balances)).await;
+    check_all_base_rewards(&harness, initial_balances).await;
 }
 
 #[tokio::test]
@@ -692,7 +691,8 @@ async fn check_all_base_rewards(
     harness: &BeaconChainHarness<EphemeralHarnessType<E>>,
     balances: Vec<u64>,
 ) {
-    check_all_base_rewards_for_subset(harness, balances, vec![]).await;
+    // The box reduces the size on the stack for a clippy lint.
+    Box::pin(check_all_base_rewards_for_subset(harness, balances, vec![])).await;
 }
 
 async fn check_all_base_rewards_for_subset(

beacon_node/beacon_chain/tests/rewards.rs

@@ -765,8 +765,8 @@ fn handle_rpc_response<E: EthSpec>(
         SupportedProtocol::PingV1 => Ok(Some(RpcSuccessResponse::Pong(Ping {
             data: u64::from_ssz_bytes(decoded_buffer)?,
         }))),
-        SupportedProtocol::MetaDataV1 => Ok(Some(RpcSuccessResponse::MetaData(MetaData::V1(
-            MetaDataV1::from_ssz_bytes(decoded_buffer)?,
+        SupportedProtocol::MetaDataV1 => Ok(Some(RpcSuccessResponse::MetaData(Arc::new(
+            MetaData::V1(MetaDataV1::from_ssz_bytes(decoded_buffer)?),
         )))),
         SupportedProtocol::LightClientBootstrapV1 => match fork_name {
             Some(fork_name) => Ok(Some(RpcSuccessResponse::LightClientBootstrap(Arc::new(
@@ -826,11 +826,11 @@ fn handle_rpc_response<E: EthSpec>(
             )),
         },
         // MetaData V2/V3 responses have no context bytes, so behave similarly to V1 responses
-        SupportedProtocol::MetaDataV3 => Ok(Some(RpcSuccessResponse::MetaData(MetaData::V3(
-            MetaDataV3::from_ssz_bytes(decoded_buffer)?,
+        SupportedProtocol::MetaDataV3 => Ok(Some(RpcSuccessResponse::MetaData(Arc::new(
+            MetaData::V3(MetaDataV3::from_ssz_bytes(decoded_buffer)?),
         )))),
-        SupportedProtocol::MetaDataV2 => Ok(Some(RpcSuccessResponse::MetaData(MetaData::V2(
-            MetaDataV2::from_ssz_bytes(decoded_buffer)?,
+        SupportedProtocol::MetaDataV2 => Ok(Some(RpcSuccessResponse::MetaData(Arc::new(
+            MetaData::V2(MetaDataV2::from_ssz_bytes(decoded_buffer)?),
         )))),
         SupportedProtocol::BlocksByRangeV2 => match fork_name {
             Some(ForkName::Altair) => Ok(Some(RpcSuccessResponse::BlocksByRange(Arc::new(
@@ -1008,6 +1008,7 @@ mod tests {
     ) -> SignedBeaconBlock<Spec> {
         let mut block: BeaconBlockBellatrix<_, FullPayload<Spec>> =
             BeaconBlockBellatrix::empty(&Spec::default_spec());
+
         let tx = VariableList::from(vec![0; 1024]);
         let txs = VariableList::from(std::iter::repeat(tx).take(5000).collect::<Vec<_>>());
 
@@ -1027,6 +1028,7 @@ mod tests {
     ) -> SignedBeaconBlock<Spec> {
         let mut block: BeaconBlockBellatrix<_, FullPayload<Spec>> =
             BeaconBlockBellatrix::empty(&Spec::default_spec());
+
         let tx = VariableList::from(vec![0; 1024]);
         let txs = VariableList::from(std::iter::repeat(tx).take(100000).collect::<Vec<_>>());
 
@@ -1105,28 +1107,31 @@ mod tests {
         Ping { data: 1 }
     }
 
-    fn metadata() -> MetaData<Spec> {
+    fn metadata() -> Arc<MetaData<Spec>> {
         MetaData::V1(MetaDataV1 {
             seq_number: 1,
             attnets: EnrAttestationBitfield::<Spec>::default(),
         })
+        .into()
     }
 
-    fn metadata_v2() -> MetaData<Spec> {
+    fn metadata_v2() -> Arc<MetaData<Spec>> {
         MetaData::V2(MetaDataV2 {
             seq_number: 1,
             attnets: EnrAttestationBitfield::<Spec>::default(),
             syncnets: EnrSyncCommitteeBitfield::<Spec>::default(),
         })
+        .into()
     }
 
-    fn metadata_v3() -> MetaData<Spec> {
+    fn metadata_v3() -> Arc<MetaData<Spec>> {
         MetaData::V3(MetaDataV3 {
             seq_number: 1,
             attnets: EnrAttestationBitfield::<Spec>::default(),
             syncnets: EnrSyncCommitteeBitfield::<Spec>::default(),
             custody_group_count: 1,
         })
+        .into()
     }
 
     /// Encodes the given protocol response as bytes.

beacon_node/lighthouse_network/src/rpc/codec.rs

@@ -578,7 +578,7 @@ pub enum RpcSuccessResponse<E: EthSpec> {
     Pong(Ping),
 
     /// A response to a META_DATA request.
-    MetaData(MetaData<E>),
+    MetaData(Arc<MetaData<E>>),
 }
 
 /// Indicates which response is being terminated by a stream termination response.

beacon_node/lighthouse_network/src/rpc/methods.rs

@@ -1190,7 +1190,7 @@ impl<E: EthSpec> Network<E> {
     ) {
         let metadata = self.network_globals.local_metadata.read().clone();
         // The encoder is responsible for sending the negotiated version of the metadata
-        let event = RpcResponse::Success(RpcSuccessResponse::MetaData(metadata));
+        let event = RpcResponse::Success(RpcSuccessResponse::MetaData(Arc::new(metadata)));
         self.eth2_rpc_mut()
             .send_response(peer_id, id, request_id, event);
     }
@@ -1605,7 +1605,7 @@ impl<E: EthSpec> Network<E> {
                     }
                     RpcSuccessResponse::MetaData(meta_data) => {
                         self.peer_manager_mut()
-                            .meta_data_response(&peer_id, meta_data);
+                            .meta_data_response(&peer_id, meta_data.as_ref().clone());
                         None
                     }
                     /* Network propagated protocols */