Replace execution state hash with zeroes in high epochs (#4824)

## Motivation

Hashing gets slow when the execution state grows to large sizes.

## Proposal

Temporarily - for testnet only - skip the computation of the execution
state hash and return zeroes in high epochs (>20 in this PR).

## Test Plan

CI

## Release Plan

- Release a new SDK
- Update all validators
- Move to epoch 20 once that's done

## Links

- [reviewer
checklist](https://github.com/linera-io/linera-protocol/blob/main/CONTRIBUTING.md#reviewer-checklist)

Author: bart-linera

linera-chain/src/chain.rs

@@ -30,7 +30,7 @@ use linera_views::{
     reentrant_collection_view::{ReadGuardedView, ReentrantCollectionView},
     register_view::RegisterView,
     set_view::SetView,
-    views::{ClonableView, CryptoHashView, RootView, View},
+    views::{ClonableView, RootView, View},
 };
 use serde::{Deserialize, Serialize};
 use tracing::instrument;

linera-core/src/chain_worker/state.rs

@@ -716,6 +716,7 @@ where
         notify_when_messages_are_delivered: Option<oneshot::Sender<()>>,
     ) -> Result<(ChainInfoResponse, NetworkActions, BlockOutcome), WorkerError> {
         let block = certificate.block();
+        let block_hash = certificate.hash();
         let height = block.header.height;
         let chain_id = block.header.chain_id;
 
@@ -854,29 +855,28 @@ where
             .await?;
         let oracle_responses = Some(block.body.oracle_responses.clone());
         let (proposed_block, outcome) = block.clone().into_proposal();
-        let verified_outcome = if let Some(mut execution_state) =
-            self.execution_state_cache.remove(&outcome.state_hash)
-        {
-            chain.execution_state = execution_state
-                .with_context(|ctx| {
-                    chain
-                        .execution_state
-                        .context()
-                        .clone_with_base_key(ctx.base_key().bytes.clone())
-                })
-                .await;
-            outcome.clone()
-        } else {
-            chain
-                .execute_block(
-                    &proposed_block,
-                    local_time,
-                    None,
-                    &published_blobs,
-                    oracle_responses,
-                )
-                .await?
-        };
+        let verified_outcome =
+            if let Some(mut execution_state) = self.execution_state_cache.remove(&block_hash) {
+                chain.execution_state = execution_state
+                    .with_context(|ctx| {
+                        chain
+                            .execution_state
+                            .context()
+                            .clone_with_base_key(ctx.base_key().bytes.clone())
+                    })
+                    .await;
+                outcome.clone()
+            } else {
+                chain
+                    .execute_block(
+                        &proposed_block,
+                        local_time,
+                        None,
+                        &published_blobs,
+                        oracle_responses,
+                    )
+                    .await?
+            };
         // We should always agree on the messages and state hash.
         ensure!(
             outcome == verified_outcome,
@@ -1553,8 +1553,10 @@ where
                     .execute_block(block, local_time, round, published_blobs, None),
             )
             .await?;
+        let block = Block::new(block.clone(), outcome.clone());
+        let block_hash = CryptoHash::new(&block);
         self.execution_state_cache.insert_owned(
-            &outcome.state_hash,
+            &block_hash,
             self.chain
                 .execution_state
                 .with_context(|ctx| InactiveContext(ctx.base_key().clone()))

linera-core/src/unit_tests/wasm_worker_tests.rs

@@ -38,11 +38,7 @@ use linera_views::dynamo_db::DynamoDbDatabase;
 use linera_views::rocks_db::RocksDbDatabase;
 #[cfg(feature = "scylladb")]
 use linera_views::scylla_db::ScyllaDbDatabase;
-use linera_views::{
-    context::Context,
-    memory::MemoryDatabase,
-    views::{CryptoHashView, View},
-};
+use linera_views::{context::Context, memory::MemoryDatabase, views::View};
 use test_case::test_case;
 
 use super::TestEnvironment;

linera-core/src/unit_tests/worker_tests.rs

@@ -52,11 +52,8 @@ use linera_execution::{
 };
 use linera_storage::{DbStorage, Storage, TestClock};
 use linera_views::{
-    context::Context,
-    memory::MemoryDatabase,
-    random::generate_test_namespace,
-    store::TestKeyValueDatabase as _,
-    views::{CryptoHashView, RootView},
+    context::Context, memory::MemoryDatabase, random::generate_test_namespace,
+    store::TestKeyValueDatabase as _, views::RootView,
 };
 use test_case::test_case;
 use test_log::test;
@@ -706,7 +703,7 @@ where
         let value = ConfirmedBlock::new(
             BlockExecutionOutcome {
                 state_hash,
-                messages: vec![vec![]],
+                messages: vec![vec![direct_credit_message(chain_2, small_transfer)]],
                 oracle_responses: vec![vec![]],
                 events: vec![vec![]],
                 blobs: vec![vec![]],
@@ -4185,14 +4182,14 @@ where
 
     let value = ConfirmedBlock::new(
         BlockExecutionOutcome {
-            messages: vec![vec![]],
+            messages: vec![vec![direct_credit_message(chain_2, small_transfer)]],
             previous_message_blocks: BTreeMap::new(),
             previous_event_blocks: BTreeMap::new(),
             events: vec![vec![]],
             blobs: vec![vec![]],
             state_hash: state.crypto_hash_mut().await?,
             oracle_responses: vec![vec![]],
-            operation_results: vec![],
+            operation_results: vec![OperationResult::default()],
         }
         .with(block),
     );

linera-execution/src/execution.rs

@@ -5,6 +5,7 @@ use std::{collections::BTreeMap, vec};
 
 use futures::{FutureExt, StreamExt};
 use linera_base::{
+    crypto::CryptoHash,
     data_types::{BlobContent, BlockHeight, StreamUpdate},
     identifiers::{AccountOwner, BlobId, StreamId},
     time::Instant,
@@ -14,9 +15,10 @@ use linera_views::{
     key_value_store_view::KeyValueStoreView,
     map_view::MapView,
     reentrant_collection_view::HashedReentrantCollectionView,
-    views::{ClonableView, ReplaceContext, View},
+    views::{ClonableView, HashableView as _, ReplaceContext, View},
+    ViewError,
 };
-use linera_views_derive::CryptoHashView;
+use linera_views_derive::HashableView;
 #[cfg(with_testing)]
 use {
     crate::{
@@ -30,14 +32,14 @@ use {
 use super::{execution_state_actor::ExecutionRequest, runtime::ServiceRuntimeRequest};
 use crate::{
     execution_state_actor::ExecutionStateActor, resources::ResourceController,
-    system::SystemExecutionStateView, ApplicationDescription, ApplicationId, ExecutionError,
-    ExecutionRuntimeConfig, ExecutionRuntimeContext, MessageContext, OperationContext,
-    ProcessStreamsContext, Query, QueryContext, QueryOutcome, ServiceSyncRuntime, Timestamp,
-    TransactionTracker,
+    system::SystemExecutionStateView, ApplicationDescription, ApplicationId, BcsHashable,
+    Deserialize, ExecutionError, ExecutionRuntimeConfig, ExecutionRuntimeContext, MessageContext,
+    OperationContext, ProcessStreamsContext, Query, QueryContext, QueryOutcome, Serialize,
+    ServiceSyncRuntime, Timestamp, TransactionTracker, FLAG_ZERO_HASH,
 };
 
 /// A view accessing the execution state of a chain.
-#[derive(Debug, ClonableView, CryptoHashView)]
+#[derive(Debug, ClonableView, HashableView)]
 pub struct ExecutionStateView<C> {
     /// System application.
     pub system: SystemExecutionStateView<C>,
@@ -47,6 +49,34 @@ pub struct ExecutionStateView<C> {
     pub stream_event_counts: MapView<C, StreamId, u32>,
 }
 
+impl<C> ExecutionStateView<C>
+where
+    C: Context + Clone + Send + Sync + 'static,
+    C::Extra: ExecutionRuntimeContext,
+{
+    pub async fn crypto_hash_mut(&mut self) -> Result<CryptoHash, ViewError> {
+        if self
+            .system
+            .current_committee()
+            .is_some_and(|(_epoch, committee)| {
+                committee
+                    .policy()
+                    .http_request_allow_list
+                    .contains(FLAG_ZERO_HASH)
+            })
+        {
+            Ok(CryptoHash::from([0; 32]))
+        } else {
+            #[derive(Serialize, Deserialize)]
+            struct ExecutionStateViewHash([u8; 32]);
+            impl BcsHashable<'_> for ExecutionStateViewHash {}
+            self.hash_mut()
+                .await
+                .map(|hash| CryptoHash::new(&ExecutionStateViewHash(hash.into())))
+        }
+    }
+}
+
 impl<C: Context, C2: Context> ReplaceContext<C2> for ExecutionStateView<C> {
     type Target = ExecutionStateView<C2>;
 

linera-execution/src/lib.rs

@@ -85,6 +85,12 @@ pub const LINERA_TYPES_SOL: &str = include_str!("../solidity/LineraTypes.sol");
 /// The maximum length of a stream name.
 const MAX_STREAM_NAME_LEN: usize = 64;
 
+/// The flag that, if present in `http_request_allow_list` field of the content policy of
+/// current committee, causes the execution state not to be hashed, and instead the hash
+/// returned to be all zeros.
+// Note: testnet-only! This should not survive to mainnet.
+pub const FLAG_ZERO_HASH: &str = "FLAG_ZERO_HASH.linera.network";
+
 /// An implementation of [`UserContractModule`].
 #[derive(Clone)]
 pub struct UserContractCode(Box<dyn UserContractModule>);

linera-execution/src/test_utils/system_execution_state.rs

@@ -13,10 +13,7 @@ use linera_base::{
     identifiers::{AccountOwner, ApplicationId, BlobId, ChainId},
     ownership::ChainOwnership,
 };
-use linera_views::{
-    context::MemoryContext,
-    views::{CryptoHashView, View},
-};
+use linera_views::{context::MemoryContext, views::View};
 
 use super::{dummy_chain_description, dummy_committees, MockApplication, RegisterMockApplication};
 use crate::{

linera-execution/src/unit_tests/system_tests.rs

@@ -7,7 +7,10 @@ use linera_base::vm::VmRuntime;
 use linera_views::context::MemoryContext;
 
 use super::*;
-use crate::{test_utils::dummy_chain_description, ExecutionStateView, TestExecutionRuntimeContext};
+use crate::{
+    test_utils::dummy_chain_description, ExecutionStateView, TestExecutionRuntimeContext,
+    FLAG_ZERO_HASH,
+};
 
 /// Returns an execution state view and a matching operation context, for epoch 1, with root
 /// chain 0 as the admin ID and one empty committee.
@@ -129,3 +132,32 @@ async fn empty_accounts_are_removed() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+#[tokio::test]
+async fn hashing_test() -> anyhow::Result<()> {
+    let (mut view, _) = new_view_and_context().await;
+
+    let zero_hash = CryptoHash::from([0u8; 32]);
+
+    assert_ne!(view.crypto_hash_mut().await?, zero_hash);
+
+    let mut committees = BTreeMap::new();
+    committees.insert(Epoch(0), Committee::default());
+    view.system.committees.set(committees.clone());
+    view.system.epoch.set(Epoch(0));
+    // The hash should still be nonzero before the threshold epoch.
+    assert_ne!(view.crypto_hash_mut().await?, zero_hash);
+
+    let mut committee = Committee::default();
+    committee
+        .policy_mut()
+        .http_request_allow_list
+        .insert(FLAG_ZERO_HASH.to_owned());
+    committees.insert(Epoch(1), committee);
+    view.system.committees.set(committees);
+    view.system.epoch.set(Epoch(1));
+    // Starting from this epoch, the hash should be all zeros.
+    assert_eq!(view.crypto_hash_mut().await?, zero_hash);
+
+    Ok(())
+}

linera-service/src/cli_wrappers/wallet.rs

@@ -1078,6 +1078,17 @@ impl ClientWrapper {
         Ok(())
     }
 
+    pub async fn change_http_whitelist(&self, list: &[&str]) -> Result<()> {
+        self.command()
+            .await?
+            .arg("resource-control-policy")
+            .arg("--http-request-allow-list")
+            .arg(list.join(","))
+            .spawn_and_wait_for_stdout()
+            .await?;
+        Ok(())
+    }
+
     /// Runs `linera keygen`.
     pub async fn keygen(&self) -> Result<AccountOwner> {
         let stdout = self

linera-service/tests/local_net_tests.rs

@@ -22,6 +22,7 @@ use linera_base::{
     vm::VmRuntime,
 };
 use linera_core::{data_types::ChainInfoQuery, node::ValidatorNode};
+use linera_execution::FLAG_ZERO_HASH;
 use linera_sdk::linera_base_types::AccountSecretKey;
 use linera_service::{
     cli_wrappers::{
@@ -87,7 +88,7 @@ async fn test_end_to_end_reconfiguration(config: LocalNetConfig) -> Result<()> {
         .open_and_assign(&client_2, Amount::from_tokens(3))
         .await?;
     let port = get_node_port().await;
-    let node_service_2 = match network {
+    let mut node_service_2 = match network {
         Network::Grpc | Network::Grpcs => {
             Some(client_2.run_node_service(port, ProcessInbox::Skip).await?)
         }
@@ -237,7 +238,7 @@ async fn test_end_to_end_reconfiguration(config: LocalNetConfig) -> Result<()> {
         )
         .await?;
 
-    if let Some(mut service) = node_service_2 {
+    if let Some(service) = &mut node_service_2 {
         assert!(
             eventually(|| async { !service.process_inbox(&chain_2).await.unwrap().is_empty() })
                 .await
@@ -271,6 +272,54 @@ async fn test_end_to_end_reconfiguration(config: LocalNetConfig) -> Result<()> {
             .await?;
     }
 
+    // Test whether setting the zero hash flag works
+    client.change_http_whitelist(&[FLAG_ZERO_HASH]).await?;
+
+    if let Some(service) = &mut node_service_2 {
+        assert!(
+            eventually(|| async { !service.process_inbox(&chain_2).await.unwrap().is_empty() })
+                .await
+        );
+        let committees = service.query_committees(&chain_2).await?;
+        let epochs = committees.into_keys().collect::<Vec<_>>();
+        assert_eq!(&epochs, &[Epoch(3), Epoch(4)]);
+
+        service.ensure_is_running()?;
+    } else {
+        client_2.sync(chain_2).await?;
+        client_2.process_inbox(chain_2).await?;
+    }
+
+    // Check that we can still produce blocks
+    let recipient =
+        AccountOwner::from(AccountSecretKey::Secp256k1(Secp256k1SecretKey::generate()).public());
+    let account_recipient = Account::new(chain_2, recipient);
+    client
+        .transfer_with_accounts(
+            Amount::from_tokens(5),
+            Account::chain(chain_1),
+            account_recipient,
+        )
+        .await?;
+
+    if let Some(mut service) = node_service_2 {
+        assert!(
+            eventually(|| async { !service.process_inbox(&chain_2).await.unwrap().is_empty() })
+                .await
+        );
+        let balance = service.balance(&account_recipient).await?;
+        assert_eq!(balance, Amount::from_tokens(5));
+
+        service.ensure_is_running()?;
+    } else {
+        client_2.sync(chain_2).await?;
+        client_2.process_inbox(chain_2).await?;
+        assert_eq!(
+            client_2.local_balance(account_recipient).await?,
+            Amount::from_tokens(5),
+        );
+    }
+
     net.ensure_is_running().await?;
     net.terminate().await?;