Introduce StateRootProviderExt and integrate StateRootProvider* with StateCommitment (#48)

* feat: introduce StateCommitment in StateProviders

* refactor: introduce StateCommimentProvider

* feat: introduce HashedPostStateProvider

* feat: HashedPostState from reverts

* feat: introduce HashedStorageProvider

* lint: revm/test-utils feature propogation

* fix: add Send + Sync bound on introduced storage state api methods

* feat: introduce KeyHasherProvider

* feat: introduce StateRootProviderExt and integrate it (and StateRootProvider) with StateCommitment

* fix: add merge files

* fix lint

* fix lint

* fmt

* add KeyHasher generic to DatabaseHashedStorage::from_reverts trait

* add merge files

* add merge files

* fix: propagate feature

* fix: merge conflicts

* reduce diff with upstream

* remove clone requirement in blockchain_tree state root calculation

Author: frisitano

Cargo.lock

@@ -54,8 +54,6 @@ reth-payload-primitives.workspace = true
 reth-payload-validator.workspace = true
 reth-basic-payload-builder.workspace = true
 reth-static-file.workspace = true
-reth-trie = { workspace = true, features = ["metrics"] }
-reth-trie-db = { workspace = true, features = ["metrics"] }
 reth-node-api.workspace = true
 reth-node-core.workspace = true
 reth-ethereum-payload-builder.workspace = true

bin/reth/Cargo.toml

@@ -43,8 +43,6 @@ use reth_transaction_pool::{
     blobstore::InMemoryBlobStore, BlobStore, EthPooledTransaction, PoolConfig, TransactionOrigin,
     TransactionPool, TransactionValidationTaskExecutor,
 };
-use reth_trie::StateRoot;
-use reth_trie_db::DatabaseStateRoot;
 use std::{path::PathBuf, str::FromStr, sync::Arc};
 use tracing::*;
 
@@ -273,10 +271,8 @@ impl<C: ChainSpecParser<ChainSpec = ChainSpec>> Command<C> {
                 debug!(target: "reth::cli", ?execution_outcome, "Executed block");
 
                 let hashed_post_state = state_provider.hashed_post_state(execution_outcome.state());
-                let (state_root, trie_updates) = StateRoot::overlay_root_with_updates(
-                    provider_factory.provider()?.tx_ref(),
-                    hashed_post_state.clone(),
-                )?;
+                let (state_root, trie_updates) =
+                    state_provider.state_root_from_state_with_updates(hashed_post_state.clone())?;
 
                 if state_root != block_with_senders.state_root {
                     eyre::bail!(

bin/reth/src/commands/debug_cmd/build_block.rs

@@ -25,14 +25,12 @@ use reth_primitives::BlockExt;
 use reth_provider::{
     providers::ProviderNodeTypes, AccountExtReader, ChainSpecProvider, DatabaseProviderFactory,
     HashedPostStateProvider, HashingWriter, HeaderProvider, LatestStateProviderRef,
-    OriginalValuesKnown, ProviderFactory, StageCheckpointReader, StateWriter, StorageLocation,
-    StorageReader,
+    OriginalValuesKnown, ProviderFactory, StageCheckpointReader, StateRootProvider,
+    StateRootProviderExt, StateWriter, StorageLocation, StorageReader,
 };
 use reth_revm::database::StateProviderDatabase;
 use reth_stages::StageId;
 use reth_tasks::TaskExecutor;
-use reth_trie::StateRoot;
-use reth_trie_db::DatabaseStateRoot;
 use std::{path::PathBuf, sync::Arc};
 use tracing::*;
 
@@ -164,10 +162,10 @@ impl<C: ChainSpecParser<ChainSpec = ChainSpec>> Command<C> {
         let execution_outcome = ExecutionOutcome::from((block_execution_output, block.number));
 
         // Unpacked `BundleState::state_root_slow` function
-        let (in_memory_state_root, in_memory_updates) = StateRoot::overlay_root_with_updates(
-            provider.tx_ref(),
-            state_provider.hashed_post_state(execution_outcome.state()),
-        )?;
+        let (in_memory_state_root, in_memory_updates) = state_provider
+            .state_root_from_state_with_updates(
+                state_provider.hashed_post_state(execution_outcome.state()),
+            )?;
 
         if in_memory_state_root == block.state_root {
             info!(target: "reth::cli", state_root = ?in_memory_state_root, "Computed in-memory state root matches");
@@ -195,10 +193,8 @@ impl<C: ChainSpecParser<ChainSpec = ChainSpec>> Command<C> {
         let accounts = provider_rw.basic_accounts(account_lists)?;
         provider_rw.insert_account_for_hashing(accounts)?;
 
-        let (state_root, incremental_trie_updates) = StateRoot::incremental_root_with_updates(
-            provider_rw.tx_ref(),
-            block.number..=block.number,
-        )?;
+        let (state_root, incremental_trie_updates) =
+            state_provider.incremental_state_root_with_updates(block.number..=block.number)?;
         if state_root != block.state_root {
             eyre::bail!(
                 "Computed incremental state root mismatch. Expected: {:?}. Got: {:?}",

bin/reth/src/commands/debug_cmd/in_memory_merkle.rs

@@ -24,7 +24,6 @@ reth-provider.workspace = true
 reth-execution-types.workspace = true
 reth-stages-api.workspace = true
 reth-trie = { workspace = true, features = ["metrics"] }
-reth-trie-db = { workspace = true, features = ["metrics"] }
 reth-trie-parallel.workspace = true
 reth-network.workspace = true
 reth-consensus.workspace = true
@@ -58,6 +57,7 @@ reth-provider = { workspace = true, features = ["test-utils"] }
 reth-evm = { workspace = true, features = ["test-utils"] }
 reth-consensus = { workspace = true, features = ["test-utils"] }
 reth-testing-utils.workspace = true
+reth-trie-db = { workspace = true, features = ["test-utils"] }
 reth-revm.workspace = true
 reth-evm-ethereum.workspace = true
 reth-execution-types.workspace = true

crates/blockchain-tree/Cargo.toml

@@ -25,12 +25,11 @@ use reth_provider::{
     BlockExecutionWriter, BlockNumReader, BlockWriter, CanonStateNotification,
     CanonStateNotificationSender, CanonStateNotifications, ChainSpecProvider, ChainSplit,
     ChainSplitTarget, DBProvider, DisplayBlocksChain, HashedPostStateProvider, HeaderProvider,
-    ProviderError, StaticFileProviderFactory, StorageLocation,
+    LatestStateProviderRef, ProviderError, StateRootProviderExt, StaticFileProviderFactory,
+    StorageLocation,
 };
 use reth_stages_api::{MetricEvent, MetricEventsSender};
 use reth_storage_errors::provider::{ProviderResult, RootMismatch};
-use reth_trie::{hashed_cursor::HashedPostStateCursorFactory, StateRoot};
-use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot};
 use std::{
     collections::{btree_map::Entry, BTreeMap, HashSet},
     sync::Arc,
@@ -1216,17 +1215,15 @@ where
     ) -> Result<(), CanonicalError> {
         let (blocks, state, chain_trie_updates) = chain.into_inner();
         let hashed_state = self.externals.provider_factory.hashed_post_state(state.state());
-        let prefix_sets = hashed_state.construct_prefix_sets().freeze();
-        let hashed_state_sorted = hashed_state.into_sorted();
 
         // Compute state root or retrieve cached trie updates before opening write transaction.
         let block_hash_numbers =
             blocks.iter().map(|(number, b)| (number, b.hash())).collect::<Vec<_>>();
-        let trie_updates = match chain_trie_updates {
+        let (trie_updates, hashed_state_sorted) = match chain_trie_updates {
             Some(updates) => {
                 debug!(target: "blockchain_tree", blocks = ?block_hash_numbers, "Using cached trie updates");
                 self.metrics.trie_updates_insert_cached.increment(1);
-                updates
+                (updates, hashed_state.into_sorted())
             }
             None => {
                 debug!(target: "blockchain_tree", blocks = ?block_hash_numbers, "Recomputing state root for insert");
@@ -1237,14 +1234,9 @@ where
                     // State root calculation can take a while, and we're sure no write transaction
                     // will be open in parallel. See https://github.com/paradigmxyz/reth/issues/6168.
                     .disable_long_read_transaction_safety();
-                let (state_root, trie_updates) = StateRoot::from_tx(provider.tx_ref())
-                    .with_hashed_cursor_factory(HashedPostStateCursorFactory::new(
-                        DatabaseHashedCursorFactory::new(provider.tx_ref()),
-                        &hashed_state_sorted,
-                    ))
-                    .with_prefix_sets(prefix_sets)
-                    .root_with_updates()
-                    .map_err(Into::<BlockValidationError>::into)?;
+                let (state_root, trie_updates, hashed_state_sorted) =
+                    LatestStateProviderRef::new(&provider)
+                        .state_root_from_state_with_updates_and_sorted_state(hashed_state)?;
                 let tip = blocks.tip();
                 if state_root != tip.state_root {
                     return Err(ProviderError::StateRootMismatch(Box::new(RootMismatch {
@@ -1255,7 +1247,7 @@ where
                     .into())
                 }
                 self.metrics.trie_updates_insert_recomputed.increment(1);
-                trie_updates
+                (trie_updates, hashed_state_sorted)
             }
         };
         recorder.record_relative(MakeCanonicalAction::RetrieveStateTrieUpdates);
@@ -1402,6 +1394,7 @@ mod tests {
     };
     use reth_stages_api::StageCheckpoint;
     use reth_trie::{root::state_root_unhashed, StateRoot};
+    use reth_trie_db::DatabaseStateRoot;
     use revm::AccountInfo;
     use std::collections::HashMap;
 

crates/blockchain-tree/src/blockchain_tree.rs

@@ -237,7 +237,7 @@ impl AppendableChain {
                 .map_err(ProviderError::from)?
             } else {
                 let hashed_state = provider.hashed_post_state(initial_execution_outcome.state());
-                let state_root = provider.state_root(hashed_state)?;
+                let state_root = provider.state_root_from_state(hashed_state)?;
                 (state_root, None)
             };
             if block.state_root != state_root {

crates/blockchain-tree/src/chain.rs

@@ -1027,15 +1027,15 @@ mod tests {
     }
 
     impl StateRootProvider for MockStateProvider {
-        fn state_root(&self, _hashed_state: HashedPostState) -> ProviderResult<B256> {
+        fn state_root_from_state(&self, _hashed_state: HashedPostState) -> ProviderResult<B256> {
             Ok(B256::random())
         }
 
         fn state_root_from_nodes(&self, _input: TrieInput) -> ProviderResult<B256> {
             Ok(B256::random())
         }
 
-        fn state_root_with_updates(
+        fn state_root_from_state_with_updates(
             &self,
             _hashed_state: HashedPostState,
         ) -> ProviderResult<(B256, TrieUpdates)> {

crates/chain-state/src/in_memory.rs

@@ -119,7 +119,7 @@ macro_rules! impl_state_provider {
         }
 
         impl $($tokens)* StateRootProvider for $type {
-            fn state_root(&self, state: HashedPostState) -> ProviderResult<B256> {
+            fn state_root_from_state(&self, state: HashedPostState) -> ProviderResult<B256> {
                 self.state_root_from_nodes(TrieInput::from_state(state))
             }
 
@@ -129,7 +129,7 @@ macro_rules! impl_state_provider {
                 self.historical.state_root_from_nodes(input)
             }
 
-            fn state_root_with_updates(
+            fn state_root_from_state_with_updates(
                 &self,
                 state: HashedPostState,
             ) -> ProviderResult<(B256, TrieUpdates)> {

crates/chain-state/src/memory_overlay.rs

@@ -45,8 +45,7 @@ reth-stages.workspace = true
 reth-stages-types = { workspace = true, optional = true }
 reth-static-file-types = { workspace = true, features = ["clap"] }
 reth-static-file.workspace = true
-reth-trie = { workspace = true, features = ["metrics"] }
-reth-trie-db = { workspace = true, features = ["metrics"] }
+reth-trie = { workspace = true, optional = true }
 reth-trie-common = { workspace = true, optional = true }
 
 # ethereum
@@ -113,6 +112,7 @@ arbitrary = [
     "reth-codecs?/arbitrary",
     "reth-prune-types?/arbitrary",
     "reth-stages-types?/arbitrary",
+    "reth-trie",
     "reth-trie-common?/arbitrary",
     "alloy-consensus/arbitrary",
 ]