Merge pull request #5 from Pana/feat/stateDump2

add logic to filter core space storage keys

Author: Pana

crates/client/src/state_dump.rs

@@ -10,7 +10,8 @@ use chrono::Utc;
 use keccak_hash::{keccak, KECCAK_EMPTY};
 use parking_lot::{Condvar, Mutex};
 use primitives::{
-    Account, SkipInputCheck, StorageKey, StorageKeyWithSpace, StorageValue,
+    Account, SkipInputCheck, SpaceStorageFilter, StorageKey,
+    StorageKeyWithSpace, StorageValue,
 };
 use rlp::Rlp;
 use std::{
@@ -293,7 +294,11 @@ pub fn export_space_accounts_with_callback<F: Fn(AccountState)>(
             }
         };
 
-        state.read_all_with_callback(start_key, &mut inner_callback)?;
+        state.read_all_with_callback(
+            start_key,
+            &mut inner_callback,
+            Some(SpaceStorageFilter::from(space)),
+        )?;
 
         if account_states.len() > 0 {
             println("Start to read account code and storage data...");

crates/dbs/statedb/src/lib.rs

@@ -12,7 +12,7 @@ pub mod global_params;
 mod in_memory_storage;
 mod statedb_ext;
 use cfx_types::H256;
-use primitives::StorageValue;
+use primitives::{SpaceStorageFilter, StorageValue};
 
 use cfx_db_errors::statedb as error;
 
@@ -191,9 +191,14 @@ mod impls {
         pub fn read_all_with_callback(
             &mut self, access_key_prefix: StorageKeyWithSpace,
             callback: &mut dyn FnMut(MptKeyValue),
+            space_storage_filter: Option<SpaceStorageFilter>,
         ) -> Result<()> {
             self.storage
-                .read_all_with_callback(access_key_prefix, callback)
+                .read_all_with_callback(
+                    access_key_prefix,
+                    callback,
+                    space_storage_filter,
+                )
                 .map_err(|err| err.into())
         }
 

crates/dbs/storage/src/impls/delta_mpt/cow_node_ref.rs

@@ -622,12 +622,19 @@ impl CowNodeRef {
         Ok(())
     }
 
+    // space_storage_filter can be used to filter the wanted space storage keys
     pub fn iterate_internal_with_callback(
         &self, owned_node_set: &OwnedNodeSet, trie: &DeltaMpt,
         guarded_trie_node: GuardedMaybeOwnedTrieNodeAsCowCallParam,
         key_prefix: CompressedPathRaw, db: &mut DeltaDbOwnedReadTraitObj,
-        callback: &mut dyn FnMut(MptKeyValue),
+        callback: &mut dyn FnMut(MptKeyValue), is_delta_mpt: bool,
+        space_storage_filter: Option<SpaceStorageFilter>,
     ) -> Result<()> {
+        if let Some(filter) = space_storage_filter {
+            if filter.is_filtered(is_delta_mpt, key_prefix.path_slice()) {
+                return Ok(());
+            }
+        }
         if guarded_trie_node.as_ref().as_ref().has_value() {
             assert!(CompressedPathRaw::has_second_nibble(
                 key_prefix.path_mask()
@@ -667,6 +674,8 @@ impl CowNodeRef {
                 key_prefix,
                 db,
                 callback,
+                is_delta_mpt,
+                space_storage_filter,
             )?;
         }
 
@@ -961,7 +970,7 @@ use super::{
     AtomicCommitTransaction, DeltaMpt, *,
 };
 use parking_lot::MutexGuard;
-use primitives::{MerkleHash, MptValue, MERKLE_NULL_NODE};
+use primitives::{MerkleHash, MptValue, SpaceStorageFilter, MERKLE_NULL_NODE};
 use rlp::*;
 use std::{
     borrow::BorrowMut, cell::Cell, convert::TryInto, ops::Deref,

crates/dbs/storage/src/impls/delta_mpt/subtrie_visitor.rs

@@ -636,7 +636,8 @@ impl<'trie, 'db: 'trie> SubTrieVisitor<'trie, 'db> {
     /// return all key/value pairs given the prefix
     pub fn traversal_with_callback(
         mut self, key: KeyPart, key_remaining: KeyPart,
-        callback: &mut dyn FnMut(MptKeyValue),
+        callback: &mut dyn FnMut(MptKeyValue), is_delta_mpt: bool,
+        space_storage_filter: Option<SpaceStorageFilter>,
     ) -> Result<()> {
         let node_memory_manager = self.node_memory_manager();
         let allocator = node_memory_manager.get_allocator();
@@ -678,7 +679,13 @@ impl<'trie, 'db: 'trie> SubTrieVisitor<'trie, 'db> {
             } => {
                 drop(trie_node_ref);
                 self.new_visitor_for_subtree(child_node.clone().into())
-                    .traversal_with_callback(key, key_remaining, callback)?;
+                    .traversal_with_callback(
+                        key,
+                        key_remaining,
+                        callback,
+                        is_delta_mpt,
+                        space_storage_filter,
+                    )?;
                 return Ok(());
             }
         }
@@ -691,6 +698,8 @@ impl<'trie, 'db: 'trie> SubTrieVisitor<'trie, 'db> {
             key_prefix,
             &mut *self.db.get_mut().to_owned_read()?,
             callback,
+            is_delta_mpt,
+            space_storage_filter,
         )?;
         Ok(())
     }
@@ -919,5 +928,5 @@ use super::{
     ChildrenTableDeltaMpt, DeltaMpt, *,
 };
 use parking_lot::MutexGuard;
-use primitives::{MerkleHash, MptValue, MERKLE_NULL_NODE};
+use primitives::{MerkleHash, MptValue, SpaceStorageFilter, MERKLE_NULL_NODE};
 use std::marker::PhantomData;

crates/dbs/storage/src/impls/recording_storage.rs

@@ -54,7 +54,7 @@ impl<Storage: StateTrait + StateTraitExt> StateTrait
             fn compute_state_root(&mut self) -> Result<StateRootWithAuxInfo>;
             fn get_state_root(&self) -> Result<StateRootWithAuxInfo>;
             fn commit(&mut self, epoch_id: EpochId) -> Result<StateRootWithAuxInfo>;
-            fn read_all_with_callback(&mut self, access_key_prefix: StorageKeyWithSpace, callback: &mut dyn FnMut(MptKeyValue)) -> Result<()>;
+            fn read_all_with_callback(&mut self, access_key_prefix: StorageKeyWithSpace, callback: &mut dyn FnMut(MptKeyValue), space_storage_filter: Option<SpaceStorageFilter>) -> Result<()>;
         }
     }
 
@@ -102,4 +102,6 @@ use crate::{
 use cfx_internal_common::StateRootWithAuxInfo;
 use delegate::delegate;
 use parking_lot::Mutex;
-use primitives::{CheckInput, EpochId, StorageKeyWithSpace};
+use primitives::{
+    CheckInput, EpochId, SpaceStorageFilter, StorageKeyWithSpace,
+};

crates/dbs/storage/src/impls/replicated_state.rs

@@ -6,7 +6,8 @@ use cfx_internal_common::StateRootWithAuxInfo;
 use cfx_types::Space;
 use parking_lot::Mutex;
 use primitives::{
-    EpochId, NodeMerkleTriplet, StaticBool, StorageKey, StorageKeyWithSpace,
+    EpochId, NodeMerkleTriplet, SpaceStorageFilter, StaticBool, StorageKey,
+    StorageKeyWithSpace,
 };
 use std::{
     sync::mpsc::{channel, Sender},
@@ -316,9 +317,13 @@ impl<Main: StateTrait> StateTrait for ReplicatedState<Main> {
     fn read_all_with_callback(
         &mut self, access_key_prefix: StorageKeyWithSpace,
         callback: &mut dyn FnMut(MptKeyValue),
+        space_storage_filter: Option<SpaceStorageFilter>,
     ) -> Result<()> {
-        self.state
-            .read_all_with_callback(access_key_prefix, callback)
+        self.state.read_all_with_callback(
+            access_key_prefix,
+            callback,
+            space_storage_filter,
+        )
     }
 
     fn compute_state_root(&mut self) -> Result<StateRootWithAuxInfo> {

crates/dbs/storage/src/impls/single_mpt_state.rs

@@ -7,8 +7,8 @@ use crate::{
 };
 use cfx_internal_common::{StateRootAuxInfo, StateRootWithAuxInfo};
 use primitives::{
-    EpochId, MerkleHash, MptValue, StateRoot, StorageKeyWithSpace,
-    MERKLE_NULL_NODE,
+    EpochId, MerkleHash, MptValue, SpaceStorageFilter, StateRoot,
+    StorageKeyWithSpace, MERKLE_NULL_NODE,
 };
 use rustc_hex::ToHex;
 use std::{cell::UnsafeCell, sync::Arc};
@@ -310,6 +310,7 @@ impl SingleMptState {
     fn read_all_with_callback_impl(
         &mut self, access_key_prefix: StorageKeyWithSpace,
         callback: &mut dyn FnMut(MptKeyValue),
+        space_storage_filter: Option<SpaceStorageFilter>,
     ) -> Result<()> {
         self.ensure_temp_slab_for_db_load();
 
@@ -340,6 +341,8 @@ impl SingleMptState {
             &key_prefix,
             &key_prefix,
             &mut inner_callback,
+            false,
+            space_storage_filter,
         )?;
 
         Ok(())
@@ -410,8 +413,13 @@ impl StateTrait for SingleMptState {
     fn read_all_with_callback(
         &mut self, access_key_prefix: StorageKeyWithSpace,
         callback: &mut dyn FnMut(MptKeyValue),
+        space_storage_filter: Option<SpaceStorageFilter>,
     ) -> Result<()> {
-        self.read_all_with_callback_impl(access_key_prefix, callback)
+        self.read_all_with_callback_impl(
+            access_key_prefix,
+            callback,
+            space_storage_filter,
+        )
     }
 
     fn compute_state_root(&mut self) -> Result<StateRootWithAuxInfo> {

crates/dbs/storage/src/impls/state.rs

@@ -312,8 +312,13 @@ impl StateTrait for State {
     fn read_all_with_callback(
         &mut self, access_key_prefix: StorageKeyWithSpace,
         callback: &mut dyn FnMut(MptKeyValue),
+        space_storage_filter: Option<SpaceStorageFilter>,
     ) -> Result<()> {
-        self.read_all_with_callback_impl(access_key_prefix, callback)
+        self.read_all_with_callback_impl(
+            access_key_prefix,
+            callback,
+            space_storage_filter,
+        )
     }
 
     fn compute_state_root(&mut self) -> Result<StateRootWithAuxInfo> {
@@ -990,6 +995,7 @@ impl State {
     pub fn read_all_with_callback_impl(
         &mut self, access_key_prefix: StorageKeyWithSpace,
         callback: &mut dyn FnMut(MptKeyValue),
+        space_storage_filter: Option<SpaceStorageFilter>,
     ) -> Result<()> {
         self.ensure_temp_slab_for_db_load();
 
@@ -1033,6 +1039,8 @@ impl State {
                 &delta_mpt_key_prefix,
                 &delta_mpt_key_prefix,
                 &mut inner_callback,
+                true,
+                space_storage_filter,
             )?;
         };
 
@@ -1073,6 +1081,8 @@ impl State {
                     &intermediate_mpt_key_prefix,
                     &intermediate_mpt_key_prefix,
                     &mut inner_callback,
+                    true,
+                    space_storage_filter,
                 )?;
             }
         }
@@ -1121,8 +1131,8 @@ use cfx_types::AddressWithSpace;
 use fallible_iterator::FallibleIterator;
 use primitives::{
     DeltaMptKeyPadding, EpochId, MerkleHash, MptValue, NodeMerkleTriplet,
-    SkipInputCheck, StateRoot, StaticBool, StorageKey, StorageKeyWithSpace,
-    StorageRoot, MERKLE_NULL_NODE, NULL_EPOCH,
+    SkipInputCheck, SpaceStorageFilter, StateRoot, StaticBool, StorageKey,
+    StorageKeyWithSpace, StorageRoot, MERKLE_NULL_NODE, NULL_EPOCH,
 };
 use rustc_hex::ToHex;
 use std::{

crates/dbs/storage/src/state.rs

@@ -39,9 +39,14 @@ pub trait StateTrait: Sync + Send {
         &mut self, access_key_prefix: StorageKeyWithSpace,
     ) -> Result<Option<Vec<MptKeyValue>>>;
 
+    /// space_storage_filter can be used to filter only the wanted contract
+    /// storage key if you specify SpaceStorageFilter::Native, it will
+    /// filter out the ethereum space storage key/value, by this way, we can
+    /// speedup space key/value traversal
     fn read_all_with_callback(
         &mut self, _access_key_prefix: StorageKeyWithSpace,
         _callback: &mut dyn FnMut(MptKeyValue),
+        _space_storage_filter: Option<SpaceStorageFilter>,
     ) -> Result<()> {
         Err(Error::Msg("Not implemented".into()))
     }
@@ -90,5 +95,6 @@ use super::{
 };
 use crate::StorageRootProof;
 use primitives::{
-    EpochId, NodeMerkleTriplet, StaticBool, StorageKeyWithSpace, StorageRoot,
+    EpochId, NodeMerkleTriplet, SpaceStorageFilter, StaticBool,
+    StorageKeyWithSpace, StorageRoot,
 };

crates/primitives/src/storage_key.rs

@@ -739,6 +739,62 @@ mod delta_mpt_storage_key {
     }
 }
 
+// This enum is used to filter only the wanted contract storage key when
+// traversal the trie for example, when traverse eth space key/value, we can
+// only filter the eSpace storage key/value to accelerate the traversal
+// speed
+// Native means filter(keep) the native space storage key/value
+// Ethereum means filter(keep) the ethereum space storage key/value
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct SpaceStorageFilter(pub Space);
+
+impl From<Space> for SpaceStorageFilter {
+    fn from(space: Space) -> Self { SpaceStorageFilter(space) }
+}
+
+impl From<SpaceStorageFilter> for Space {
+    fn from(filter: SpaceStorageFilter) -> Self { filter.0 }
+}
+
+impl SpaceStorageFilter {
+    pub fn is_native(&self) -> bool { matches!(self.0, Space::Native) }
+
+    pub fn is_ethereum(&self) -> bool { matches!(self.0, Space::Ethereum) }
+
+    // return the flag index according the trie type
+    // if is_delta_mpt is true, then the space flag is at the 32th index
+    // otherwise, the space flag is at the 20th index
+    fn flag_index(&self, is_delta_mpt: bool) -> usize {
+        if is_delta_mpt {
+            delta_mpt_storage_key::KEY_PADDING_BYTES
+        } else {
+            StorageKeyWithSpace::ACCOUNT_BYTES
+        }
+    }
+
+    // return true if the key is filtered out
+    pub fn is_filtered(&self, is_delta_mpt: bool, key: &[u8]) -> bool {
+        let flag_index = self.flag_index(is_delta_mpt);
+        if key.len() > flag_index {
+            match self.0 {
+                Space::Native => {
+                    if key[flag_index] == StorageKeyWithSpace::EVM_SPACE_TYPE[0]
+                    {
+                        return true;
+                    }
+                }
+                Space::Ethereum => {
+                    if key[flag_index] != StorageKeyWithSpace::EVM_SPACE_TYPE[0]
+                    {
+                        return true;
+                    }
+                }
+            }
+        }
+        false
+    }
+}
+
 use super::{MerkleHash, MERKLE_NULL_NODE};
 use crate::{
     hash::keccak, storage_key::delta_mpt_storage_key::ACCOUNT_KEYPART_BYTES,