Return witness metadata: node hashes, addresses, slots, bytecode

- only log trie node hashes present in DB
- import_block includes MetaWitness by default, mining requires calling
mine_block_extended(), to make for easier API changes
- tiny typo fix in release note: 1st and last 2 bytes, instead of 1st 3
- prefer frozenset when order doesn't matter
- Add new release notes for this feature

Author: carver

docs/guides/understanding_the_mining_process.rst

@@ -186,7 +186,8 @@ Now that we have the building blocks available, let's put it all together and mi
 
     # We have to finalize the block first in order to be able read the
     # attributes that are important for the PoW algorithm
-    block = chain.get_vm().finalize_block(chain.get_block())
+    block_result = chain.get_vm().finalize_block(chain.get_block())
+    block = block_result.block
 
     # based on mining_hash, block number and difficulty we can perform
     # the actual Proof of Work (PoW) mechanism to mine the correct
@@ -373,7 +374,8 @@ zero value transfer transaction.
   (<ByzantiumBlock(#Block #1...)
   >>> # We have to finalize the block first in order to be able read the
   >>> # attributes that are important for the PoW algorithm
-  >>> block = chain.get_vm().finalize_block(chain.get_block())
+  >>> block_result = chain.get_vm().finalize_block(chain.get_block())
+  >>> block = block_result.block
 
   >>> # based on mining_hash, block number and difficulty we can perform
   >>> # the actual Proof of Work (PoW) mechanism to mine the correct

eth/abc.py

@@ -8,6 +8,7 @@
     ClassVar,
     ContextManager,
     Dict,
+    FrozenSet,
     Iterable,
     Iterator,
     MutableMapping,
@@ -334,10 +335,53 @@ def is_genesis(self) -> bool:
         ...
 
 
+class MetaWitnessAPI(ABC):
+    @property
+    @abstractmethod
+    def hashes(self) -> FrozenSet[Hash32]:
+        ...
+
+    @property
+    @abstractmethod
+    def accounts_queried(self) -> FrozenSet[Address]:
+        ...
+
+    @property
+    @abstractmethod
+    def account_bytecodes_queried(self) -> FrozenSet[Address]:
+        ...
+
+    @abstractmethod
+    def get_slots_queried(self, address: Address) -> FrozenSet[int]:
+        ...
+
+    @property
+    @abstractmethod
+    def total_slots_queried(self) -> int:
+        """
+        Summed across all accounts, how many storage slots were queried?
+        """
+        ...
+
+
+class BlockAndMetaWitness(NamedTuple):
+    """
+    After evaluating a block using the VirtualMachine, this information
+    becomes available.
+    """
+    block: BlockAPI
+    meta_witness: MetaWitnessAPI
+
+
 class BlockImportResult(NamedTuple):
+    """
+    After importing and persisting a block into the active chain, this information
+    becomes available.
+    """
     imported_block: BlockAPI
     new_canonical_blocks: Tuple[BlockAPI, ...]
     old_canonical_blocks: Tuple[BlockAPI, ...]
+    meta_witness: MetaWitnessAPI
 
 
 class SchemaAPI(ABC):
@@ -1631,6 +1675,13 @@ def persist(self, db: DatabaseAPI) -> None:
         """
         ...
 
+    @abstractmethod
+    def get_accessed_slots(self) -> FrozenSet[int]:
+        """
+        List all the slots that had been accessed since object creation.
+        """
+        ...
+
 
 class AccountDatabaseAPI(ABC):
     """
@@ -1835,7 +1886,7 @@ def make_state_root(self) -> Hash32:
         ...
 
     @abstractmethod
-    def persist(self) -> None:
+    def persist(self) -> MetaWitnessAPI:
         """
         Send changes to underlying database, including the trie state
         so that it will forever be possible to read the trie from this checkpoint.
@@ -2192,7 +2243,7 @@ def lock_changes(self) -> None:
         ...
 
     @abstractmethod
-    def persist(self) -> None:
+    def persist(self) -> MetaWitnessAPI:
         """
         Persist the current state to the database.
         """
@@ -2490,14 +2541,14 @@ def make_receipt(self,
     # Mining
     #
     @abstractmethod
-    def import_block(self, block: BlockAPI) -> BlockAPI:
+    def import_block(self, block: BlockAPI) -> BlockAndMetaWitness:
         """
         Import the given block to the chain.
         """
         ...
 
     @abstractmethod
-    def mine_block(self, *args: Any, **kwargs: Any) -> BlockAPI:
+    def mine_block(self, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
         """
         Mine the current block. Proxies to self.pack_block method.
         """
@@ -2518,7 +2569,7 @@ def set_block_transactions(self,
     # Finalization
     #
     @abstractmethod
-    def finalize_block(self, block: BlockAPI) -> BlockAPI:
+    def finalize_block(self, block: BlockAPI) -> BlockAndMetaWitness:
         """
         Perform any finalization steps like awarding the block mining reward,
         and persisting the final state root.
@@ -3312,3 +3363,11 @@ def mine_block(self, *args: Any, **kwargs: Any) -> BlockAPI:
         See VM. :meth:`~eth.vm.base.VM.mine_block`
         """
         ...
+
+    @abstractmethod
+    def mine_block_extended(self, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
+        """
+        Just like :meth:`~mine_block`, but includes extra returned info. Currently,
+        the only extra info returned is the :class:`MetaWitness`.
+        """
+        ...

eth/chains/base.py

@@ -39,9 +39,11 @@
 )
 from eth.abc import (
     BlockAPI,
+    BlockAndMetaWitness,
     MiningChainAPI,
     AtomicDatabaseAPI,
     BlockHeaderAPI,
+    BlockImportResult,
     ChainAPI,
     ChainDatabaseAPI,
     ConsensusContextAPI,
@@ -51,7 +53,6 @@
     StateAPI,
     SignedTransactionAPI,
     UnsignedTransactionAPI,
-    BlockImportResult
 )
 from eth.consensus import (
     ConsensusContext,
@@ -467,7 +468,8 @@ def import_block(self,
             )
 
         base_header_for_import = self.create_header_from_parent(parent_header)
-        imported_block = self.get_vm(base_header_for_import).import_block(block)
+        block_result = self.get_vm(base_header_for_import).import_block(block)
+        imported_block = block_result.block
 
         # Validate the imported block.
         if perform_validation:
@@ -503,7 +505,8 @@ def import_block(self,
         return BlockImportResult(
             imported_block=imported_block,
             new_canonical_blocks=new_canonical_blocks,
-            old_canonical_blocks=old_canonical_blocks
+            old_canonical_blocks=old_canonical_blocks,
+            meta_witness=block_result.meta_witness,
         )
 
     #
@@ -662,13 +665,17 @@ def import_block(self,
         return result
 
     def mine_block(self, *args: Any, **kwargs: Any) -> BlockAPI:
-        mined_block = self.get_vm(self.header).mine_block(*args, **kwargs)
+        return self.mine_block_extended(*args, **kwargs).block
+
+    def mine_block_extended(self, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
+        mine_result = self.get_vm(self.header).mine_block(*args, **kwargs)
+        mined_block = mine_result.block
 
         self.validate_block(mined_block)
 
         self.chaindb.persist_block(mined_block)
         self.header = self.create_header_from_parent(mined_block.header)
-        return mined_block
+        return mine_result
 
     def get_vm(self, at_header: BlockHeaderAPI = None) -> VirtualMachineAPI:
         if at_header is None: