Add persist_unexecuted_block API to ChainDB

Author: cburgdorf

eth/abc.py

@@ -631,10 +631,35 @@ def persist_block(self,
             as genesis. Providing a ``genesis_parent_hash`` allows storage of blocks that
             aren't (yet) connected back to the true genesis header.
 
-        Assumes all block transactions have been persisted already.
+        .. warning::
+            This API assumes all block transactions have been persisted already. Use
+            :meth:`eth.abc.ChainDatabaseAPI.persist_unexecuted_block` to persist blocks that were
+            not executed.
         """
         ...
 
+    @abstractmethod
+    def persist_unexecuted_block(self,
+                                 block: BlockAPI,
+                                 receipts: Tuple[ReceiptAPI, ...],
+                                 genesis_parent_hash: Hash32 = None
+                                 ) -> Tuple[Tuple[Hash32, ...], Tuple[Hash32, ...]]:
+        """
+        Persist the given block's header, uncles, transactions, and receipts. Does
+        **not** validate if state transitions are valid.
+
+        :param block: the block that gets persisted
+        :param receipts: the receipts for the given block
+        :param genesis_parent_hash: *optional* parent hash of the header that is treated
+            as genesis. Providing a ``genesis_parent_hash`` allows storage of blocks that
+            aren't (yet) connected back to the true genesis header.
+
+        This API should be used to persist blocks that the EVM does not execute but which it
+        stores to make them available. It ensures to persist receipts and transactions which
+        :meth:`eth.abc.ChainDatabaseAPI.persist_block` in contrast assumes to be persisted
+        separately.
+        """
+
     @abstractmethod
     def persist_uncles(self, uncles: Tuple[BlockHeaderAPI]) -> Hash32:
         """

eth/db/chain.py

@@ -34,6 +34,7 @@
     EMPTY_UNCLE_HASH,
     GENESIS_PARENT_HASH,
 )
+from eth.db.trie import make_trie_root_and_nodes
 from eth.exceptions import (
     HeaderNotFound,
     ReceiptNotFound,
@@ -129,6 +130,34 @@ def persist_block(self,
         with self.db.atomic_batch() as db:
             return self._persist_block(db, block, genesis_parent_hash)
 
+    def persist_unexecuted_block(self,
+                                 block: BlockAPI,
+                                 receipts: Tuple[ReceiptAPI, ...],
+                                 genesis_parent_hash: Hash32 = GENESIS_PARENT_HASH
+                                 ) -> Tuple[Tuple[Hash32, ...], Tuple[Hash32, ...]]:
+
+        tx_root_hash, tx_kv_nodes = make_trie_root_and_nodes(block.transactions)
+
+        if tx_root_hash != block.header.transaction_root:
+            raise ValidationError(
+                f"Block's transaction_root ({block.header.transaction_root}) "
+                f"does not match expected value: {tx_root_hash}"
+            )
+
+        receipt_root_hash, receipt_kv_nodes = make_trie_root_and_nodes(receipts)
+
+        if receipt_root_hash != block.header.receipt_root:
+            raise ValidationError(
+                f"Block's receipt_root ({block.header.receipt_root}) "
+                f"does not match expected value: {receipt_root_hash}"
+            )
+
+        with self.db.atomic_batch() as db:
+            self._persist_trie_data_dict(db, receipt_kv_nodes)
+            self._persist_trie_data_dict(db, tx_kv_nodes)
+
+            return self._persist_block(db, block, genesis_parent_hash)
+
     @classmethod
     def _persist_block(
             cls,
@@ -347,6 +376,9 @@ def get(self, key: bytes) -> bytes:
         return self.db[key]
 
     def persist_trie_data_dict(self, trie_data_dict: Dict[Hash32, bytes]) -> None:
-        with self.db.atomic_batch() as db:
-            for key, value in trie_data_dict.items():
-                db[key] = value
+        self._persist_trie_data_dict(self.db, trie_data_dict)
+
+    @classmethod
+    def _persist_trie_data_dict(cls, db: DatabaseAPI, trie_data_dict: Dict[Hash32, bytes]) -> None:
+        for key, value in trie_data_dict.items():
+            db[key] = value

newsfragments/1925.feature.rst

@@ -0,0 +1,4 @@
+Add a new ``persist_unexecuted_block`` API to ``ChainDB``. This API should be used to persist
+a block without executing the EVM on it. The API is used by
+syncing strategies that do not execute all blocks but fill old blocks
+back in (e.g. ``beam`` or ``fast`` sync)

tests/conftest.py

@@ -242,6 +242,11 @@ def chain_without_block_validation(
     return _chain_without_block_validation(request, VM, base_db, genesis_state)
 
 
+@pytest.fixture(params=[Chain, MiningChain])
+def chain_without_block_validation_factory(request, VM, genesis_state):
+    return lambda db: _chain_without_block_validation(request, VM, db, genesis_state)
+
+
 @pytest.fixture(params=[Chain, MiningChain])
 def chain_without_block_validation_from_vm(request, base_db, genesis_state):
     """

tests/database/test_eth1_chaindb.py

@@ -15,6 +15,7 @@
 from eth.chains.base import (
     MiningChain,
 )
+from eth.db.atomic import AtomicDB
 from eth.db.chain import (
     ChainDB,
 )
@@ -149,30 +150,45 @@ def test_chaindb_get_canonical_block_hash(chaindb, block):
     assert block_hash == block.hash
 
 
-def test_chaindb_get_receipt_by_index(
-        chain,
-        funded_address,
-        funded_address_private_key):
-    NUMBER_BLOCKS_IN_CHAIN = 5
-    TRANSACTIONS_IN_BLOCK = 10
-    REQUIRED_BLOCK_NUMBER = 2
-    REQUIRED_RECEIPT_INDEX = 3
+def mine_blocks_with_receipts(chain,
+                              num_blocks,
+                              num_tx_per_block,
+                              funded_address,
+                              funded_address_private_key):
 
-    for block_number in range(NUMBER_BLOCKS_IN_CHAIN):
-        for tx_index in range(TRANSACTIONS_IN_BLOCK):
+    for _ in range(num_blocks):
+        block_receipts = []
+        for _ in range(num_tx_per_block):
             tx = new_transaction(
                 chain.get_vm(),
                 from_=funded_address,
                 to=force_bytes_to_address(b'\x10\x10'),
                 private_key=funded_address_private_key,
             )
             new_block, tx_receipt, computation = chain.apply_transaction(tx)
+            block_receipts.append(tx_receipt)
             computation.raise_if_error()
 
-            if (block_number + 1) == REQUIRED_BLOCK_NUMBER and tx_index == REQUIRED_RECEIPT_INDEX:
-                actual_receipt = tx_receipt
+        yield chain.mine_block(), block_receipts
+
 
-        chain.mine_block()
+def test_chaindb_get_receipt_and_tx_by_index(chain, funded_address, funded_address_private_key):
+    NUMBER_BLOCKS_IN_CHAIN = 5
+    TRANSACTIONS_IN_BLOCK = 10
+    REQUIRED_BLOCK_NUMBER = 2
+    REQUIRED_RECEIPT_INDEX = 3
+
+    for (block, receipts) in mine_blocks_with_receipts(
+            chain,
+            NUMBER_BLOCKS_IN_CHAIN,
+            TRANSACTIONS_IN_BLOCK,
+            funded_address,
+            funded_address_private_key,
+    ):
+        if block.header.block_number == REQUIRED_BLOCK_NUMBER:
+            actual_receipt = receipts[REQUIRED_RECEIPT_INDEX]
+            actual_tx = block.transactions[REQUIRED_RECEIPT_INDEX]
+            tx_class = block.transaction_class
 
     # Check that the receipt retrieved is indeed the actual one
     chaindb_retrieved_receipt = chain.chaindb.get_receipt_by_index(
@@ -181,6 +197,10 @@ def test_chaindb_get_receipt_by_index(
     )
     assert chaindb_retrieved_receipt == actual_receipt
 
+    chaindb_retrieved_tx = chain.chaindb.get_transaction_by_index(
+        REQUIRED_BLOCK_NUMBER, REQUIRED_RECEIPT_INDEX, tx_class)
+    assert chaindb_retrieved_tx == actual_tx
+
     # Raise error if block number is not found
     with pytest.raises(ReceiptNotFound):
         chain.chaindb.get_receipt_by_index(
@@ -194,3 +214,80 @@ def test_chaindb_get_receipt_by_index(
             NUMBER_BLOCKS_IN_CHAIN,
             TRANSACTIONS_IN_BLOCK + 1,
         )
+
+
+@pytest.mark.parametrize(
+    "use_persist_unexecuted_block",
+    (
+        True,
+        pytest.param(
+            False,
+            marks=pytest.mark.xfail(
+                reason=(
+                    "The `persist_block` API relies on block execution to persist"
+                    "transactions and receipts. It is expected to fail this test."
+                )
+            ),
+        ),
+    )
+)
+def test_chaindb_persist_unexecuted_block(chain,
+                                          chain_without_block_validation_factory,
+                                          funded_address,
+                                          funded_address_private_key,
+                                          use_persist_unexecuted_block):
+
+    # We need one chain to create blocks and a second one with a pristine database to test
+    # persisting blocks that have not been executed.
+    second_chain = chain_without_block_validation_factory(AtomicDB())
+    assert chain.get_canonical_head() == second_chain.get_canonical_head()
+    assert chain != second_chain
+
+    NUMBER_BLOCKS_IN_CHAIN = 5
+    TRANSACTIONS_IN_BLOCK = 10
+    REQUIRED_BLOCK_NUMBER = 2
+    REQUIRED_RECEIPT_INDEX = 3
+
+    for (block, receipts) in mine_blocks_with_receipts(
+            chain,
+            NUMBER_BLOCKS_IN_CHAIN,
+            TRANSACTIONS_IN_BLOCK,
+            funded_address,
+            funded_address_private_key,
+    ):
+        if block.header.block_number == REQUIRED_BLOCK_NUMBER:
+            actual_receipt = receipts[REQUIRED_RECEIPT_INDEX]
+            actual_tx = block.transactions[REQUIRED_RECEIPT_INDEX]
+            tx_class = block.transaction_class
+
+        if use_persist_unexecuted_block:
+            second_chain.chaindb.persist_unexecuted_block(block, receipts)
+        else:
+            # We just use this for an XFAIL to prove `persist_block` does not properly
+            # persist blocks that were not executed.
+            second_chain.chaindb.persist_block(block)
+
+    chaindb_retrieved_tx = second_chain.chaindb.get_transaction_by_index(
+        REQUIRED_BLOCK_NUMBER, REQUIRED_RECEIPT_INDEX, tx_class)
+    assert chaindb_retrieved_tx == actual_tx
+
+    # Check that the receipt retrieved is indeed the actual one
+    chaindb_retrieved_receipt = second_chain.chaindb.get_receipt_by_index(
+        REQUIRED_BLOCK_NUMBER,
+        REQUIRED_RECEIPT_INDEX,
+    )
+    assert chaindb_retrieved_receipt == actual_receipt
+
+    # Raise error if block number is not found
+    with pytest.raises(ReceiptNotFound):
+        second_chain.chaindb.get_receipt_by_index(
+            NUMBER_BLOCKS_IN_CHAIN + 1,
+            REQUIRED_RECEIPT_INDEX,
+        )
+
+    # Raise error if receipt index is out of range
+    with pytest.raises(ReceiptNotFound):
+        second_chain.chaindb.get_receipt_by_index(
+            NUMBER_BLOCKS_IN_CHAIN,
+            TRANSACTIONS_IN_BLOCK + 1,
+        )