Merge pull request #1967 from carver/mine-batched-txns

Benchmarks: Mine transactions in a batch to show real import speed

Author: carver

eth/abc.py

@@ -480,6 +480,16 @@ class BlockAndMetaWitness(NamedTuple):
     meta_witness: MetaWitnessAPI
 
 
+class BlockPersistResult(NamedTuple):
+    """
+    After persisting a block into the active chain, this information
+    becomes available.
+    """
+    imported_block: BlockAPI
+    new_canonical_blocks: Tuple[BlockAPI, ...]
+    old_canonical_blocks: Tuple[BlockAPI, ...]
+
+
 class BlockImportResult(NamedTuple):
     """
     After importing and persisting a block into the active chain, this information
@@ -2785,9 +2795,9 @@ def import_block(self, block: BlockAPI) -> BlockAndMetaWitness:
         ...
 
     @abstractmethod
-    def mine_block(self, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
+    def mine_block(self, block: BlockAPI, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
         """
-        Mine the current block. Proxies to self.pack_block method.
+        Mine the given block. Proxies to self.pack_block method.
         """
         ...
 
@@ -3586,6 +3596,24 @@ def __init__(self, base_db: AtomicDatabaseAPI, header: BlockHeaderAPI = None) ->
         """
         ...
 
+    @abstractmethod
+    def mine_all(
+            self,
+            transactions: Sequence[SignedTransactionAPI],
+            *args: Any,
+            parent_header: BlockHeaderAPI = None,
+            **kwargs: Any,
+    ) -> Tuple[BlockImportResult, Tuple[ReceiptAPI, ...], Tuple[ComputationAPI, ...]]:
+        """
+        Build a block with the given transactions, and mine it.
+
+        Optionally, supply the parent block header to mine on top of.
+
+        This is much faster than individually running :meth:`apply_transaction`
+        and then :meth:`mine_block`.
+        """
+        ...
+
     @abstractmethod
     def apply_transaction(self,
                           transaction: SignedTransactionAPI

eth/chains/base.py

@@ -44,6 +44,7 @@
     AtomicDatabaseAPI,
     BlockHeaderAPI,
     BlockImportResult,
+    BlockPersistResult,
     ChainAPI,
     ChainDatabaseAPI,
     ConsensusContextAPI,
@@ -481,17 +482,27 @@ def import_block(self,
             except ValidationError:
                 self.logger.warning("Proposed %s doesn't follow EVM rules, rejecting...", block)
                 raise
-            self.validate_block(imported_block)
+
+        persist_result = self.persist_block(imported_block, perform_validation)
+        return BlockImportResult(*persist_result, block_result.meta_witness)
+
+    def persist_block(
+            self,
+            block: BlockAPI,
+            perform_validation: bool = True) -> BlockPersistResult:
+
+        if perform_validation:
+            self.validate_block(block)
 
         (
             new_canonical_hashes,
             old_canonical_hashes,
-        ) = self.chaindb.persist_block(imported_block)
+        ) = self.chaindb.persist_block(block)
 
         self.logger.debug(
-            'IMPORTED_BLOCK: number %s | hash %s',
-            imported_block.number,
-            encode_hex(imported_block.hash),
+            'Persisted block: number %s | hash %s',
+            block.number,
+            encode_hex(block.hash),
         )
 
         new_canonical_blocks = tuple(
@@ -505,11 +516,10 @@ def import_block(self,
             in old_canonical_hashes
         )
 
-        return BlockImportResult(
-            imported_block=imported_block,
+        return BlockPersistResult(
+            imported_block=block,
             new_canonical_blocks=new_canonical_blocks,
             old_canonical_blocks=old_canonical_blocks,
-            meta_witness=block_result.meta_witness,
         )
 
     #
@@ -667,11 +677,43 @@ def import_block(self,
         self.header = self.ensure_header()
         return result
 
+    def mine_all(
+            self,
+            transactions: Sequence[SignedTransactionAPI],
+            *args: Any,
+            parent_header: BlockHeaderAPI = None,
+            **kwargs: Any,
+    ) -> Tuple[BlockImportResult, Tuple[ReceiptAPI, ...], Tuple[ComputationAPI, ...]]:
+
+        if parent_header is None:
+            base_header = self.header
+        else:
+            base_header = self.create_header_from_parent(parent_header)
+
+        vm = self.get_vm(base_header)
+
+        new_header, receipts, computations = vm.apply_all_transactions(transactions, base_header)
+        filled_block = vm.set_block_transactions(vm.get_block(), new_header, transactions, receipts)
+
+        block_result = vm.mine_block(filled_block, *args, **kwargs)
+        imported_block = block_result.block
+
+        block_persist_result = self.persist_block(imported_block)
+        block_import_result = BlockImportResult(*block_persist_result, block_result.meta_witness)
+
+        self.header = self.create_header_from_parent(imported_block.header)
+        return (block_import_result, receipts, computations)
+
     def mine_block(self, *args: Any, **kwargs: Any) -> BlockAPI:
+        """
+        Mine whatever transactions have been incrementally applied so far.
+        """
         return self.mine_block_extended(*args, **kwargs).block
 
     def mine_block_extended(self, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
-        mine_result = self.get_vm(self.header).mine_block(*args, **kwargs)
+        vm = self.get_vm(self.header)
+        current_block = vm.get_block()
+        mine_result = vm.mine_block(current_block, *args, **kwargs)
         mined_block = mine_result.block
 
         self.validate_block(mined_block)

eth/tools/builder/chain/builders.py

@@ -366,10 +366,8 @@ def mine_block(chain: MiningChainAPI, **kwargs: Any) -> MiningChainAPI:
         raise ValidationError('`mine_block` may only be used on MiningChain instances')
 
     transactions = kwargs.pop('transactions', ())
-    for tx in transactions:
-        chain.apply_transaction(tx)
 
-    chain.mine_block(**kwargs)
+    chain.mine_all(transactions, **kwargs)
     return chain
 
 

eth/tools/factories/transaction.py

@@ -15,13 +15,16 @@ def new_transaction(
         gas_price=10,
         gas=100000,
         data=b'',
+        nonce=None,
         chain_id=None):
     """
     Create and return a transaction sending amount from <from_> to <to>.
 
     The transaction will be signed with the given private key.
     """
-    nonce = vm.state.get_nonce(from_)
+    if nonce is None:
+        nonce = vm.state.get_nonce(from_)
+
     tx = vm.create_unsigned_transaction(
         nonce=nonce,
         gas_price=gas_price,

eth/vm/base.py

@@ -305,17 +305,17 @@ def import_block(self, block: BlockAPI) -> BlockAndMetaWitness:
         # run all of the transactions.
         new_header, receipts, _ = self.apply_all_transactions(block.transactions, header)
 
-        self._block = self.set_block_transactions(
+        block_with_transactions = self.set_block_transactions(
             self.get_block(),
             new_header,
             block.transactions,
             receipts,
         )
 
-        return self.mine_block()
+        return self.mine_block(block_with_transactions)
 
-    def mine_block(self, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
-        packed_block = self.pack_block(self.get_block(), *args, **kwargs)
+    def mine_block(self, block: BlockAPI, *args: Any, **kwargs: Any) -> BlockAndMetaWitness:
+        packed_block = self.pack_block(block, *args, **kwargs)
 
         block_result = self.finalize_block(packed_block)
 

newsfragments/1967.performance.rst

@@ -0,0 +1,3 @@
+Got a >10x speedup of some benchmarks and other tests, by adding a new :meth`MiningChain.mine_all`
+API and using it. This is a public API, and should be used whenever all the transactions are known
+up front, to get a significant speedup.