promote block-creation setting to user-facing (#19545)

* make the mempool block creation more configurable

* update tests to cover both block creation functions

Author: arvidn

benchmarks/mempool.py

@@ -243,7 +243,7 @@ async def add_spend_bundles(spend_bundles: list[SpendBundle]) -> None:
         with enable_profiler(True, f"create-{suffix}"):
             start = monotonic()
             for _ in range(10):
-                mempool.create_block_generator(last_tb_header_hash=rec.header_hash)
+                mempool.create_block_generator(rec.header_hash, 2.0)
             stop = monotonic()
         print(f"  time: {stop - start:0.4f}s")
         print(f"  per call: {(stop - start) / 10 * 1000:0.2f}ms")
@@ -252,7 +252,7 @@ async def add_spend_bundles(spend_bundles: list[SpendBundle]) -> None:
         with enable_profiler(True, f"create2-{suffix}"):
             start = monotonic()
             for _ in range(10):
-                mempool.create_block_generator2(last_tb_header_hash=rec.header_hash)
+                mempool.create_block_generator2(rec.header_hash, 2.0)
             stop = monotonic()
         print(f"  time: {stop - start:0.4f}s")
         print(f"  per call: {(stop - start) / 10 * 1000:0.2f}ms")

chia/_tests/core/full_node/test_full_node.py

@@ -12,6 +12,7 @@
 import pytest
 from chia_rs import (
     AugSchemeMPL,
+    ConsensusConstants,
     Foliage,
     FoliageTransactionBlock,
     FullBlock,
@@ -38,7 +39,11 @@
 from chia._tests.core.make_block_generator import make_spend_bundle
 from chia._tests.core.node_height import node_height_at_least
 from chia._tests.util.misc import wallet_height_at_least
-from chia._tests.util.setup_nodes import OldSimulatorsAndWallets, SimulatorsAndWalletsServices
+from chia._tests.util.setup_nodes import (
+    OldSimulatorsAndWallets,
+    SimulatorsAndWalletsServices,
+    setup_simulators_and_wallets,
+)
 from chia._tests.util.time_out_assert import time_out_assert, time_out_assert_custom_interval, time_out_messages
 from chia.consensus.augmented_chain import AugmentedBlockchain
 from chia.consensus.block_body_validation import ForkInfo
@@ -2912,87 +2917,97 @@ async def test_eviction_from_bls_cache(one_node_one_block: tuple[FullNodeSimulat
 
 
 @pytest.mark.limit_consensus_modes(allowed=[ConsensusMode.HARD_FORK_2_0], reason="irrelevant")
+@pytest.mark.parametrize("block_creation", [0, 1, 2])
 @pytest.mark.anyio
 async def test_declare_proof_of_space_no_overflow(
-    wallet_nodes: tuple[
-        FullNodeSimulator, FullNodeSimulator, ChiaServer, ChiaServer, WalletTool, WalletTool, BlockTools
-    ],
+    blockchain_constants: ConsensusConstants,
     self_hostname: str,
-    bt: BlockTools,
+    block_creation: int,
 ) -> None:
-    full_node_api, _, server_1, _, _, _, _ = wallet_nodes
-    wallet = WalletTool(test_constants)
-    coinbase_puzzlehash = wallet.get_new_puzzlehash()
-    blocks = bt.get_consecutive_blocks(
-        num_blocks=10,
-        skip_overflow=True,
-        force_overflow=False,
-        farmer_reward_puzzle_hash=coinbase_puzzlehash,
-        guarantee_transaction_block=True,
-    )
-    await add_blocks_in_batches(blocks, full_node_api.full_node)
-    _, dummy_node_id = await add_dummy_connection(server_1, self_hostname, 12312)
-    dummy_peer = server_1.all_connections[dummy_node_id]
-    assert full_node_api.full_node.blockchain.get_peak_height() == blocks[-1].height
-    for i in range(10, 100):
-        sb = await add_tx_to_mempool(
-            full_node_api, wallet, blocks[-8], coinbase_puzzlehash, bytes32(i.to_bytes(32, "big")), uint64(i)
-        )
+    async with setup_simulators_and_wallets(
+        1, 1, blockchain_constants, config_overrides={"full_node.block_creation": block_creation}
+    ) as new:
+        full_node_api = new.simulators[0].peer_api
+        server_1 = full_node_api.full_node.server
+        bt = new.bt
+
+        wallet = WalletTool(test_constants)
+        coinbase_puzzlehash = wallet.get_new_puzzlehash()
         blocks = bt.get_consecutive_blocks(
-            block_list_input=blocks,
-            num_blocks=1,
+            num_blocks=10,
+            skip_overflow=True,
+            force_overflow=False,
             farmer_reward_puzzle_hash=coinbase_puzzlehash,
             guarantee_transaction_block=True,
-            transaction_data=sb,
         )
-        block = blocks[-1]
-        unfinised_block = await declare_pos_unfinished_block(full_node_api, dummy_peer, block)
-        compare_unfinished_blocks(unfinished_from_full_block(block), unfinised_block)
-        await full_node_api.full_node.add_block(block)
-        assert full_node_api.full_node.blockchain.get_peak_height() == block.height
+        await add_blocks_in_batches(blocks, full_node_api.full_node)
+        _, dummy_node_id = await add_dummy_connection(server_1, self_hostname, 12312)
+        dummy_peer = server_1.all_connections[dummy_node_id]
+        assert full_node_api.full_node.blockchain.get_peak_height() == blocks[-1].height
+        for i in range(10, 100):
+            sb = await add_tx_to_mempool(
+                full_node_api, wallet, blocks[-8], coinbase_puzzlehash, bytes32(i.to_bytes(32, "big")), uint64(i)
+            )
+            blocks = bt.get_consecutive_blocks(
+                block_list_input=blocks,
+                num_blocks=1,
+                farmer_reward_puzzle_hash=coinbase_puzzlehash,
+                guarantee_transaction_block=True,
+                transaction_data=sb,
+            )
+            block = blocks[-1]
+            unfinised_block = await declare_pos_unfinished_block(full_node_api, dummy_peer, block)
+            compare_unfinished_blocks(unfinished_from_full_block(block), unfinised_block)
+            await full_node_api.full_node.add_block(block)
+            assert full_node_api.full_node.blockchain.get_peak_height() == block.height
 
 
 @pytest.mark.limit_consensus_modes(allowed=[ConsensusMode.HARD_FORK_2_0], reason="irrelevant")
+@pytest.mark.parametrize("block_creation", [0, 1, 2])
 @pytest.mark.anyio
 async def test_declare_proof_of_space_overflow(
-    wallet_nodes: tuple[
-        FullNodeSimulator, FullNodeSimulator, ChiaServer, ChiaServer, WalletTool, WalletTool, BlockTools
-    ],
+    blockchain_constants: ConsensusConstants,
     self_hostname: str,
-    bt: BlockTools,
+    block_creation: int,
 ) -> None:
-    full_node_api, _, server_1, _, _, _, _ = wallet_nodes
-    wallet = WalletTool(test_constants)
-    coinbase_puzzlehash = wallet.get_new_puzzlehash()
-    blocks = bt.get_consecutive_blocks(
-        num_blocks=10,
-        farmer_reward_puzzle_hash=coinbase_puzzlehash,
-        guarantee_transaction_block=True,
-    )
-    await add_blocks_in_batches(blocks, full_node_api.full_node)
-    _, dummy_node_id = await add_dummy_connection(server_1, self_hostname, 12312)
-    dummy_peer = server_1.all_connections[dummy_node_id]
-    assert full_node_api.full_node.blockchain.get_peak_height() == blocks[-1].height
-    for i in range(10, 100):
-        sb = await add_tx_to_mempool(
-            full_node_api, wallet, blocks[-8], coinbase_puzzlehash, bytes32(i.to_bytes(32, "big")), uint64(i)
-        )
-
+    async with setup_simulators_and_wallets(
+        1, 1, blockchain_constants, config_overrides={"full_node.block_creation": block_creation}
+    ) as new:
+        full_node_api = new.simulators[0].peer_api
+        server_1 = full_node_api.full_node.server
+        bt = new.bt
+
+        wallet = WalletTool(test_constants)
+        coinbase_puzzlehash = wallet.get_new_puzzlehash()
         blocks = bt.get_consecutive_blocks(
-            block_list_input=blocks,
-            num_blocks=1,
-            skip_overflow=False,
-            force_overflow=(i % 10 == 0),
+            num_blocks=10,
             farmer_reward_puzzle_hash=coinbase_puzzlehash,
             guarantee_transaction_block=True,
-            transaction_data=sb,
         )
+        await add_blocks_in_batches(blocks, full_node_api.full_node)
+        _, dummy_node_id = await add_dummy_connection(server_1, self_hostname, 12312)
+        dummy_peer = server_1.all_connections[dummy_node_id]
+        assert full_node_api.full_node.blockchain.get_peak_height() == blocks[-1].height
+        for i in range(10, 100):
+            sb = await add_tx_to_mempool(
+                full_node_api, wallet, blocks[-8], coinbase_puzzlehash, bytes32(i.to_bytes(32, "big")), uint64(i)
+            )
+
+            blocks = bt.get_consecutive_blocks(
+                block_list_input=blocks,
+                num_blocks=1,
+                skip_overflow=False,
+                force_overflow=(i % 10 == 0),
+                farmer_reward_puzzle_hash=coinbase_puzzlehash,
+                guarantee_transaction_block=True,
+                transaction_data=sb,
+            )
 
-        block = blocks[-1]
-        unfinised_block = await declare_pos_unfinished_block(full_node_api, dummy_peer, block)
-        compare_unfinished_blocks(unfinished_from_full_block(block), unfinised_block)
-        await full_node_api.full_node.add_block(block)
-        assert full_node_api.full_node.blockchain.get_peak_height() == block.height
+            block = blocks[-1]
+            unfinised_block = await declare_pos_unfinished_block(full_node_api, dummy_peer, block)
+            compare_unfinished_blocks(unfinished_from_full_block(block), unfinised_block)
+            await full_node_api.full_node.add_block(block)
+            assert full_node_api.full_node.blockchain.get_peak_height() == block.height
 
 
 @pytest.mark.anyio

chia/_tests/core/mempool/test_mempool_manager.py

@@ -2979,7 +2979,7 @@ async def test_check_removals_with_block_creation(flags: int, old: bool) -> None
     assert bundle_add_info2.status == MempoolInclusionStatus.SUCCESS
     assert mempool_manager.peak is not None
     create_block = mempool_manager.create_block_generator if old else mempool_manager.create_block_generator2
-    new_block_gen = create_block(mempool_manager.peak.header_hash)
+    new_block_gen = create_block(mempool_manager.peak.header_hash, 10.0)
     assert new_block_gen is not None
     assert len(new_block_gen.additions) == 1
     assert set(new_block_gen.additions) == {

chia/full_node/full_node_api.py

@@ -865,13 +865,18 @@ async def declare_proof_of_space(
                         curr_l_tb = self.full_node.blockchain.block_record(curr_l_tb.prev_hash)
                     try:
                         # TODO: once we're confident in the new block creation,
-                        # switch to it by default
-                        if self.full_node.config.get("original_block_creation", True):
+                        # make it default to 1
+                        block_version = self.full_node.config.get("block_creation", 0)
+                        block_timeout = self.full_node.config.get("block_creation_timeout", 2.0)
+                        if block_version == 0:
                             create_block = self.full_node.mempool_manager.create_block_generator
-                        else:
+                        elif block_version == 1:
                             create_block = self.full_node.mempool_manager.create_block_generator2
+                        else:
+                            self.log.warning(f"Unknown 'block_creation' config: {block_version}")
+                            create_block = self.full_node.mempool_manager.create_block_generator
 
-                        new_block_gen = create_block(curr_l_tb.header_hash)
+                        new_block_gen = create_block(curr_l_tb.header_hash, block_timeout)
 
                         if (
                             new_block_gen is not None and peak.height < self.full_node.constants.HARD_FORK_HEIGHT

chia/full_node/mempool_manager.py

@@ -370,17 +370,15 @@ def create_bundle_from_mempool(self, last_tb_header_hash: bytes32) -> Optional[t
             return None
         return self.mempool.create_bundle_from_mempool_items(self.constants, self.peak.height)
 
-    def create_block_generator(self, last_tb_header_hash: bytes32, timeout: float = 2.0) -> Optional[NewBlockGenerator]:
+    def create_block_generator(self, last_tb_header_hash: bytes32, timeout: float) -> Optional[NewBlockGenerator]:
         """
         Returns a block generator program, the aggregate signature and all additions and removals, for a new block
         """
         if self.peak is None or self.peak.header_hash != last_tb_header_hash:
             return None
         return self.mempool.create_block_generator(self.constants, self.peak.height, timeout)
 
-    def create_block_generator2(
-        self, last_tb_header_hash: bytes32, timeout: float = 2.0
-    ) -> Optional[NewBlockGenerator]:
+    def create_block_generator2(self, last_tb_header_hash: bytes32, timeout: float) -> Optional[NewBlockGenerator]:
         """
         Returns a block generator program, the aggregate signature and all additions, for a new block
         """
@@ -560,12 +558,13 @@ async def validate_spend_bundle(
         get_unspent_lineage_info_for_puzzle_hash: Callable[[bytes32], Awaitable[Optional[UnspentLineageInfo]]],
     ) -> tuple[Optional[Err], Optional[MempoolItem], list[bytes32]]:
         """
-        Validates new_spend with the given NPCResult, and spend_name, and the current mempool. The mempool should
+        Validates new_spend with the given SpendBundleConditions, and
+        spend_name, and the current mempool. The mempool should
         be locked during this call (blockchain lock).
 
         Args:
             new_spend: spend bundle to validate
-            conds: result of running the clvm transaction in a fake block
+            conds: result of running the clvm transaction
             spend_name: hash of the spend bundle data, passed in as an optimization
             first_added_height: The block height that `new_spend`  first entered this node's mempool.
                 Used to estimate how long a spend has taken to be included on the chain.

chia/rpc/full_node_rpc_api.py

@@ -869,7 +869,7 @@ async def create_block_generator(self, _: dict[str, Any]) -> EndpointResult:
             start_time = time.monotonic()
 
             try:
-                maybe_gen = self.service.mempool_manager.create_block_generator2(curr_l_tb.header_hash)
+                maybe_gen = self.service.mempool_manager.create_block_generator2(curr_l_tb.header_hash, 2.0)
                 if maybe_gen is None:
                     self.service.log.error(f"failed to create block generator, peak: {peak}")
                 else:

chia/util/initial-config.yaml

@@ -305,6 +305,21 @@ full_node:
   #         the particular system we're running on. Defaults to "full".
   db_sync: "auto"
 
+  # Determine which version of block creation function to use:
+  # 0: Original function, first fill a block with transactions, then compress
+  #    it, then run it to compute its cost. The final block may not use all
+  #    available block cost.
+  # 1: Compressed block function. Compress transactions into the block as they
+  #    are added. Use the new, optimized, compression function. This produces
+  #    blocks that fill up the allowed block cost.
+
+  # block_creation: 0
+
+  # If we spend more than this many seconds forming the new transactions block
+  # generator, stop and just go with what we have so far. This is a safety
+  # measure to not spend too much time building the block generator.
+  # block_creation_timeout: 2.0
+
   # the number of threads used to read from the blockchain database
   # concurrently. There's always only 1 writer, but the number of readers is
   # configurable