[CHIA-3923] Chia rs new pos (#20567)

* update chia_rs dependency with the new ProofOfSpace type. use k-22 for v2 plots in simulator

* use new test cache

* review comments

* update test plots and chains to use k-20 for v2 plots, and only alternate between strength 2 and 3 (not 4)

Author: arvidn

.github/workflows/benchmarks.yml

@@ -59,7 +59,7 @@ jobs:
         python-version: ["3.12"]
     env:
       CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet
-      BLOCKS_AND_PLOTS_VERSION: 0.45.3
+      BLOCKS_AND_PLOTS_VERSION: 0.45.6
 
     steps:
       - name: Clean workspace

.github/workflows/test-single.yml

@@ -130,7 +130,7 @@ jobs:
       CHIA_ROOT: ${{ github.workspace }}/.chia/mainnet
       CHIA_SIMULATOR_ROOT: ${{ github.workspace }}/.chia/simulator
       JOB_FILE_NAME: tests_${{ matrix.os.file_name }}_python-${{ matrix.python.file_name }}_${{ matrix.configuration.module_import_path }}${{ matrix.configuration.file_name_index }}
-      BLOCKS_AND_PLOTS_VERSION: 0.45.3
+      BLOCKS_AND_PLOTS_VERSION: 0.45.6
 
     steps:
       - name: Checkout code

benchmarks/block_store.py

@@ -20,7 +20,7 @@
     TransactionsInfo,
 )
 from chia_rs.sized_bytes import bytes32
-from chia_rs.sized_ints import uint8, uint32, uint64, uint128
+from chia_rs.sized_ints import uint8, uint16, uint32, uint64, uint128
 
 from benchmarks.utils import setup_db
 from chia._tests.util.benchmarks import (
@@ -109,7 +109,11 @@ async def run_add_block_benchmark(version: int) -> None:
                 rand_g1() if has_pool_pk else None,
                 rand_hash() if not has_pool_pk else None,
                 rand_g1(),  # plot_public_key
-                uint8(32),
+                uint8(0),  # v1
+                uint16(0),  # plot_index
+                uint8(0),  # group_id
+                uint8(0),  # strength
+                uint8(32),  # size
                 random.randbytes(8 * 32),
             )
 

chia/_tests/blockchain/test_blockchain.py

@@ -1217,7 +1217,10 @@ async def test_bad_pos(self, empty_blockchain: Blockchain, bt: BlockTools) -> No
         )
         await _validate_and_add_block(empty_blockchain, block_bad, expected_error=Err.INVALID_POSPACE)
 
-        block_bad = recursive_replace(blocks[-1], "reward_chain_block.proof_of_space.version_and_size", 62)
+        if blocks[-1].reward_chain_block.proof_of_space.version == 0:
+            block_bad = recursive_replace(blocks[-1], "reward_chain_block.proof_of_space.size", 62)
+        else:
+            block_bad = recursive_replace(blocks[-1], "reward_chain_block.proof_of_space.strength", 1)
         await _validate_and_add_block(empty_blockchain, block_bad, expected_error=Err.INVALID_POSPACE)
 
         block_bad = recursive_replace(
@@ -1226,16 +1229,24 @@ async def test_bad_pos(self, empty_blockchain: Blockchain, bt: BlockTools) -> No
             AugSchemeMPL.key_gen(std_hash(b"1231n")).get_g1(),
         )
         await _validate_and_add_block(empty_blockchain, block_bad, expected_error=Err.INVALID_POSPACE)
-        block_bad = recursive_replace(
-            blocks[-1],
-            "reward_chain_block.proof_of_space.version_and_size",
-            32,
-        )
+
+        if blocks[-1].reward_chain_block.proof_of_space.version == 0:
+            block_bad = recursive_replace(
+                blocks[-1],
+                "reward_chain_block.proof_of_space.size",
+                32,
+            )
+        else:
+            block_bad = recursive_replace(
+                blocks[-1],
+                "reward_chain_block.proof_of_space.strength",
+                67,
+            )
         await _validate_and_add_block(empty_blockchain, block_bad, expected_error=Err.INVALID_POSPACE)
         block_bad = recursive_replace(
             blocks[-1],
             "reward_chain_block.proof_of_space.proof",
-            bytes([1] * int((blocks[-1].reward_chain_block.proof_of_space.version_and_size & 0x7F) * 64 / 8)),
+            bytes([1] * len(blocks[-1].reward_chain_block.proof_of_space.proof)),
         )
         await _validate_and_add_block(empty_blockchain, block_bad, expected_error=Err.INVALID_POSPACE)
 
@@ -1451,6 +1462,12 @@ async def test_pool_target_pre_farm(self, empty_blockchain: Blockchain, bt: Bloc
         await _validate_and_add_block(empty_blockchain, block_bad, expected_error=Err.INVALID_PREFARM)
 
     @pytest.mark.anyio
+    # TODO: todo_v2_plots fix this test and remove limit_consensus_modes
+    @pytest.mark.limit_consensus_modes(
+        allowed=[ConsensusMode.PLAIN, ConsensusMode.HARD_FORK_2_0, ConsensusMode.HARD_FORK_3_0],
+        reason="It seams ConsensusMode.HARD_FORK_3_0_AFTER_PHASE_OUT fails to "
+        "find any proofs with pool keys in a timely manner",
+    )
     async def test_pool_target_signature(self, empty_blockchain: Blockchain, bt: BlockTools) -> None:
         # 20b
         blocks_initial = bt.get_consecutive_blocks(2)
@@ -3416,6 +3433,8 @@ async def test_long_reorg(
     ) -> None:
         if light_blocks:
             reorg_blocks = test_long_reorg_blocks_light[:1650]
+        elif consensus_mode >= ConsensusMode.HARD_FORK_3_0:
+            reorg_blocks = test_long_reorg_blocks[:1350]
         else:
             reorg_blocks = test_long_reorg_blocks[:1200]
 
@@ -4360,7 +4379,7 @@ async def test_get_header_blocks_in_range_tx_filter_non_tx_block(empty_blockchai
     transactions filter, on a non transaction block.
     """
     b = empty_blockchain
-    blocks = bt.get_consecutive_blocks(2)
+    blocks = bt.get_consecutive_blocks(10)
     for block in blocks:
         await _validate_and_add_block(b, block)
     non_tx_block = next(block for block in blocks if not block.is_transaction_block())

chia/_tests/core/consensus/test_pot_iterations.py

@@ -93,9 +93,9 @@ def test_win_percentage(self):
             PlotParam.make_v1(34): 100,
             PlotParam.make_v1(35): 100,
             PlotParam.make_v1(36): 100,
-            PlotParam.make_v2(2): 200,
-            PlotParam.make_v2(3): 200,
-            PlotParam.make_v2(4): 200,
+            PlotParam.make_v2(0, 0, 2): 200,
+            PlotParam.make_v2(0, 0, 3): 200,
+            PlotParam.make_v2(0, 0, 4): 200,
         }
         farmer_space = {k: _expected_plot_size(k, test_constants) * count for k, count in farmer_ks.items()}
         wins = {k: 0 for k in farmer_ks.keys()}
@@ -141,5 +141,7 @@ def test_expected_plot_size_v1() -> None:
 
 def test_expected_plot_size_v2() -> None:
     for strength in [2, 3, 4, 5, 6, 7]:
-        plot_size = _expected_plot_size(PlotParam.make_v2(strength), test_constants)
-        assert plot_size == 988_513_566
+        for plot_index in [2, 300, 400, 500, 600, 700]:
+            for group_id in [2, 3, 4, 5, 6, 7]:
+                plot_size = _expected_plot_size(PlotParam.make_v2(plot_index, group_id, strength), test_constants)
+                assert plot_size == 988_513_566

chia/_tests/core/custom_types/test_proof_of_space.py

@@ -43,7 +43,7 @@ def b32(key: str) -> bytes32:
     return bytes32.from_hexstr(key)
 
 
-# TODO: todo_v2_plots more test cases
+# TODO: todo_v2_plots more test cases, cover plot_index and group_id
 @datacases(
     ProofOfSpaceCase(
         id="Neither pool public key nor pool contract puzzle hash",
@@ -113,23 +113,23 @@ def b32(key: str) -> bytes32:
     ProofOfSpaceCase(
         id="v2 plot strength 0",
         pos_challenge=bytes32(b"1" * 32),
-        plot_size=PlotParam.make_v2(0),
+        plot_size=PlotParam.make_v2(0, 0, 0),
         pool_contract_puzzle_hash=bytes32(b"1" * 32),
         plot_public_key=G1Element(),
         expected_error="Plot strength (0) is lower than the minimum (2)",
     ),
     ProofOfSpaceCase(
         id="v2 plot strength 33",
         pos_challenge=bytes32(b"1" * 32),
-        plot_size=PlotParam.make_v2(33),
+        plot_size=PlotParam.make_v2(0, 0, 33),
         pool_contract_puzzle_hash=bytes32(b"1" * 32),
         plot_public_key=G1Element(),
         expected_error="Plot strength (33) is too high (max is 32)",
     ),
     ProofOfSpaceCase(
         id="Not passing the plot filter v2",
-        pos_challenge=b32("0cbb408989205e9a94cc012f0d2717e9336795b0853f972965ad0c75c5700ed9"),
-        plot_size=PlotParam.make_v2(32),
+        pos_challenge=b32("5862b9d5210b008c0c30e2673e327b4a6c98c34d93dd05c7a253320eb5db1712"),
+        plot_size=PlotParam.make_v2(0, 0, 32),
         pool_contract_puzzle_hash=bytes32(b"1" * 32),
         plot_public_key=g1(
             "879526b4e7b616cfd64984d8ad140d0798b048392a6f11e2faf09054ef467ea44dc0dab5e5edb2afdfa850c5c8b629cc"
@@ -139,7 +139,7 @@ def b32(key: str) -> bytes32:
     ProofOfSpaceCase(
         id="v2 not activated",
         pos_challenge=bytes32(b"1" * 32),
-        plot_size=PlotParam.make_v2(2),
+        plot_size=PlotParam.make_v2(0, 0, 2),
         pool_contract_puzzle_hash=bytes32(b"1" * 32),
         plot_public_key=G1Element(),
         height=uint32(DEFAULT_CONSTANTS.HARD_FORK2_HEIGHT - 1),
@@ -153,13 +153,13 @@ def test_verify_and_get_quality_string(caplog: pytest.LogCaptureFixture, case: P
         pool_public_key=case.pool_public_key,
         pool_contract_puzzle_hash=case.pool_contract_puzzle_hash,
         plot_public_key=case.plot_public_key,
-        version_and_size=case.plot_size,
+        params=case.plot_size,
         proof=b"1",
     )
     quality_string = verify_and_get_quality_string(
         pos=pos,
         constants=DEFAULT_CONSTANTS,
-        original_challenge_hash=b32("0x73490e166d0b88347c37d921660b216c27316aae9a3450933d3ff3b854e5831a"),
+        original_challenge_hash=b32("73490e166d0b88347c37d921660b216c27316aae9a3450933d3ff3b854e5831a"),
         signage_point=b32("0x7b3e23dbd438f9aceefa9827e2c5538898189987f49b06eceb7a43067e77b531"),
         height=case.height,
         prev_transaction_block_height=case.height,
@@ -171,8 +171,8 @@ def test_verify_and_get_quality_string(caplog: pytest.LogCaptureFixture, case: P
 @datacases(
     ProofOfSpaceCase(
         id="not passing the plot filter v2",
-        plot_size=PlotParam.make_v2(2),
-        pos_challenge=b32("389f3f2f67d37a63d81a21d0e96898ff4eaa7007e1507630bea6a2608795a418"),
+        plot_size=PlotParam.make_v2(0, 0, 2),
+        pos_challenge=b32("9483df4e178307ae677d86664daaef4fc52689b2b6cd7825351f2a2ad7075adb"),
         plot_public_key=g1(
             "afa3aaf09c03885154be49216ee7fb2e4581b9c4a4d7e9cc402e27280bf0cfdbdf1b9ba674e301fd1d1450234b3b1868"
         ),
@@ -187,7 +187,7 @@ def test_verify_and_get_quality_string_v2(caplog: pytest.LogCaptureFixture, case
         pool_public_key=case.pool_public_key,
         pool_contract_puzzle_hash=case.pool_contract_puzzle_hash,
         plot_public_key=case.plot_public_key,
-        version_and_size=case.plot_size,
+        params=case.plot_size,
         proof=b"1",
     )
     plot_param = pos.param()
@@ -197,7 +197,7 @@ def test_verify_and_get_quality_string_v2(caplog: pytest.LogCaptureFixture, case
         quality_string = verify_and_get_quality_string(
             pos=pos,
             constants=DEFAULT_CONSTANTS,
-            original_challenge_hash=b32("0x73490e166d0b88347c37d921660b216c27316aae9a3450933d3ff3b854e5831a"),
+            original_challenge_hash=b32("73490e166d0b88347c37d921660b216c27316aae9a3450933d3ff3b854e5831a"),
             signage_point=b32("0xf7c1bd874da5e709d4713d60c8a70639eb1167b367a9c3787c65c1e582e2e662"),
             height=case.height,
             prev_transaction_block_height=case.height,
@@ -223,11 +223,11 @@ def test_verify_and_get_quality_string_v2(caplog: pytest.LogCaptureFixture, case
         (PlotParam.make_v1(49), True),
         (PlotParam.make_v1(50), True),
         (PlotParam.make_v1(51), False),  # too large
-        (PlotParam.make_v2(1), False),  # too small
-        (PlotParam.make_v2(2), True),
-        (PlotParam.make_v2(3), True),
-        (PlotParam.make_v2(32), True),
-        (PlotParam.make_v2(33), False),  # strength too high
+        (PlotParam.make_v2(0, 0, 1), False),  # too small
+        (PlotParam.make_v2(0, 0, 2), True),
+        (PlotParam.make_v2(0, 0, 3), True),
+        (PlotParam.make_v2(0, 0, 32), True),
+        (PlotParam.make_v2(0, 0, 33), False),  # strength too high
     ],
 )
 def test_check_plot_param(plot_param: PlotParam, valid: bool) -> None:
@@ -290,7 +290,7 @@ def test_calculate_prefix_bits_v1(height: uint32, expected: int) -> None:
     ],
 )
 def test_calculate_prefix_bits_v2(height: uint32, expected: int) -> None:
-    assert calculate_prefix_bits(DEFAULT_CONSTANTS, height, PlotParam.make_v2(28)) == expected
+    assert calculate_prefix_bits(DEFAULT_CONSTANTS, height, PlotParam.make_v2(0, 0, 28)) == expected
 
 
 def test_v1_phase_out() -> None:

chia/_tests/core/full_node/test_full_node.py

@@ -1641,7 +1641,11 @@ async def test_unfinished_block_with_replaced_generator(
                 pos.pool_public_key,
                 pos.pool_contract_puzzle_hash,
                 public_key,
-                pos.version_and_size,
+                pos.version,
+                pos.plot_index,
+                pos.meta_group,
+                pos.strength,
+                pos.size,
                 pos.proof,
             )
 
@@ -2153,13 +2157,14 @@ async def test_compact_protocol_invalid_messages(
         wrong_challenge = block.reward_chain_block.challenge_chain_sp_vdf.challenge
         wrong_iters = block.reward_chain_block.challenge_chain_sp_vdf.number_of_iterations
 
-    wrong_vdf_info, wrong_vdf_proof = get_vdf_info_and_proof(
+    wrong_vdf_info, _ = get_vdf_info_and_proof(
         bt.constants,
         ClassgroupElement.get_default_element(),
         wrong_challenge,
         wrong_iters,
         True,
     )
+    wrong_vdf_proof = VDFProof(uint8(0), b"1239819023890", True)
     timelord_protocol_invalid_messages: list[timelord_protocol.RespondCompactProofOfTime] = []
     full_node_protocol_invalid_messages: list[fnp.RespondCompactVDF] = []
     for block in blocks_2[:2]:
@@ -2171,7 +2176,6 @@ async def test_compact_protocol_invalid_messages(
                 sub_slot.challenge_chain.challenge_chain_end_of_slot_vdf.number_of_iterations,
                 True,
             )
-            assert wrong_vdf_proof != correct_vdf_proof
             timelord_protocol_invalid_messages.append(
                 timelord_protocol.RespondCompactProofOfTime(
                     vdf_info,
@@ -2198,7 +2202,6 @@ async def test_compact_protocol_invalid_messages(
                     sub_slot.infused_challenge_chain.infused_challenge_chain_end_of_slot_vdf.number_of_iterations,
                     True,
                 )
-                assert wrong_vdf_proof != correct_vdf_proof
                 timelord_protocol_invalid_messages.append(
                     timelord_protocol.RespondCompactProofOfTime(
                         vdf_info,
@@ -2226,14 +2229,10 @@ async def test_compact_protocol_invalid_messages(
                 block.reward_chain_block.challenge_chain_sp_vdf.number_of_iterations,
                 True,
             )
-            sp_vdf_proof = wrong_vdf_proof
-            if wrong_vdf_proof == correct_vdf_proof:
-                # This can actually happen...
-                sp_vdf_proof = VDFProof(uint8(0), b"1239819023890", True)
             timelord_protocol_invalid_messages.append(
                 timelord_protocol.RespondCompactProofOfTime(
                     vdf_info,
-                    sp_vdf_proof,
+                    wrong_vdf_proof,
                     block.header_hash,
                     block.height,
                     uint8(CompressibleVDFField.CC_SP_VDF),
@@ -2245,7 +2244,7 @@ async def test_compact_protocol_invalid_messages(
                     block.header_hash,
                     uint8(CompressibleVDFField.CC_SP_VDF),
                     vdf_info,
-                    sp_vdf_proof,
+                    wrong_vdf_proof,
                 )
             )
 

chia/_tests/core/full_node/test_generator_tools.py

@@ -1,13 +1,15 @@
 from __future__ import annotations
 
 import pytest
-from chia_rs import Program as SerializedProgram
 from chia_rs import (
+    CoinSpend,
     SpendBundleConditions,
     SpendConditions,
     get_spends_for_trusted_block,
     get_spends_for_trusted_block_with_conditions,
 )
+from chia_rs import Program as SerializedProgram
+from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint32, uint64
 
 from chia.consensus.generator_tools import tx_removals_and_additions
@@ -110,11 +112,31 @@ def test_get_spends_for_block(caplog: pytest.LogCaptureFixture) -> None:
     conditions = get_spends_for_trusted_block(
         test_constants, TEST_GENERATOR.program, TEST_GENERATOR.generator_refs, 100
     )
-    assert conditions["block_spends"] == []
+    assert conditions["block_spends"] == [
+        CoinSpend(
+            Coin(
+                bytes32.fromhex("0101010101010101010101010101010101010101010101010101010101010101"),
+                bytes32.fromhex("6c04a09251046f8dd47efe681af7e47f6e61e68fb2f9ad47c5031ec3e36c5564"),
+                uint64(123),
+            ),
+            SerializedProgram.fromhex("80"),
+            SerializedProgram.fromhex("ff80ffff018080"),
+        )
+    ]
 
 
 def test_get_spends_for_block_with_conditions(caplog: pytest.LogCaptureFixture) -> None:
     conditions = get_spends_for_trusted_block_with_conditions(
         test_constants, TEST_GENERATOR.program, TEST_GENERATOR.generator_refs, 100
     )
-    assert conditions == []
+    assert len(conditions) == 1
+    assert conditions[0]["coin_spend"] == CoinSpend(
+        Coin(
+            bytes32.fromhex("0101010101010101010101010101010101010101010101010101010101010101"),
+            bytes32.fromhex("6c04a09251046f8dd47efe681af7e47f6e61e68fb2f9ad47c5031ec3e36c5564"),
+            uint64(123),
+        ),
+        SerializedProgram.fromhex("80"),
+        SerializedProgram.fromhex("ff80ffff018080"),
+    )
+    assert len(conditions[0]["conditions"]) == 1024

chia/_tests/core/mempool/test_mempool_manager.py

@@ -2764,15 +2764,15 @@ def transactions_1000_fixture(test_wallet: WalletTool, seeded_random: random.Ran
 # if we try to fill the mempool with more than 550, all spends won't
 # necessarily fit in the block, which the test assumes
 @pytest.mark.anyio
-@pytest.mark.parametrize("mempool_size", [1, 2, 100, 300, 400, 550, 730])
+@pytest.mark.parametrize("mempool_size", [1, 2, 100, 300, 400, 550, 630])
 @pytest.mark.parametrize("seed", [0, 1, 2, 3, 4])
 @pytest.mark.parametrize("old", [True, False])
 async def test_create_block_generator(
     mempool_size: int, seed: int, old: bool, transactions_1000: list[SpendBundle]
 ) -> None:
     # the old way of creating bloks doesn't fit this many transactions, so we
     # expect it to fail
-    expect_failure = mempool_size == 730 and old
+    expect_failure = mempool_size == 630 and old
 
     bundles = transactions_1000
     all_coins = [s.coin for b in bundles for s in b.coin_spends]