Merge branch 'main' into farmer_solver_networking

Author: almogdepaz

chia/_tests/cmds/cmd_test_utils.py

@@ -22,7 +22,6 @@
 from chia.full_node.full_node_rpc_client import FullNodeRpcClient
 from chia.rpc.rpc_client import RpcClient
 from chia.simulator.simulator_full_node_rpc_client import SimulatorFullNodeRpcClient
-from chia.types.coin_record import CoinRecord
 from chia.types.signing_mode import SigningMode
 from chia.util.bech32m import encode_puzzle_hash
 from chia.util.config import load_config
@@ -31,7 +30,7 @@
 from chia.wallet.nft_wallet.nft_wallet import NFTWallet
 from chia.wallet.transaction_record import TransactionRecord
 from chia.wallet.util.transaction_type import TransactionType
-from chia.wallet.util.tx_config import CoinSelectionConfig, TXConfig
+from chia.wallet.util.tx_config import TXConfig
 from chia.wallet.util.wallet_types import WalletType
 from chia.wallet.wallet_request_types import (
     GetSyncStatusResponse,
@@ -232,46 +231,6 @@ async def nft_calculate_royalties(
             )
         )
 
-    async def get_spendable_coins(
-        self,
-        wallet_id: int,
-        coin_selection_config: CoinSelectionConfig,
-    ) -> tuple[list[CoinRecord], list[CoinRecord], list[Coin]]:
-        """
-        We return a tuple containing: (confirmed records, unconfirmed removals, unconfirmed additions)
-        """
-        self.add_to_log(
-            "get_spendable_coins",
-            (wallet_id, coin_selection_config),
-        )
-        confirmed_records = [
-            CoinRecord(
-                Coin(bytes32([1] * 32), bytes32([2] * 32), uint64(1234560000)),
-                uint32(123456),
-                uint32(0),
-                False,
-                uint64(0),
-            ),
-            CoinRecord(
-                Coin(bytes32([3] * 32), bytes32([4] * 32), uint64(1234560000)),
-                uint32(123456),
-                uint32(0),
-                False,
-                uint64(0),
-            ),
-        ]
-        unconfirmed_removals = [
-            CoinRecord(
-                Coin(bytes32([5] * 32), bytes32([6] * 32), uint64(1234570000)),
-                uint32(123457),
-                uint32(0),
-                True,
-                uint64(0),
-            )
-        ]
-        unconfirmed_additions = [Coin(bytes32([7] * 32), bytes32([8] * 32), uint64(1234580000))]
-        return confirmed_records, unconfirmed_removals, unconfirmed_additions
-
     async def send_transaction_multi(
         self,
         wallet_id: int,

chia/_tests/core/full_node/test_full_node.py

@@ -3294,3 +3294,62 @@ def compare_unfinished_blocks(block1: UnfinishedBlock, block2: UnfinishedBlock)
     # Final assertion to check the entire block
     assert block1 == block2, "The entire block objects are not identical"
     return True
+
+
+@pytest.mark.anyio
+@pytest.mark.parametrize(
+    "condition, error",
+    [
+        (ConditionOpcode.ASSERT_HEIGHT_RELATIVE, "ASSERT_HEIGHT_RELATIVE_FAILED"),
+        (ConditionOpcode.ASSERT_HEIGHT_ABSOLUTE, "ASSERT_HEIGHT_ABSOLUTE_FAILED"),
+    ],
+)
+async def test_pending_tx_cache_retry_on_new_peak(
+    condition: ConditionOpcode, error: str, blockchain_constants: ConsensusConstants, caplog: pytest.LogCaptureFixture
+) -> None:
+    """
+    Covers PendingTXCache items that are placed there due to unmet relative or
+    absolute height conditions, to make sure those items get retried at peak
+    post processing when those conditions are met.
+    """
+    async with setup_simulators_and_wallets(1, 0, blockchain_constants) as new:
+        full_node_api = new.simulators[0].peer_api
+        bt = new.bt
+        wallet = WalletTool(test_constants)
+        ph = wallet.get_new_puzzlehash()
+        blocks = bt.get_consecutive_blocks(
+            3, guarantee_transaction_block=True, farmer_reward_puzzle_hash=ph, pool_reward_puzzle_hash=ph
+        )
+        for block in blocks:
+            await full_node_api.full_node.add_block(block)
+        peak = full_node_api.full_node.blockchain.get_peak()
+        assert peak is not None
+        current_height = peak.height
+        # Create a transaction with a height condition that makes it pending
+        coin = blocks[-1].get_included_reward_coins()[0]
+        if condition == ConditionOpcode.ASSERT_HEIGHT_RELATIVE:
+            condition_height = 1
+        else:
+            condition_height = current_height + 1
+        condition_dic = {condition: [ConditionWithArgs(condition, [int_to_bytes(condition_height)])]}
+        sb = wallet.generate_signed_transaction(uint64(42), ph, coin, condition_dic)
+        sb_name = sb.name()
+        # Send the transaction
+        res = await full_node_api.send_transaction(SendTransaction(sb))
+        assert res is not None
+        assert ProtocolMessageTypes(res.type) == ProtocolMessageTypes.transaction_ack
+        transaction_ack = TransactionAck.from_bytes(res.data)
+        assert transaction_ack.status == MempoolInclusionStatus.PENDING.value
+        assert transaction_ack.error == error
+        # Make sure it ends up in the pending cache, not the mempool
+        assert full_node_api.full_node.mempool_manager.get_mempool_item(sb_name, include_pending=False) is None
+        assert full_node_api.full_node.mempool_manager.get_mempool_item(sb_name, include_pending=True) is not None
+        # Advance peak to meet the asserted height condition
+        with caplog.at_level(logging.DEBUG):
+            blocks = bt.get_consecutive_blocks(2, block_list_input=blocks, guarantee_transaction_block=True)
+            for block in blocks:
+                await full_node_api.full_node.add_block(block)
+        # This should trigger peak post processing with the added transaction
+        assert f"Added transaction to mempool: {sb_name}\n" in caplog.text
+        # Make sure the transaction was retried and got added to the mempool
+        assert full_node_api.full_node.mempool_manager.get_mempool_item(sb_name, include_pending=False) is not None

chia/_tests/core/mempool/test_mempool_manager.py

@@ -41,7 +41,6 @@
     QUOTE_BYTES,
     QUOTE_EXECUTION_COST,
     MempoolManager,
-    NewPeakItem,
     TimelockConditions,
     can_replace,
     check_removals,
@@ -3288,7 +3287,7 @@ async def test_new_peak_txs_added(condition_and_error: tuple[ConditionOpcode, Er
     assert mempool_manager.peak is not None
     condition_height = mempool_manager.peak.height + 1
     condition, expected_error = condition_and_error
-    sb, sb_name, result = await generate_and_add_spendbundle(mempool_manager, [[condition, condition_height]])
+    _, sb_name, result = await generate_and_add_spendbundle(mempool_manager, [[condition, condition_height]])
     _, status, error = result
     assert status == MempoolInclusionStatus.PENDING
     assert error == expected_error
@@ -3299,14 +3298,13 @@ async def test_new_peak_txs_added(condition_and_error: tuple[ConditionOpcode, Er
             create_test_block_record(height=uint32(condition_height)), spent_coins
         )
         # We're not there yet (needs to be higher, not equal)
+        assert new_peak_info.spend_bundle_ids == []
         assert mempool_manager.get_mempool_item(sb_name, include_pending=False) is None
-        assert new_peak_info.items == []
     else:
         spent_coins = None
     new_peak_info = await mempool_manager.new_peak(
         create_test_block_record(height=uint32(condition_height + 1)), spent_coins
     )
     # The item gets retried successfully now
-    mi = mempool_manager.get_mempool_item(sb_name, include_pending=False)
-    assert mi is not None
-    assert new_peak_info.items == [NewPeakItem(sb_name, sb, mi.conds)]
+    assert new_peak_info.spend_bundle_ids == [sb_name]
+    assert mempool_manager.get_mempool_item(sb_name, include_pending=False) is not None

chia/_tests/core/server/test_rate_limits.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import Any, cast
+from typing import Union
 
 import pytest
 from chia_rs.sized_ints import uint32
@@ -13,8 +13,7 @@
 from chia.protocols.outbound_message import make_msg
 from chia.protocols.protocol_message_types import ProtocolMessageTypes
 from chia.protocols.shared_protocol import Capability
-from chia.server.rate_limit_numbers import RLSettings, compose_rate_limits, get_rate_limits_to_use
-from chia.server.rate_limit_numbers import rate_limits as rl_numbers
+from chia.server.rate_limit_numbers import RLSettings, Unlimited, get_rate_limits_to_use
 from chia.server.rate_limits import RateLimiter
 from chia.server.server import ChiaServer
 from chia.server.ws_connection import WSChiaConnection
@@ -67,39 +66,22 @@ async def test_limits_v2(incoming: bool, tx_msg: bool, limit_size: bool, monkeyp
     message_data = b"\0" * 1024
     msg_type = ProtocolMessageTypes.new_transaction
 
-    limits: dict[str, Any] = {}
+    limits: dict[ProtocolMessageTypes, Union[RLSettings, Unlimited]]
 
     if limit_size:
-        limits.update(
-            {
-                # this is the rate limit across all (non-tx) messages
-                "non_tx_freq": count * 2,
-                # this is the byte size limit across all (non-tx) messages
-                "non_tx_max_total_size": count * len(message_data),
-            }
-        )
+        agg_limit = RLSettings(False, count * 2, len(message_data), count * len(message_data))
     else:
-        limits.update(
-            {
-                # this is the rate limit across all (non-tx) messages
-                "non_tx_freq": count,
-                # this is the byte size limit across all (non-tx) messages
-                "non_tx_max_total_size": count * 2 * len(message_data),
-            }
-        )
+        agg_limit = RLSettings(False, count, len(message_data), count * 2 * len(message_data))
 
     if limit_size:
-        rate_limit = {msg_type: RLSettings(count * 2, 1024, count * len(message_data))}
+        limits = {msg_type: RLSettings(not tx_msg, count * 2, len(message_data), count * len(message_data))}
     else:
-        rate_limit = {msg_type: RLSettings(count, 1024, count * 2 * len(message_data))}
+        limits = {msg_type: RLSettings(not tx_msg, count, len(message_data), count * 2 * len(message_data))}
 
-    if tx_msg:
-        limits.update({"rate_limits_tx": rate_limit, "rate_limits_other": {}})
-    else:
-        limits.update({"rate_limits_other": rate_limit, "rate_limits_tx": {}})
-
-    def mock_get_limits(our_capabilities: list[Capability], peer_capabilities: list[Capability]) -> dict[str, Any]:
-        return limits
+    def mock_get_limits(
+        our_capabilities: list[Capability], peer_capabilities: list[Capability]
+    ) -> tuple[dict[ProtocolMessageTypes, Union[RLSettings, Unlimited]], RLSettings]:
+        return limits, agg_limit
 
     import chia.server.rate_limits
 
@@ -327,18 +309,18 @@ async def test_too_many_outgoing_messages() -> None:
     # Too many messages
     r = RateLimiter(incoming=False, get_time=lambda: 0)
     new_peers_message = make_msg(ProtocolMessageTypes.respond_peers, bytes([1]))
-    non_tx_freq = get_rate_limits_to_use(rl_v2, rl_v2)["non_tx_freq"]
+    _, agg_limit = get_rate_limits_to_use(rl_v2, rl_v2)
 
     passed = 0
     blocked = 0
-    for i in range(non_tx_freq):
+    for i in range(agg_limit.frequency):
         if r.process_msg_and_check(new_peers_message, rl_v2, rl_v2) is None:
             passed += 1
         else:
             blocked += 1
 
     assert passed == 10
-    assert blocked == non_tx_freq - passed
+    assert blocked == agg_limit.frequency - passed
 
     # ensure that *another* message type is not blocked because of this
 
@@ -351,18 +333,18 @@ async def test_too_many_incoming_messages() -> None:
     # Too many messages
     r = RateLimiter(incoming=True, get_time=lambda: 0)
     new_peers_message = make_msg(ProtocolMessageTypes.respond_peers, bytes([1]))
-    non_tx_freq = get_rate_limits_to_use(rl_v2, rl_v2)["non_tx_freq"]
+    _, agg_limit = get_rate_limits_to_use(rl_v2, rl_v2)
 
     passed = 0
     blocked = 0
-    for i in range(non_tx_freq):
+    for i in range(agg_limit.frequency):
         if r.process_msg_and_check(new_peers_message, rl_v2, rl_v2) is None:
             passed += 1
         else:
             blocked += 1
 
     assert passed == 10
-    assert blocked == non_tx_freq - passed
+    assert blocked == agg_limit.frequency - passed
 
     # ensure that other message types *are* blocked because of this
 
@@ -436,43 +418,13 @@ async def test_different_versions(
     # The following code checks whether all of the runs resulted in the same number of items in "rate_limits_tx",
     # which would mean the same rate limits are always used. This should not happen, since two nodes with V2
     # will use V2.
-    total_tx_msg_count = len(
-        get_rate_limits_to_use(a_con.local_capabilities, a_con.peer_capabilities)["rate_limits_tx"]
-    )
+    total_tx_msg_count = len(get_rate_limits_to_use(a_con.local_capabilities, a_con.peer_capabilities))
 
     test_different_versions_results.append(total_tx_msg_count)
     if len(test_different_versions_results) >= 4:
         assert len(set(test_different_versions_results)) >= 2
 
 
-@pytest.mark.anyio
-async def test_compose() -> None:
-    rl_1 = rl_numbers[1]
-    rl_2 = rl_numbers[2]
-    rl_1_rate_limits_other = cast(dict[ProtocolMessageTypes, RLSettings], rl_1["rate_limits_other"])
-    rl_2_rate_limits_other = cast(dict[ProtocolMessageTypes, RLSettings], rl_2["rate_limits_other"])
-    rl_1_rate_limits_tx = cast(dict[ProtocolMessageTypes, RLSettings], rl_1["rate_limits_tx"])
-    rl_2_rate_limits_tx = cast(dict[ProtocolMessageTypes, RLSettings], rl_2["rate_limits_tx"])
-    assert ProtocolMessageTypes.respond_children in rl_1_rate_limits_other
-    assert ProtocolMessageTypes.respond_children not in rl_1_rate_limits_tx
-    assert ProtocolMessageTypes.respond_children not in rl_2_rate_limits_other
-    assert ProtocolMessageTypes.respond_children in rl_2_rate_limits_tx
-
-    assert ProtocolMessageTypes.request_block in rl_1_rate_limits_other
-    assert ProtocolMessageTypes.request_block not in rl_1_rate_limits_tx
-    assert ProtocolMessageTypes.request_block not in rl_2_rate_limits_other
-    assert ProtocolMessageTypes.request_block not in rl_2_rate_limits_tx
-
-    comps = compose_rate_limits(rl_1, rl_2)
-    # v2 limits are used if present
-    assert ProtocolMessageTypes.respond_children not in comps["rate_limits_other"]
-    assert ProtocolMessageTypes.respond_children in comps["rate_limits_tx"]
-
-    # Otherwise, fall back to v1
-    assert ProtocolMessageTypes.request_block in rl_1_rate_limits_other
-    assert ProtocolMessageTypes.request_block not in rl_1_rate_limits_tx
-
-
 @pytest.mark.anyio
 @pytest.mark.parametrize(
     "msg_type, size",

chia/_tests/util/test_network_protocol_test.py

@@ -4,6 +4,8 @@
 import inspect
 from typing import Any, cast
 
+import pytest
+
 from chia.protocols import (
     farmer_protocol,
     full_node_protocol,
@@ -264,3 +266,19 @@ def test_missing_messages() -> None:
     assert types_in_module(shared_protocol) == shared_msgs, (
         f"message types were added or removed from shared_protocol. {STANDARD_ADVICE}"
     )
+
+
+@pytest.mark.parametrize("version", [1, 2])
+def test_rate_limits_complete(version: int) -> None:
+    from chia.protocols.protocol_message_types import ProtocolMessageTypes
+    from chia.server.rate_limit_numbers import rate_limits
+
+    if version == 1:
+        composed = rate_limits[1]
+    elif version == 2:
+        composed = {
+            **rate_limits[1],
+            **rate_limits[2],
+        }
+
+    assert set(composed.keys()) == set(ProtocolMessageTypes)