support previous harvester protocol (#19994)

* support previous harvester protocol

* Update chia/protocols/harvester_protocol.py

Co-authored-by: Amine Khaldi <[email protected]>

---------

Co-authored-by: Amine Khaldi <[email protected]>

Author: arvidn

chia/_tests/util/build_network_protocol_files.py

@@ -117,6 +117,7 @@ def visit_harvester_protocol(visitor: Callable[[Any, str], None]) -> None:
     visitor(pool_difficulty, "pool_difficulty")
     visitor(harvester_handhsake, "harvester_handhsake")
     visitor(new_signage_point_harvester, "new_signage_point_harvester")
+    visitor(new_signage_point_harvester2, "new_signage_point_harvester2")
     visitor(new_proof_of_space, "new_proof_of_space")
     visitor(request_signatures, "request_signatures")
     visitor(respond_signatures, "respond_signatures")

chia/_tests/util/network_protocol_data.py

@@ -789,6 +789,16 @@
 )
 
 new_signage_point_harvester = harvester_protocol.NewSignagePointHarvester(
+    bytes32(bytes.fromhex("e342c21b4aeaa52349d42492be934692db58494ca9bce4a8697d06fdf8e583bb")),
+    uint64(15615706268399948682),
+    uint64(10520767421667792980),
+    uint8(148),
+    bytes32(bytes.fromhex("b78c9fca155e9742df835cbe84bb7e518bee70d78b6be6e39996c0a02e0cfe4c")),
+    [pool_difficulty],
+    uint8(3),
+)
+
+new_signage_point_harvester2 = harvester_protocol.NewSignagePointHarvester2(
     bytes32(bytes.fromhex("e342c21b4aeaa52349d42492be934692db58494ca9bce4a8697d06fdf8e583bb")),
     uint64(15615706268399948682),
     uint64(10520767421667792980),
@@ -799,6 +809,7 @@
     uint32(0),
 )
 
+
 new_proof_of_space = harvester_protocol.NewProofOfSpace(
     bytes32.fromhex("1b64ec6bf3fe33bb80eca5b64ff1c88be07771eaed1e98a7199510522087e56e"),
     bytes32.fromhex("ad1f8a74376ce8c5c93b7fbb355c2fb6d689ae4f4a7134166593d95265a3da30"),

chia/_tests/util/protocol_messages_bytes-v1.0

@@ -2148,6 +2148,22 @@
 }
 
 new_signage_point_harvester_json: dict[str, Any] = {
+    "challenge_hash": "0xe342c21b4aeaa52349d42492be934692db58494ca9bce4a8697d06fdf8e583bb",
+    "difficulty": 15615706268399948682,
+    "sub_slot_iters": 10520767421667792980,
+    "signage_point_index": 148,
+    "sp_hash": "0xb78c9fca155e9742df835cbe84bb7e518bee70d78b6be6e39996c0a02e0cfe4c",
+    "pool_difficulties": [
+        {
+            "difficulty": 14819251421858580996,
+            "sub_slot_iters": 12852879676624401630,
+            "pool_contract_puzzle_hash": "0xc9423123ea65e6923e973b95531b4874570dae942cb757a2daec4a6971753886",
+        }
+    ],
+    "filter_prefix_bits": 3,
+}
+
+new_signage_point_harvester2_json: dict[str, Any] = {
     "challenge_hash": "0xe342c21b4aeaa52349d42492be934692db58494ca9bce4a8697d06fdf8e583bb",
     "difficulty": 15615706268399948682,
     "sub_slot_iters": 10520767421667792980,

chia/_tests/util/protocol_messages_json.py

@@ -426,153 +426,158 @@ def test_protocol_bytes() -> None:
     assert bytes(message_79) == bytes(new_signage_point_harvester)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_80 = type(new_proof_of_space).from_bytes(message_bytes)
-    assert message_80 == new_proof_of_space
-    assert bytes(message_80) == bytes(new_proof_of_space)
+    message_80 = type(new_signage_point_harvester2).from_bytes(message_bytes)
+    assert message_80 == new_signage_point_harvester2
+    assert bytes(message_80) == bytes(new_signage_point_harvester2)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_81 = type(request_signatures).from_bytes(message_bytes)
-    assert message_81 == request_signatures
-    assert bytes(message_81) == bytes(request_signatures)
+    message_81 = type(new_proof_of_space).from_bytes(message_bytes)
+    assert message_81 == new_proof_of_space
+    assert bytes(message_81) == bytes(new_proof_of_space)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_82 = type(respond_signatures).from_bytes(message_bytes)
-    assert message_82 == respond_signatures
-    assert bytes(message_82) == bytes(respond_signatures)
+    message_82 = type(request_signatures).from_bytes(message_bytes)
+    assert message_82 == request_signatures
+    assert bytes(message_82) == bytes(request_signatures)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_83 = type(plot).from_bytes(message_bytes)
-    assert message_83 == plot
-    assert bytes(message_83) == bytes(plot)
+    message_83 = type(respond_signatures).from_bytes(message_bytes)
+    assert message_83 == respond_signatures
+    assert bytes(message_83) == bytes(respond_signatures)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_84 = type(request_plots).from_bytes(message_bytes)
-    assert message_84 == request_plots
-    assert bytes(message_84) == bytes(request_plots)
+    message_84 = type(plot).from_bytes(message_bytes)
+    assert message_84 == plot
+    assert bytes(message_84) == bytes(plot)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_85 = type(respond_plots).from_bytes(message_bytes)
-    assert message_85 == respond_plots
-    assert bytes(message_85) == bytes(respond_plots)
+    message_85 = type(request_plots).from_bytes(message_bytes)
+    assert message_85 == request_plots
+    assert bytes(message_85) == bytes(request_plots)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_86 = type(request_peers_introducer).from_bytes(message_bytes)
-    assert message_86 == request_peers_introducer
-    assert bytes(message_86) == bytes(request_peers_introducer)
+    message_86 = type(respond_plots).from_bytes(message_bytes)
+    assert message_86 == respond_plots
+    assert bytes(message_86) == bytes(respond_plots)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_87 = type(respond_peers_introducer).from_bytes(message_bytes)
-    assert message_87 == respond_peers_introducer
-    assert bytes(message_87) == bytes(respond_peers_introducer)
+    message_87 = type(request_peers_introducer).from_bytes(message_bytes)
+    assert message_87 == request_peers_introducer
+    assert bytes(message_87) == bytes(request_peers_introducer)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_88 = type(authentication_payload).from_bytes(message_bytes)
-    assert message_88 == authentication_payload
-    assert bytes(message_88) == bytes(authentication_payload)
+    message_88 = type(respond_peers_introducer).from_bytes(message_bytes)
+    assert message_88 == respond_peers_introducer
+    assert bytes(message_88) == bytes(respond_peers_introducer)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_89 = type(get_pool_info_response).from_bytes(message_bytes)
-    assert message_89 == get_pool_info_response
-    assert bytes(message_89) == bytes(get_pool_info_response)
+    message_89 = type(authentication_payload).from_bytes(message_bytes)
+    assert message_89 == authentication_payload
+    assert bytes(message_89) == bytes(authentication_payload)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_90 = type(post_partial_payload).from_bytes(message_bytes)
-    assert message_90 == post_partial_payload
-    assert bytes(message_90) == bytes(post_partial_payload)
+    message_90 = type(get_pool_info_response).from_bytes(message_bytes)
+    assert message_90 == get_pool_info_response
+    assert bytes(message_90) == bytes(get_pool_info_response)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_91 = type(post_partial_request).from_bytes(message_bytes)
-    assert message_91 == post_partial_request
-    assert bytes(message_91) == bytes(post_partial_request)
+    message_91 = type(post_partial_payload).from_bytes(message_bytes)
+    assert message_91 == post_partial_payload
+    assert bytes(message_91) == bytes(post_partial_payload)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_92 = type(post_partial_response).from_bytes(message_bytes)
-    assert message_92 == post_partial_response
-    assert bytes(message_92) == bytes(post_partial_response)
+    message_92 = type(post_partial_request).from_bytes(message_bytes)
+    assert message_92 == post_partial_request
+    assert bytes(message_92) == bytes(post_partial_request)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_93 = type(get_farmer_response).from_bytes(message_bytes)
-    assert message_93 == get_farmer_response
-    assert bytes(message_93) == bytes(get_farmer_response)
+    message_93 = type(post_partial_response).from_bytes(message_bytes)
+    assert message_93 == post_partial_response
+    assert bytes(message_93) == bytes(post_partial_response)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_94 = type(post_farmer_payload).from_bytes(message_bytes)
-    assert message_94 == post_farmer_payload
-    assert bytes(message_94) == bytes(post_farmer_payload)
+    message_94 = type(get_farmer_response).from_bytes(message_bytes)
+    assert message_94 == get_farmer_response
+    assert bytes(message_94) == bytes(get_farmer_response)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_95 = type(post_farmer_request).from_bytes(message_bytes)
-    assert message_95 == post_farmer_request
-    assert bytes(message_95) == bytes(post_farmer_request)
+    message_95 = type(post_farmer_payload).from_bytes(message_bytes)
+    assert message_95 == post_farmer_payload
+    assert bytes(message_95) == bytes(post_farmer_payload)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_96 = type(post_farmer_response).from_bytes(message_bytes)
-    assert message_96 == post_farmer_response
-    assert bytes(message_96) == bytes(post_farmer_response)
+    message_96 = type(post_farmer_request).from_bytes(message_bytes)
+    assert message_96 == post_farmer_request
+    assert bytes(message_96) == bytes(post_farmer_request)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_97 = type(put_farmer_payload).from_bytes(message_bytes)
-    assert message_97 == put_farmer_payload
-    assert bytes(message_97) == bytes(put_farmer_payload)
+    message_97 = type(post_farmer_response).from_bytes(message_bytes)
+    assert message_97 == post_farmer_response
+    assert bytes(message_97) == bytes(post_farmer_response)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_98 = type(put_farmer_request).from_bytes(message_bytes)
-    assert message_98 == put_farmer_request
-    assert bytes(message_98) == bytes(put_farmer_request)
+    message_98 = type(put_farmer_payload).from_bytes(message_bytes)
+    assert message_98 == put_farmer_payload
+    assert bytes(message_98) == bytes(put_farmer_payload)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_99 = type(put_farmer_response).from_bytes(message_bytes)
-    assert message_99 == put_farmer_response
-    assert bytes(message_99) == bytes(put_farmer_response)
+    message_99 = type(put_farmer_request).from_bytes(message_bytes)
+    assert message_99 == put_farmer_request
+    assert bytes(message_99) == bytes(put_farmer_request)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_100 = type(error_response).from_bytes(message_bytes)
-    assert message_100 == error_response
-    assert bytes(message_100) == bytes(error_response)
+    message_100 = type(put_farmer_response).from_bytes(message_bytes)
+    assert message_100 == put_farmer_response
+    assert bytes(message_100) == bytes(put_farmer_response)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_101 = type(new_peak_timelord).from_bytes(message_bytes)
-    assert message_101 == new_peak_timelord
-    assert bytes(message_101) == bytes(new_peak_timelord)
+    message_101 = type(error_response).from_bytes(message_bytes)
+    assert message_101 == error_response
+    assert bytes(message_101) == bytes(error_response)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_102 = type(new_unfinished_block_timelord).from_bytes(message_bytes)
-    assert message_102 == new_unfinished_block_timelord
-    assert bytes(message_102) == bytes(new_unfinished_block_timelord)
+    message_102 = type(new_peak_timelord).from_bytes(message_bytes)
+    assert message_102 == new_peak_timelord
+    assert bytes(message_102) == bytes(new_peak_timelord)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_103 = type(new_infusion_point_vdf).from_bytes(message_bytes)
-    assert message_103 == new_infusion_point_vdf
-    assert bytes(message_103) == bytes(new_infusion_point_vdf)
+    message_103 = type(new_unfinished_block_timelord).from_bytes(message_bytes)
+    assert message_103 == new_unfinished_block_timelord
+    assert bytes(message_103) == bytes(new_unfinished_block_timelord)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_104 = type(new_signage_point_vdf).from_bytes(message_bytes)
-    assert message_104 == new_signage_point_vdf
-    assert bytes(message_104) == bytes(new_signage_point_vdf)
+    message_104 = type(new_infusion_point_vdf).from_bytes(message_bytes)
+    assert message_104 == new_infusion_point_vdf
+    assert bytes(message_104) == bytes(new_infusion_point_vdf)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_105 = type(new_end_of_sub_slot_bundle).from_bytes(message_bytes)
-    assert message_105 == new_end_of_sub_slot_bundle
-    assert bytes(message_105) == bytes(new_end_of_sub_slot_bundle)
+    message_105 = type(new_signage_point_vdf).from_bytes(message_bytes)
+    assert message_105 == new_signage_point_vdf
+    assert bytes(message_105) == bytes(new_signage_point_vdf)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_106 = type(request_compact_proof_of_time).from_bytes(message_bytes)
-    assert message_106 == request_compact_proof_of_time
-    assert bytes(message_106) == bytes(request_compact_proof_of_time)
+    message_106 = type(new_end_of_sub_slot_bundle).from_bytes(message_bytes)
+    assert message_106 == new_end_of_sub_slot_bundle
+    assert bytes(message_106) == bytes(new_end_of_sub_slot_bundle)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_107 = type(respond_compact_proof_of_time).from_bytes(message_bytes)
-    assert message_107 == respond_compact_proof_of_time
-    assert bytes(message_107) == bytes(respond_compact_proof_of_time)
+    message_107 = type(request_compact_proof_of_time).from_bytes(message_bytes)
+    assert message_107 == request_compact_proof_of_time
+    assert bytes(message_107) == bytes(request_compact_proof_of_time)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_108 = type(error_without_data).from_bytes(message_bytes)
-    assert message_108 == error_without_data
-    assert bytes(message_108) == bytes(error_without_data)
+    message_108 = type(respond_compact_proof_of_time).from_bytes(message_bytes)
+    assert message_108 == respond_compact_proof_of_time
+    assert bytes(message_108) == bytes(respond_compact_proof_of_time)
 
     message_bytes, input_bytes = parse_blob(input_bytes)
-    message_109 = type(error_with_data).from_bytes(message_bytes)
-    assert message_109 == error_with_data
-    assert bytes(message_109) == bytes(error_with_data)
+    message_109 = type(error_without_data).from_bytes(message_bytes)
+    assert message_109 == error_without_data
+    assert bytes(message_109) == bytes(error_without_data)
+
+    message_bytes, input_bytes = parse_blob(input_bytes)
+    message_110 = type(error_with_data).from_bytes(message_bytes)
+    assert message_110 == error_with_data
+    assert bytes(message_110) == bytes(error_with_data)
 
     assert input_bytes == b""

chia/_tests/util/test_network_protocol_files.py

@@ -194,6 +194,11 @@ def test_protocol_json() -> None:
         type(new_signage_point_harvester).from_json_dict(new_signage_point_harvester_json)
         == new_signage_point_harvester
     )
+    assert str(new_signage_point_harvester2_json) == str(new_signage_point_harvester2.to_json_dict())
+    assert (
+        type(new_signage_point_harvester2).from_json_dict(new_signage_point_harvester2_json)
+        == new_signage_point_harvester2
+    )
     assert str(new_proof_of_space_json) == str(new_proof_of_space.to_json_dict())
     assert type(new_proof_of_space).from_json_dict(new_proof_of_space_json) == new_proof_of_space
     assert str(request_signatures_json) == str(request_signatures.to_json_dict())

chia/_tests/util/test_network_protocol_json.py

@@ -172,6 +172,7 @@ def test_missing_messages() -> None:
         "ProofOfSpaceFeeInfo",
         "NewProofOfSpace",
         "NewSignagePointHarvester",
+        "NewSignagePointHarvester2",
         "Plot",
         "PlotSyncDone",
         "PlotSyncError",

chia/_tests/util/test_network_protocol_test.py

@@ -6,9 +6,10 @@
 from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union, cast
 
 import aiohttp
-from chia_rs import AugSchemeMPL, G2Element, PoolTarget, PrivateKey
+from chia_rs import AugSchemeMPL, G2Element, PlotSize, PoolTarget, PrivateKey
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint8, uint16, uint32, uint64
+from packaging.version import Version
 
 from chia import __version__
 from chia.consensus.pot_iterations import calculate_iterations_quality, calculate_sp_interval_iters
@@ -38,6 +39,7 @@
 from chia.server.ws_connection import WSChiaConnection
 from chia.ssl.create_ssl import get_mozilla_ca_crt
 from chia.types.blockchain_format.proof_of_space import (
+    calculate_prefix_bits,
     generate_plot_public_key,
     generate_taproot_sk,
     get_plot_id,
@@ -528,7 +530,8 @@ async def new_signage_point(self, new_signage_point: farmer_protocol.NewSignageP
                         p2_singleton_puzzle_hash,
                     )
                 )
-            message = harvester_protocol.NewSignagePointHarvester(
+
+            message2 = harvester_protocol.NewSignagePointHarvester2(
                 new_signage_point.challenge_hash,
                 new_signage_point.difficulty,
                 new_signage_point.sub_slot_iters,
@@ -539,8 +542,31 @@ async def new_signage_point(self, new_signage_point: farmer_protocol.NewSignageP
                 new_signage_point.last_tx_height,
             )
 
-            msg = make_msg(ProtocolMessageTypes.new_signage_point_harvester, message)
-            await self.farmer.server.send_to_all([msg], NodeType.HARVESTER)
+            # The plot size in the call to calculate_prefix_bits is only used
+            # to distinguish v1 and v2 plots. The value does not matter
+            message1 = harvester_protocol.NewSignagePointHarvester(
+                new_signage_point.challenge_hash,
+                new_signage_point.difficulty,
+                new_signage_point.sub_slot_iters,
+                new_signage_point.signage_point_index,
+                new_signage_point.challenge_chain_sp,
+                pool_difficulties,
+                uint8(
+                    calculate_prefix_bits(self.farmer.constants, new_signage_point.peak_height, PlotSize.make_v1(32))
+                ),
+            )
+
+            def old_harvesters(conn: WSChiaConnection) -> bool:
+                return conn.protocol_version <= Version("0.0.36")
+
+            def new_harvesters(conn: WSChiaConnection) -> bool:
+                return conn.protocol_version > Version("0.0.36")
+
+            msg1 = make_msg(ProtocolMessageTypes.new_signage_point_harvester, message1)
+            await self.farmer.server.send_to_all_if([msg1], NodeType.HARVESTER, old_harvesters)
+
+            msg2 = make_msg(ProtocolMessageTypes.new_signage_point_harvester, message2)
+            await self.farmer.server.send_to_all_if([msg2], NodeType.HARVESTER, new_harvesters)
         except Exception as exception:
             # Remove here, as we want to reprocess the SP should it be sent again
             self.farmer.sps[new_signage_point.challenge_chain_sp].remove(new_signage_point)