fix serialization of partial-proofs (#20188)

* fix serialization of partial-proofs (list[uint64]) to bytes, for use as key in dictionary

* improve tests for partial-proof serialization

Author: arvidn

chia/_tests/farmer_harvester/test_farmer_harvester.py

@@ -17,6 +17,7 @@
 from chia._tests.util.time_out_assert import time_out_assert
 from chia.cmds.cmds_util import get_any_service_client
 from chia.farmer.farmer import Farmer
+from chia.farmer.farmer_api import serialize
 from chia.farmer.farmer_service import FarmerService
 from chia.harvester.harvester_rpc_client import HarvesterRpcClient
 from chia.harvester.harvester_service import HarvesterService
@@ -338,7 +339,7 @@ async def test_v2_partial_proofs_new_sp_hash(
         challenge_hash=bytes32(b"2" * 32),
         sp_hash=sp_hash,
         plot_identifier="test_plot_id",
-        partial_proofs=[[uint64(1), uint64(2), uint64(3), uint64(4)]],
+        partial_proofs=[[uint64(100), uint64(200), uint64(300), uint64(400)]],
         signage_point_index=uint8(0),
         plot_size=uint8(32),
         strength=uint8(5),
@@ -369,7 +370,7 @@ async def test_v2_partial_proofs_missing_sp_hash(
         challenge_hash=bytes32(b"2" * 32),
         sp_hash=sp_hash,
         plot_identifier="test_plot_id",
-        partial_proofs=[[uint64(1), uint64(2), uint64(3), uint64(4)]],
+        partial_proofs=[[uint64(100), uint64(200), uint64(300), uint64(400)]],
         signage_point_index=uint8(0),
         plot_size=uint8(32),
         plot_id=bytes32.fromhex("abababababababababababababababababababababababababababababababab"),
@@ -413,7 +414,10 @@ async def test_v2_partial_proofs_with_existing_sp(
         challenge_hash=challenge_hash,
         sp_hash=sp_hash,
         plot_identifier="test_plot_id",
-        partial_proofs=[[uint64(1), uint64(2), uint64(3), uint64(4)], [uint64(2), uint64(3), uint64(4), uint64(5)]],
+        partial_proofs=[
+            [uint64(100), uint64(200), uint64(300), uint64(400)],
+            [uint64(2222), uint64(3333), uint64(4444), uint64(5555)],
+        ],
         signage_point_index=uint8(0),
         plot_size=uint8(32),
         plot_id=bytes32.fromhex("abababababababababababababababababababababababababababababababab"),
@@ -447,7 +451,7 @@ async def test_solution_response_handler(
         challenge_hash=challenge_hash,
         sp_hash=sp_hash,
         plot_identifier="test_plot_id",
-        partial_proofs=[[uint64(1), uint64(2), uint64(3), uint64(4)]],
+        partial_proofs=[[uint64(1111), uint64(2222), uint64(3333), uint64(4444)]],
         signage_point_index=uint8(0),
         plot_size=uint8(32),
         plot_id=bytes32.fromhex("abababababababababababababababababababababababababababababababab"),
@@ -460,7 +464,7 @@ async def test_solution_response_handler(
     harvester_peer = await get_harvester_peer(farmer)
 
     # manually add pending request
-    key = bytes(partial_proofs.partial_proofs[0])
+    key = serialize(partial_proofs.partial_proofs[0])
     farmer.pending_solver_requests[key] = {
         "proof_data": partial_proofs,
         "peer": harvester_peer,
@@ -486,7 +490,7 @@ async def test_solution_response_handler(
         assert original_peer == harvester_peer
 
         # verify pending request was removed
-        key = bytes(partial_proofs.partial_proofs[0])
+        key = serialize(partial_proofs.partial_proofs[0])
         assert key not in farmer.pending_solver_requests
 
 
@@ -530,7 +534,10 @@ async def test_solution_response_empty_proof(
         challenge_hash=challenge_hash,
         sp_hash=sp_hash,
         plot_identifier="test_plot_id",
-        partial_proofs=[[uint64(1), uint64(2), uint64(3), uint64(4)], [uint64(2), uint64(3), uint64(4), uint64(5)]],
+        partial_proofs=[
+            [uint64(100), uint64(200), uint64(300), uint64(400)],
+            [uint64(2222), uint64(3333), uint64(4444), uint64(5555)],
+        ],
         signage_point_index=uint8(0),
         plot_size=uint8(32),
         plot_id=bytes32.fromhex("abababababababababababababababababababababababababababababababab"),
@@ -544,7 +551,7 @@ async def test_solution_response_empty_proof(
     harvester_peer.peer_node_id = "harvester_peer"
 
     # manually add pending request
-    key = bytes(partial_proofs.partial_proofs[0])
+    key = serialize(partial_proofs.partial_proofs[0])
     farmer.pending_solver_requests[key] = {
         "proof_data": partial_proofs,
         "peer": harvester_peer,
@@ -563,7 +570,7 @@ async def test_solution_response_empty_proof(
         mock_new_proof.assert_not_called()
 
         # verify pending request was removed (cleanup still happens)
-        key = bytes(partial_proofs.partial_proofs[0])
+        key = serialize(partial_proofs.partial_proofs[0])
         assert key not in farmer.pending_solver_requests
 
 
@@ -595,7 +602,10 @@ async def test_v2_partial_proofs_solver_exception(
         challenge_hash=challenge_hash,
         sp_hash=sp_hash,
         plot_identifier="test_plot_id",
-        partial_proofs=[[uint64(1), uint64(2), uint64(3), uint64(4)], [uint64(2), uint64(3), uint64(4), uint64(5)]],
+        partial_proofs=[
+            [uint64(100), uint64(200), uint64(300), uint64(400)],
+            [uint64(2222), uint64(3333), uint64(4444), uint64(5555)],
+        ],
         signage_point_index=uint8(0),
         plot_size=uint8(32),
         plot_id=bytes32.fromhex("abababababababababababababababababababababababababababababababab"),
@@ -612,5 +622,5 @@ async def test_v2_partial_proofs_solver_exception(
         await farmer_api.partial_proofs(partial_proofs, harvester_peer)
 
         # verify pending request was cleaned up after exception
-        key = bytes(partial_proofs.partial_proofs[0])
+        key = serialize(partial_proofs.partial_proofs[0])
         assert key not in farmer.pending_solver_requests

chia/_tests/solver/test_solver_service.py

@@ -24,7 +24,7 @@ async def test_solver_api_methods(blockchain_constants: ConsensusConstants, tmp_
             solver_api = solver_service._api
             assert solver_api.ready() is True
             test_info = SolverInfo(
-                partial_proof=[uint64(1), uint64(2), uint64(3), uint64(4)],
+                partial_proof=[uint64(1000), uint64(2000), uint64(3000), uint64(4000)],
                 plot_id=bytes32.fromhex("abababababababababababababababababababababababababababababababab"),
                 strength=uint8(5),
                 size=uint8(28),

chia/_tests/util/network_protocol_data.py

@@ -155,7 +155,10 @@
     bytes32.fromhex("42743566108589c11bb3811b347900b6351fd3e25bad6c956c0bf1c05a4d93fb"),
     bytes32.fromhex("8a346e8dc02e9b44c0571caa74fd99f163d4c5d7deaedac87125528721493f7a"),
     "plot-filename",
-    [[uint64(1), uint64(2), uint64(3), uint64(4)], [uint64(2), uint64(3), uint64(4), uint64(5)]],
+    [
+        [uint64(1111), uint64(2222), uint64(3333), uint64(4444)],
+        [uint64(2222), uint64(3333), uint64(4444), uint64(5555)],
+    ],
     uint8(4),
     uint8(32),
     uint8(5),

chia/_tests/util/protocol_messages_bytes-v1.0

@@ -69,7 +69,7 @@
     "challenge_hash": "0x42743566108589c11bb3811b347900b6351fd3e25bad6c956c0bf1c05a4d93fb",
     "sp_hash": "0x8a346e8dc02e9b44c0571caa74fd99f163d4c5d7deaedac87125528721493f7a",
     "plot_identifier": "plot-filename",
-    "partial_proofs": [[1, 2, 3, 4], [2, 3, 4, 5]],
+    "partial_proofs": [[1111, 2222, 3333, 4444], [2222, 3333, 4444, 5555]],
     "signage_point_index": 4,
     "plot_size": 32,
     "strength": 5,

chia/_tests/util/protocol_messages_json.py

@@ -49,6 +49,13 @@
 )
 
 
+def serialize(partial_proof: list[uint64]) -> bytes:
+    key = bytearray()
+    for val in partial_proof:
+        key += val.stream_to_bytes()
+    return bytes(key)
+
+
 class FarmerAPI:
     if TYPE_CHECKING:
         from chia.server.api_protocol import ApiProtocol
@@ -514,7 +521,7 @@ async def partial_proofs(self, partial_proof_data: PartialProofsData, peer: WSCh
                 size=partial_proof_data.plot_size,
             )
 
-            key = bytes(partial_proof)
+            key = serialize(partial_proof)
             try:
                 # store pending request data for matching with response
                 self.farmer.pending_solver_requests[key] = {
@@ -543,7 +550,7 @@ async def solution_response(self, response: SolverResponse, peer: WSChiaConnecti
 
         # find the matching pending request using partial_proof
 
-        key = bytes(response.partial_proof)
+        key = serialize(response.partial_proof)
         if key not in self.farmer.pending_solver_requests:
             self.farmer.log.warning(f"Received solver response for unknown partial proof {response.partial_proof[:5]}")
             return