harvester and farmer v2 support

Author: almogdepaz

chia/_tests/harvester/test_harvester_api.py

@@ -0,0 +1,209 @@
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import pytest
+from chia_rs import ProofOfSpace
+from chia_rs.sized_bytes import bytes32
+from chia_rs.sized_ints import uint64
+
+from chia._tests.conftest import HarvesterFarmerEnvironment
+from chia.harvester.harvester_api import HarvesterAPI
+from chia.plotting.util import PlotInfo
+from chia.protocols import harvester_protocol
+from chia.protocols.harvester_protocol import PoolDifficulty
+from chia.server.ws_connection import WSChiaConnection
+from chia.simulator.block_tools import BlockTools
+
+
+def create_signage_point_harvester_from_constants(bt: BlockTools) -> harvester_protocol.NewSignagePointHarvester:
+    """create a NewSignagePointHarvester using real constants from block tools"""
+    # use the pre-generated signage point data from network_protocol_data.py
+    # but with real constants from block_tools
+    from chia._tests.util.network_protocol_data import new_signage_point_harvester
+
+    # create a version with real constants values
+    return harvester_protocol.NewSignagePointHarvester(
+        challenge_hash=new_signage_point_harvester.challenge_hash,
+        difficulty=uint64(bt.constants.DIFFICULTY_STARTING),
+        sub_slot_iters=uint64(bt.constants.SUB_SLOT_ITERS_STARTING),
+        signage_point_index=new_signage_point_harvester.signage_point_index,
+        sp_hash=new_signage_point_harvester.sp_hash,
+        pool_difficulties=[],  # empty for simplicity, unless testing pool functionality
+        peak_height=new_signage_point_harvester.peak_height,
+        last_tx_height=new_signage_point_harvester.last_tx_height,
+    )
+
+
+@pytest.mark.anyio
+async def test_new_signage_point_harvester_no_keys(
+    harvester_farmer_environment: HarvesterFarmerEnvironment,
+) -> None:
+    """test that new_signage_point_harvester returns early when no keys available"""
+    _farmer_service, _farmer_rpc_client, harvester_service, _harvester_rpc_client, bt = harvester_farmer_environment
+    harvester_api = harvester_service._server.api
+    assert isinstance(harvester_api, HarvesterAPI)
+
+    # create real signage point data from block tools
+    new_challenge = create_signage_point_harvester_from_constants(bt)
+
+    # mock plot manager to return false for public_keys_available
+    with patch.object(harvester_api.harvester.plot_manager, "public_keys_available", return_value=False):
+        mock_peer = MagicMock(spec=WSChiaConnection)
+
+        result = harvester_api.new_signage_point_harvester(new_challenge, mock_peer)
+        assert result is None
+
+
+@pytest.mark.anyio
+async def test_new_signage_point_harvester_happy_path(
+    harvester_farmer_environment: HarvesterFarmerEnvironment,
+) -> None:
+    """test successful signage point processing with valid plots"""
+    _farmer_service, _farmer_rpc_client, harvester_service, _harvester_rpc_client, bt = harvester_farmer_environment
+    harvester_api = harvester_service._server.api
+    assert isinstance(harvester_api, HarvesterAPI)
+
+    # create real signage point data from block tools
+    new_challenge = create_signage_point_harvester_from_constants(bt)
+
+    mock_peer = MagicMock(spec=WSChiaConnection)
+
+    # create mock plot info
+    mock_prover = MagicMock()
+    mock_prover.get_id.return_value = bytes32(b"2" * 32)
+    mock_prover.get_size.return_value = 32
+    mock_prover.get_qualities_for_challenge.return_value = [bytes32(b"quality" + b"0" * 25)]
+
+    mock_plot_info = MagicMock(spec=PlotInfo)
+    mock_plot_info.prover = mock_prover
+    mock_plot_info.pool_contract_puzzle_hash = None
+
+    plot_path = Path("/fake/plot.plot")
+
+    with patch.object(harvester_api.harvester.plot_manager, "public_keys_available", return_value=True):
+        with patch.object(harvester_api.harvester.plot_manager, "plots", {plot_path: mock_plot_info}):
+            with patch("chia.harvester.harvester_api.passes_plot_filter", return_value=True):
+                with patch("chia.harvester.harvester_api.calculate_pos_challenge") as mock_calc_pos:
+                    mock_calc_pos.return_value = bytes32(b"sp_challenge" + b"0" * 20)
+
+                    with patch("chia.harvester.harvester_api.calculate_iterations_quality") as mock_calc_iter:
+                        # set required_iters low enough to pass the sp_interval_iters check
+                        mock_calc_iter.return_value = uint64(1000)
+
+                        with patch("chia.harvester.harvester_api.calculate_sp_interval_iters") as mock_sp_interval:
+                            mock_sp_interval.return_value = uint64(10000)
+
+                            with patch.object(mock_prover, "get_full_proof") as mock_get_proof:
+                                mock_proof = MagicMock(spec=ProofOfSpace)
+                                mock_get_proof.return_value = mock_proof, None
+
+                                result = harvester_api.new_signage_point_harvester(new_challenge, mock_peer)
+                                # function returns None but should have processed the plot
+                                assert result is None
+
+
+@pytest.mark.anyio
+async def test_new_signage_point_harvester_pool_difficulty_override(
+    harvester_farmer_environment: HarvesterFarmerEnvironment,
+) -> None:
+    """test that pool difficulty overrides are applied correctly"""
+    _farmer_service, _farmer_rpc_client, harvester_service, _harvester_rpc_client, bt = harvester_farmer_environment
+    harvester_api = harvester_service._server.api
+    assert isinstance(harvester_api, HarvesterAPI)
+
+    mock_peer = MagicMock(spec=WSChiaConnection)
+
+    pool_puzzle_hash = bytes32(b"pool" + b"0" * 28)
+
+    mock_prover = MagicMock()
+    mock_prover.get_id.return_value = bytes32(b"2" * 32)
+    mock_prover.get_size.return_value = 32
+    mock_prover.get_qualities_for_challenge.return_value = [bytes32(b"quality" + b"0" * 25)]
+
+    mock_plot_info = MagicMock(spec=PlotInfo)
+    mock_plot_info.prover = mock_prover
+    mock_plot_info.pool_contract_puzzle_hash = pool_puzzle_hash
+
+    plot_path = Path("/fake/plot.plot")
+
+    pool_difficulty = PoolDifficulty(
+        pool_contract_puzzle_hash=pool_puzzle_hash,
+        difficulty=uint64(500),  # lower than main difficulty
+        sub_slot_iters=uint64(67108864),  # different from main
+    )
+
+    # create real signage point data from constants with pool difficulty
+    new_challenge = create_signage_point_harvester_from_constants(bt)
+    # override with pool difficulty for this test
+    new_challenge = harvester_protocol.NewSignagePointHarvester(
+        challenge_hash=new_challenge.challenge_hash,
+        difficulty=new_challenge.difficulty,
+        sub_slot_iters=new_challenge.sub_slot_iters,
+        signage_point_index=new_challenge.signage_point_index,
+        sp_hash=new_challenge.sp_hash,
+        pool_difficulties=[pool_difficulty],  # add pool difficulty
+        peak_height=new_challenge.peak_height,
+        last_tx_height=new_challenge.last_tx_height,
+    )
+
+    with patch.object(harvester_api.harvester.plot_manager, "public_keys_available", return_value=True):
+        with patch.object(harvester_api.harvester.plot_manager, "plots", {plot_path: mock_plot_info}):
+            with patch("chia.harvester.harvester_api.passes_plot_filter", return_value=True):
+                with patch("chia.harvester.harvester_api.calculate_pos_challenge") as mock_calc_pos:
+                    mock_calc_pos.return_value = bytes32(b"sp_challenge" + b"0" * 20)
+
+                    with patch("chia.harvester.harvester_api.calculate_iterations_quality") as mock_calc_iter:
+                        mock_calc_iter.return_value = uint64(1000)
+
+                        with patch("chia.harvester.harvester_api.calculate_sp_interval_iters") as mock_sp_interval:
+                            mock_sp_interval.return_value = uint64(10000)
+
+                            with patch.object(mock_prover, "get_full_proof") as mock_get_proof:
+                                mock_proof = MagicMock(spec=ProofOfSpace)
+                                mock_get_proof.return_value = mock_proof, None
+
+                                result = harvester_api.new_signage_point_harvester(new_challenge, mock_peer)
+
+                                # verify that calculate_iterations_quality was called with pool difficulty
+                                mock_calc_iter.assert_called()
+                                call_args = mock_calc_iter.call_args[0]
+                                assert call_args[3] == uint64(500)  # pool difficulty was used
+
+                                assert result is None
+
+
+@pytest.mark.anyio
+async def test_new_signage_point_harvester_prover_error(
+    harvester_farmer_environment: HarvesterFarmerEnvironment,
+) -> None:
+    """test error handling when prover fails"""
+    _farmer_service, _farmer_rpc_client, harvester_service, _harvester_rpc_client, bt = harvester_farmer_environment
+    harvester_api = harvester_service._server.api
+    assert isinstance(harvester_api, HarvesterAPI)
+
+    # create real signage point data from block tools
+    new_challenge = create_signage_point_harvester_from_constants(bt)
+
+    mock_peer = MagicMock(spec=WSChiaConnection)
+
+    mock_prover = MagicMock()
+    mock_prover.get_id.return_value = bytes32(b"2" * 32)
+    mock_prover.get_qualities_for_challenge.side_effect = RuntimeError("test error")
+
+    mock_plot_info = MagicMock(spec=PlotInfo)
+    mock_plot_info.prover = mock_prover
+    mock_plot_info.pool_contract_puzzle_hash = None
+
+    plot_path = Path("/fake/plot.plot")
+
+    with patch.object(harvester_api.harvester.plot_manager, "public_keys_available", return_value=True):
+        with patch.object(harvester_api.harvester.plot_manager, "plots", {plot_path: mock_plot_info}):
+            with patch("chia.harvester.harvester_api.passes_plot_filter", return_value=True):
+                with patch("chia.harvester.harvester_api.calculate_pos_challenge") as mock_calc_pos:
+                    mock_calc_pos.return_value = bytes32(b"sp_challenge" + b"0" * 20)
+
+                    # should not raise exception, should handle error gracefully
+                    result = harvester_api.new_signage_point_harvester(new_challenge, mock_peer)
+                    assert result is None

chia/farmer/farmer_api.py

@@ -6,7 +6,7 @@
 from typing import TYPE_CHECKING, Any, ClassVar, Optional, Union, cast
 
 import aiohttp
-from chia_rs import AugSchemeMPL, G2Element, PoolTarget, PrivateKey
+from chia_rs import AugSchemeMPL, G2Element, PoolTarget, PrivateKey, ProofOfSpace
 from chia_rs.sized_bytes import bytes32
 from chia_rs.sized_ints import uint8, uint16, uint32, uint64
 
@@ -15,7 +15,7 @@
 from chia.farmer.farmer import Farmer, increment_pool_stats, strip_old_entries
 from chia.harvester.harvester_api import HarvesterAPI
 from chia.protocols import farmer_protocol, harvester_protocol
-from chia.protocols.farmer_protocol import DeclareProofOfSpace, SignedValues
+from chia.protocols.farmer_protocol import DeclareProofOfSpace, SignedValues, SolutionResponse
 from chia.protocols.harvester_protocol import (
     PlotSyncDone,
     PlotSyncPathList,
@@ -24,6 +24,7 @@
     PoolDifficulty,
     SignatureRequestSourceData,
     SigningDataKind,
+    V2Qualities,
 )
 from chia.protocols.outbound_message import Message, NodeType, make_msg
 from chia.protocols.pool_protocol import (
@@ -33,6 +34,7 @@
     get_current_authentication_token,
 )
 from chia.protocols.protocol_message_types import ProtocolMessageTypes
+from chia.protocols.solver_protocol import SolverInfo
 from chia.server.api_protocol import ApiMetadata
 from chia.server.server import ssl_context_for_root
 from chia.server.ws_connection import WSChiaConnection
@@ -73,7 +75,7 @@ async def new_proof_of_space(
         """
         if new_proof_of_space.sp_hash not in self.farmer.number_of_responses:
             self.farmer.number_of_responses[new_proof_of_space.sp_hash] = 0
-            self.farmer.cache_add_time[new_proof_of_space.sp_hash] = uint64(time.time())
+            self.farmer.cache_add_time[new_proof_of_space.sp_hash] = uint64(int(time.time()))
 
         max_pos_per_sp = 5
 
@@ -170,14 +172,14 @@ async def new_proof_of_space(
                         new_proof_of_space.proof,
                     )
                 )
-                self.farmer.cache_add_time[new_proof_of_space.sp_hash] = uint64(time.time())
+                self.farmer.cache_add_time[new_proof_of_space.sp_hash] = uint64(int(time.time()))
                 self.farmer.quality_str_to_identifiers[computed_quality_string] = (
                     new_proof_of_space.plot_identifier,
                     new_proof_of_space.challenge_hash,
                     new_proof_of_space.sp_hash,
                     peer.peer_node_id,
                 )
-                self.farmer.cache_add_time[computed_quality_string] = uint64(time.time())
+                self.farmer.cache_add_time[computed_quality_string] = uint64(int(time.time()))
 
                 await peer.send_message(make_msg(ProtocolMessageTypes.request_signatures, request))
 
@@ -478,6 +480,92 @@ async def new_proof_of_space(
 
                 return
 
+    @metadata.request(peer_required=True)
+    async def v2_qualities(self, quality_collection: V2Qualities, peer: WSChiaConnection) -> None:
+        """
+        This is a response from the harvester for V2 plots, containing only qualities.
+        We store these qualities and will later use solver service to generate proofs when needed.
+        """
+        if quality_collection.sp_hash not in self.farmer.number_of_responses:
+            self.farmer.number_of_responses[quality_collection.sp_hash] = 0
+            self.farmer.cache_add_time[quality_collection.sp_hash] = uint64(int(time.time()))
+
+        if quality_collection.sp_hash not in self.farmer.sps:
+            self.farmer.log.warning(
+                f"Received V2 quality collection for a signage point that we do not have {quality_collection.sp_hash}"
+            )
+            return None
+
+        self.farmer.cache_add_time[quality_collection.sp_hash] = uint64(int(time.time()))
+
+        self.farmer.log.info(
+            f"Received V2 quality collection with {len(quality_collection.qualities)} qualities "
+            f"for plot {quality_collection.plot_identifier[:10]}... from {peer.peer_node_id}"
+        )
+
+        # Process each quality through solver service to get full proofs
+        for quality in quality_collection.qualities:
+            solver_info = SolverInfo(
+                plot_size=quality_collection.plot_size,
+                plot_diffculty=quality_collection.difficulty,  # Note: typo in SolverInfo field name
+                quality_string=quality,
+            )
+
+            # Call solver service to get proof
+            # TODO: Add proper solver service node connection to farmer
+            # For now, assume solver service is available via server connections
+            proof_bytes = None
+
+            # Try to call solver service - this requires farmer to have solver connections configured
+            try:
+                # Send solve request to solver service
+                # This would work if farmer is connected to a solver service node
+                solver_response = await self.farmer.server.send_to_all_and_wait_first(
+                    [make_msg(ProtocolMessageTypes.solve, solver_info)],
+                    NodeType.FARMER,  # TODO: Need SOLVER node type
+                )
+
+                if solver_response is not None and isinstance(solver_response, SolutionResponse):
+                    proof_bytes = solver_response.proof
+                    self.farmer.log.debug(f"Received {len(proof_bytes)} byte proof from solver")
+                else:
+                    self.farmer.log.warning(f"No valid solver response for quality {quality.hex()[:10]}...")
+
+            except Exception as e:
+                self.farmer.log.error(f"Failed to call solver service for quality {quality.hex()[:10]}...: {e}")
+
+            # Fall back to stub if solver service unavailable
+            if proof_bytes is None:
+                self.farmer.log.warning("Using stub proof - solver service not available")
+                proof_bytes = b"stub_proof_from_solver"
+
+            # Create ProofOfSpace object using solver response and plot metadata
+            # Need to calculate sp_challenge_hash for ProofOfSpace constructor
+            # TODO: We need plot_id to calculate this properly - may need to add to V2Qualities
+            sp_challenge_hash = quality_collection.challenge_hash  # Approximation for now
+
+            # Create a NewProofOfSpace object that can go through existing flow
+            new_proof_of_space = harvester_protocol.NewProofOfSpace(
+                quality_collection.challenge_hash,
+                quality_collection.sp_hash,
+                quality_collection.plot_identifier,
+                ProofOfSpace(
+                    sp_challenge_hash,
+                    quality_collection.pool_public_key,
+                    quality_collection.pool_contract_puzzle_hash,
+                    quality_collection.plot_public_key,
+                    quality_collection.plot_size,
+                    proof_bytes,
+                ),
+                quality_collection.signage_point_index,
+                include_source_signature_data=False,
+                farmer_reward_address_override=None,
+                fee_info=None,
+            )
+
+            # Route through existing new_proof_of_space flow
+            await self.new_proof_of_space(new_proof_of_space, peer)
+
     @metadata.request()
     async def respond_signatures(self, response: harvester_protocol.RespondSignatures) -> None:
         request = self._process_respond_signatures(response)
@@ -558,7 +646,7 @@ async def new_signage_point(self, new_signage_point: farmer_protocol.NewSignageP
 
                     pool_dict[key] = strip_old_entries(pairs=pool_dict[key], before=cutoff_24h)
 
-        now = uint64(time.time())
+        now = uint64(int(time.time()))
         self.farmer.cache_add_time[new_signage_point.challenge_chain_sp] = now
         missing_signage_points = self.farmer.check_missing_signage_points(now, new_signage_point)
         self.farmer.state_changed(