Ms.parallel harvester (#194)

* Harvester fetches qualities in parallel and returns qualities asynchronously to improve efficiency

Author: mariano54

src/harvester.py

@@ -158,8 +158,11 @@ async def new_challenge(self, new_challenge: harvester_protocol.NewChallenge):
             raise ValueError(
                 f"Invalid challenge size {challenge_size}, 32 was expected"
             )
-        all_responses = []
-        for filename, prover in self.provers.items():
+
+        async def lookup_challenge(
+            filename: Path, prover: DiskProver
+        ) -> List[harvester_protocol.ChallengeResponse]:
+            all_responses: List[harvester_protocol.ChallengeResponse] = []
             try:
                 quality_strings = prover.get_qualities_for_challenge(
                     new_challenge.challenge_hash
@@ -187,12 +190,21 @@ async def new_challenge(self, new_challenge: harvester_protocol.NewChallenge):
                         new_challenge.challenge_hash, quality_str, prover.get_size()
                     )
                     all_responses.append(response)
-        for response in all_responses:
-            yield OutboundMessage(
-                NodeType.FARMER,
-                Message("challenge_response", response),
-                Delivery.RESPOND,
-            )
+            return all_responses
+
+        awaitables = [
+            lookup_challenge(filename, prover)
+            for filename, prover in self.provers.items()
+        ]
+
+        # Concurrently executes all lookups on disk, to take advantage of multiple disk parallelism
+        for sublist_awaitable in asyncio.as_completed(awaitables):
+            for response in await sublist_awaitable:
+                yield OutboundMessage(
+                    NodeType.FARMER,
+                    Message("challenge_response", response),
+                    Delivery.RESPOND,
+                )
 
     @api_request
     async def request_proof_of_space(