Find the chacing problem.

Migrate most actor based tests to normal force cloud rq tests (for future parametrization of the Apify clients)

Author: Pijukatel

src/apify/storage_clients/_apify/_request_queue_client_simple.py

@@ -32,6 +32,7 @@ class ApifyRequestQueueClientSimple(ApifyRequestQueueClient):
     - Multiple producers can put requests to the queue, but their forefront requests are not guaranteed to be handled
       so quickly as this client does not aggressively fetch the forefront and relies on local head estimation.
     - Requests are only added to the queue, never deleted. (Marking as handled is ok.)
+    - Other producers can add new requests, but not modify existing ones (otherwise caching can miss the updates)
 
     If the constraints are not met, the client might work in an unpredictable way.
     """
@@ -247,7 +248,13 @@ async def _list_head(self) -> None:
                 # Do not cache fully handled requests, we do not need them. Just cache their unique_key.
                 self._requests_already_handled.add(request.unique_key)
             else:
-                self._requests_cache[request.unique_key] = request
+                # Only fetch the request if we do not know it yet.
+                if request.unique_key not in self._requests_cache:
+                    request = Request.model_validate(
+                        await self._api_client.get_request(unique_key_to_request_id(request.unique_key))
+                    )
+                    self._requests_cache[request.unique_key] = request
+
                 # Add new requests to the end of the head, unless already present in head
                 if request.unique_key not in self._head_requests:
                     self._head_requests.appendleft(request.unique_key)

tests/integration/test_crawlers_with_storages.py

@@ -41,35 +41,6 @@ async def default_handler(context: ParselCrawlingContext) -> None:
     assert run_result.status == 'SUCCEEDED'
 
 
-async def test_actor_on_platform_max_crawl_depth(
-) -> None:
-    """Test that the actor respects max_crawl_depth."""
-
-    """The crawler entry point."""
-    import re
-
-    from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
-
-    from apify import Actor
-
-    async with Actor:
-        rq= await Actor.open_request_queue(force_cloud=True)
-        crawler = ParselCrawler(max_crawl_depth=2, request_manager=rq)
-        finished = []
-        enqueue_pattern = re.compile(r'http://localhost:8080/2+$')
-
-        @crawler.router.default_handler
-        async def default_handler(context: ParselCrawlingContext) -> None:
-            """Default request handler."""
-            context.log.info(f'Processing {context.request.url} ...')
-            await context.enqueue_links(include=[enqueue_pattern])
-            finished.append(context.request.url)
-
-        await crawler.run(['http://localhost:8080/'])
-        assert finished == ['http://localhost:8080/', 'http://localhost:8080/2', 'http://localhost:8080/22']
-
-
-
 async def test_actor_on_platform_max_requests_per_crawl(
     make_actor: MakeActorFunction,
     run_actor: RunActorFunction,