WIP

Author: Pijukatel

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "2.7.4"
+version = "2.8.1"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "[email protected]" }]
 license = { file = "LICENSE" }

src/apify/storage_clients/_apify/_request_queue_client_simple.py

@@ -248,8 +248,9 @@ async def _list_head(self) -> None:
                 self._requests_already_handled.add(request.unique_key)
             else:
                 self._requests_cache[request.unique_key] = request
-                # Add new requests to the end of the head
-                self._head_requests.appendleft(request.unique_key)
+                # Add new requests to the end of the head, unless already present in head
+                if request.unique_key not in self._head_requests:
+                    self._head_requests.appendleft(request.unique_key)
 
     @override
     async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | None:
@@ -269,7 +270,7 @@ async def mark_request_as_handled(self, request: Request) -> ProcessedRequest |
             request.handled_at = datetime.now(tz=timezone.utc)
             self._metadata.handled_request_count += 1
 
-        if cached_request := self._requests_cache[request.unique_key]:
+        if cached_request := self._requests_cache.get(request.unique_key):
             cached_request.handled_at = request.handled_at
 
         try:

tests/integration/test_crawlers_with_storages.py

@@ -41,6 +41,35 @@ async def default_handler(context: ParselCrawlingContext) -> None:
     assert run_result.status == 'SUCCEEDED'
 
 
+async def test_actor_on_platform_max_crawl_depth(
+) -> None:
+    """Test that the actor respects max_crawl_depth."""
+
+    """The crawler entry point."""
+    import re
+
+    from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+    from apify import Actor
+
+    async with Actor:
+        rq= await Actor.open_request_queue(force_cloud=True)
+        crawler = ParselCrawler(max_crawl_depth=2, request_manager=rq)
+        finished = []
+        enqueue_pattern = re.compile(r'http://localhost:8080/2+$')
+
+        @crawler.router.default_handler
+        async def default_handler(context: ParselCrawlingContext) -> None:
+            """Default request handler."""
+            context.log.info(f'Processing {context.request.url} ...')
+            await context.enqueue_links(include=[enqueue_pattern])
+            finished.append(context.request.url)
+
+        await crawler.run(['http://localhost:8080/'])
+        assert finished == ['http://localhost:8080/', 'http://localhost:8080/2', 'http://localhost:8080/22']
+
+
+
 async def test_actor_on_platform_max_requests_per_crawl(
     make_actor: MakeActorFunction,
     run_actor: RunActorFunction,