Add max retries test

Author: Pijukatel

tests/integration/test_crawlers_with_storages.py

@@ -1,3 +1,4 @@
+from crawlee._types import BasicCrawlingContext
 from tests.integration.conftest import MakeActorFunction, RunActorFunction
 
 
@@ -25,14 +26,12 @@ async def default_handler(context: ParselCrawlingContext) -> None:
                 """Default request handler."""
                 context.log.info(f'Processing {context.request.url} ...')
                 await context.enqueue_links(include=[enqueue_pattern])
-                await context.push_data({'Url': context.request.url})
                 finished.append(context.request.url)
 
             await crawler.run(['http://localhost:8080/'])
             assert finished == ['http://localhost:8080/', 'http://localhost:8080/2', 'http://localhost:8080/22']
-            # assert some dataset
 
-    actor = await make_actor(label='parsel-crawler', main_func=main)
+    actor = await make_actor(label='crawler-max-depth', main_func=main)
     run_result = await run_actor(actor)
 
     assert run_result.status == 'SUCCEEDED'
@@ -62,14 +61,48 @@ async def default_handler(context: ParselCrawlingContext) -> None:
                 """Default request handler."""
                 context.log.info(f'Processing {context.request.url} ...')
                 await context.enqueue_links()
-                await context.push_data({'Url': context.request.url})
                 finished.append(context.request.url)
 
             await crawler.run(['http://localhost:8080/'])
             assert len(finished) == 3
-            # assert some dataset
 
-    actor = await make_actor(label='parsel-crawler', main_func=main)
+    actor = await make_actor(label='crawler-max-requests', main_func=main)
+    run_result = await run_actor(actor)
+
+    assert run_result.status == 'SUCCEEDED'
+
+
+async def test_actor_on_platform_max_request_retries(
+    make_actor: MakeActorFunction,
+    run_actor: RunActorFunction,
+) -> None:
+    """Test that the actor respects max_requests_per_crawl."""
+
+    async def main() -> None:
+        """The crawler entry point."""
+        from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
+
+        from apify import Actor
+
+        async with Actor:
+            max_retries = 2
+            crawler = ParselCrawler(max_request_retries=max_retries)
+            finished = []
+            failed = []
+
+            @crawler.failed_request_handler
+            async def failed_handler(context: BasicCrawlingContext, _: Exception) -> None:
+                failed.add(context.request.url)
+
+            @crawler.router.default_handler
+            async def default_handler(context: ParselCrawlingContext) -> None:
+                finished.append(context.request.url)
+
+            await crawler.run(['http://localhost:8080/non-existing-url'])
+            assert len(finished) == 0
+            assert len(failed) == max_retries + 1
+
+    actor = await make_actor(label='crawler-max-retries', main_func=main)
     run_result = await run_actor(actor)
 
     assert run_result.status == 'SUCCEEDED'