|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +from typing import TYPE_CHECKING |
| 4 | + |
| 5 | +if TYPE_CHECKING: |
| 6 | + from .conftest import MakeActorFunction, RunActorFunction |
| 7 | + |
| 8 | + |
| 9 | +async def test_migration_through_reboot(make_actor: MakeActorFunction, run_actor: RunActorFunction) -> None: |
| 10 | + """Test that actor works as expected after migration through testing behavior after reboot. |
| 11 | +
|
| 12 | + Handle two requests. Migrate in between the two requests.""" |
| 13 | + |
| 14 | + async def main() -> None: |
| 15 | + from crawlee._types import BasicCrawlingContext, ConcurrencySettings |
| 16 | + from crawlee.crawlers import BasicCrawler |
| 17 | + |
| 18 | + from apify import Actor |
| 19 | + |
| 20 | + async with Actor: |
| 21 | + crawler = BasicCrawler(concurrency_settings=ConcurrencySettings(max_concurrency=1)) |
| 22 | + requests = ['https://example.com/1', 'https://example.com/2'] |
| 23 | + |
| 24 | + run = await Actor.apify_client.run(Actor.config.actor_run_id or '').get() |
| 25 | + assert run |
| 26 | + first_run = run.get('stats', {}).get('rebootCount', 0) == 0 |
| 27 | + Actor.log.warning(run) |
| 28 | + |
| 29 | + @crawler.router.default_handler |
| 30 | + async def default_handler(context: BasicCrawlingContext) -> None: |
| 31 | + context.log.info(f'Processing {context.request.url} ...') |
| 32 | + |
| 33 | + # Simulate migration through reboot |
| 34 | + if context.request.url == requests[1] and first_run: |
| 35 | + context.log.info(f'Reclaiming {context.request.url} ...') |
| 36 | + rq = await crawler.get_request_manager() |
| 37 | + await rq.reclaim_request(context.request) |
| 38 | + await Actor.reboot() |
| 39 | + |
| 40 | + await crawler.run(requests) |
| 41 | + |
| 42 | + # Each time one request is finished. |
| 43 | + assert crawler.statistics.state.requests_finished == 1 |
| 44 | + |
| 45 | + actor = await make_actor(label='migration', main_func=main) |
| 46 | + run_result = await run_actor(actor) |
| 47 | + |
| 48 | + assert run_result.status == 'SUCCEEDED' |
0 commit comments