Skip to content

Commit b256876

Browse files
committed
Add migration test
1 parent 19b79f1 commit b256876

File tree

1 file changed

+48
-0
lines changed

1 file changed

+48
-0
lines changed
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
from __future__ import annotations
2+
3+
from typing import TYPE_CHECKING
4+
5+
if TYPE_CHECKING:
6+
from .conftest import MakeActorFunction, RunActorFunction
7+
8+
9+
async def test_migration_through_reboot(make_actor: MakeActorFunction, run_actor: RunActorFunction) -> None:
10+
"""Test that actor works as expected after migration through testing behavior after reboot.
11+
12+
Handle two requests. Migrate in between the two requests."""
13+
14+
async def main() -> None:
15+
from crawlee._types import BasicCrawlingContext, ConcurrencySettings
16+
from crawlee.crawlers import BasicCrawler
17+
18+
from apify import Actor
19+
20+
async with Actor:
21+
crawler = BasicCrawler(concurrency_settings=ConcurrencySettings(max_concurrency=1))
22+
requests = ['https://example.com/1', 'https://example.com/2']
23+
24+
run = await Actor.apify_client.run(Actor.config.actor_run_id or '').get()
25+
assert run
26+
first_run = run.get('stats', {}).get('rebootCount', 0) == 0
27+
Actor.log.warning(run)
28+
29+
@crawler.router.default_handler
30+
async def default_handler(context: BasicCrawlingContext) -> None:
31+
context.log.info(f'Processing {context.request.url} ...')
32+
33+
# Simulate migration through reboot
34+
if context.request.url == requests[1] and first_run:
35+
context.log.info(f'Reclaiming {context.request.url} ...')
36+
rq = await crawler.get_request_manager()
37+
await rq.reclaim_request(context.request)
38+
await Actor.reboot()
39+
40+
await crawler.run(requests)
41+
42+
# Each time one request is finished.
43+
assert crawler.statistics.state.requests_finished == 1
44+
45+
actor = await make_actor(label='migration', main_func=main)
46+
run_result = await run_actor(actor)
47+
48+
assert run_result.status == 'SUCCEEDED'

0 commit comments

Comments
 (0)