Skip to content

Commit 3a28d42

Browse files
committed
Start _crawler_state_rec_task when active contexts (to allow persistent state init)
1 parent f3b9812 commit 3a28d42

File tree

1 file changed

+2
-3
lines changed

1 file changed

+2
-3
lines changed

src/crawlee/crawlers/_basic/_basic_crawler.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -440,7 +440,7 @@ def __init__(
440440
self._statistics = statistics or cast(
441441
'Statistics[TStatisticsState]',
442442
Statistics.with_default_state(
443-
persistence_enabled=True, # TODO: Why does changing this to True breaks unrelated tests?
443+
persistence_enabled=True,
444444
periodic_message_logger=self._logger,
445445
statistics_log_format=self._statistics_log_format,
446446
log_message='Current request statistics:',
@@ -722,8 +722,6 @@ def sigint_handler() -> None:
722722
async def _run_crawler(self) -> None:
723723
event_manager = self._service_locator.get_event_manager()
724724

725-
self._crawler_state_rec_task.start()
726-
727725
# Collect the context managers to be entered. Context managers that are already active are excluded,
728726
# as they were likely entered by the caller, who will also be responsible for exiting them.
729727
contexts_to_enter = [
@@ -743,6 +741,7 @@ async def _run_crawler(self) -> None:
743741
for context in contexts_to_enter:
744742
await exit_stack.enter_async_context(context) # type: ignore[arg-type]
745743

744+
self._crawler_state_rec_task.start()
746745
await self._autoscaled_pool.run()
747746

748747
# Emit PERSIST_STATE event when crawler is finishing to allow listeners to persist their state if needed

0 commit comments

Comments
 (0)