Rebase & fix tests

Author: vdusek

src/crawlee/basic_crawler/_basic_crawler.py

@@ -478,12 +478,14 @@ def sigint_handler() -> None:
         return final_statistics
 
     async def _run_crawler(self) -> None:
+        event_manager = service_container.get_event_manager()
+
         # Collect the context managers to be entered. Context managers that are already active are excluded,
         # as they were likely entered by the caller, who will also be responsible for exiting them.
         contexts_to_enter = [
             cm
             for cm in (
-                self._event_manager,
+                event_manager,
                 self._snapshotter,
                 self._statistics,
                 self._session_pool if self._use_session_pool else None,

src/crawlee/errors.py

@@ -14,7 +14,6 @@
     'HttpStatusCodeError',
     'ProxyError',
     'RequestHandlerError',
-    'ServiceConflictError',
     'SessionError',
     'UserDefinedErrorHandlerError',
 ]

src/crawlee/events/_event_manager.py

@@ -189,9 +189,6 @@ def emit(self, *, event: Event, event_data: EventData) -> None:
             event: The event which will be emitted.
             event_data: The data which will be passed to the event listeners.
         """
-        if not self._initialized:
-            raise RuntimeError('EventManager is not initialized. Please use it within async context manager.')
-
         self._event_emitter.emit(event.value, event_data)
 
     @ensure_context
@@ -202,8 +199,6 @@ async def wait_for_all_listeners_to_complete(self, *, timeout: timedelta | None
             timeout: The maximum time to wait for the event listeners to finish. If they do not complete within
                 the specified timeout, they will be canceled.
         """
-        if not self._initialized:
-            raise RuntimeError('EventManager is not initialized. Please use it within async context manager.')
 
         async def wait_for_listeners() -> None:
             """Gathers all listener tasks and awaits their completion, logging any exceptions encountered."""

src/crawlee/service_container.py

@@ -12,14 +12,11 @@
 
 __all__ = [
     'get_configuration',
-    'get_configuration_if_set',
     'get_event_manager',
     'get_storage_client',
-    'set_cloud_storage_client',
     'set_configuration',
-    'set_default_storage_client_type',
     'set_event_manager',
-    'set_local_storage_client',
+    'set_storage_client',
 ]
 
 

src/crawlee/sessions/_session_pool.py

@@ -6,6 +6,7 @@
 from logging import getLogger
 from typing import TYPE_CHECKING, Callable, Literal, overload
 
+from crawlee import service_container
 from crawlee._utils.context import ensure_context
 from crawlee._utils.docs import docs_group
 from crawlee.events._types import Event, EventPersistStateData

src/crawlee/statistics/_statistics.py

@@ -8,7 +8,7 @@
 
 from typing_extensions import Self, TypeVar
 
-import crawlee.service_container
+from crawlee import service_container
 from crawlee._utils.context import ensure_context
 from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
@@ -131,7 +131,7 @@ async def __aenter__(self) -> Self:
             self._key_value_store = await KeyValueStore.open(name=self._persist_state_kvs_name)
 
         await self._maybe_load_statistics()
-        self._events.on(event=Event.PERSIST_STATE, listener=self._persist_state)
+        self._event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_state)
         self._periodic_logger.start()
 
         return self

src/crawlee/storages/_key_value_store.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from collections.abc import AsyncIterator
 from typing import TYPE_CHECKING, Any, TypeVar, overload
 
 from typing_extensions import override
@@ -12,8 +13,6 @@
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
 
-    from crawlee.base_storage_client import BaseStorageClient
-
 T = TypeVar('T')
 
 

tests/unit/_autoscaling/test_snapshotter.py

@@ -33,9 +33,12 @@ def event_system_data_info() -> EventSystemInfoData:
 
 
 async def test_start_stop_lifecycle() -> None:
-    async with LocalEventManager() as event_manager, Snapshotter(
-        event_manager=event_manager,
-        available_memory_ratio=0.25,
+    async with (
+        LocalEventManager() as event_manager,
+        Snapshotter(
+            event_manager=event_manager,
+            available_memory_ratio=0.25,
+        ),
     ):
         pass
 
@@ -98,7 +101,7 @@ async def test_get_cpu_sample(snapshotter: Snapshotter) -> None:
 
 async def test_methods_raise_error_when_not_active() -> None:
     event_manager = AsyncMock(spec=EventManager)
-    snapshotter = Snapshotter(event_manager, available_memory_ratio=0.25)
+    snapshotter = Snapshotter(event_manager=event_manager, available_memory_ratio=0.25)
 
     assert snapshotter.active is False
 

tests/unit/_autoscaling/test_system_status.py

@@ -25,7 +25,7 @@
 async def snapshotter() -> AsyncGenerator[Snapshotter, None]:
     async with (
         LocalEventManager() as event_manager,
-        Snapshotter(event_manager, available_memory_ratio=0.25) as snapshotter,
+        Snapshotter(event_manager=event_manager, available_memory_ratio=0.25) as snapshotter,
     ):
         yield snapshotter
 
@@ -38,7 +38,7 @@ def now() -> datetime:
 async def test_start_stop_lifecycle() -> None:
     async with (
         LocalEventManager() as event_manager,
-        Snapshotter(event_manager, available_memory_ratio=0.25) as snapshotter,
+        Snapshotter(event_manager=event_manager, available_memory_ratio=0.25) as snapshotter,
     ):
         system_status = SystemStatus(snapshotter)
         system_status.get_current_system_info()

tests/unit/_memory_storage_client/test_memory_storage_client.py

@@ -8,7 +8,7 @@
 
 import pytest
 
-from crawlee import Request
+from crawlee import Request, service_container
 from crawlee._consts import METADATA_FILENAME
 from crawlee.configuration import Configuration
 from crawlee.memory_storage_client import MemoryStorageClient
@@ -224,13 +224,15 @@ async def test_not_implemented_method(tmp_path: Path) -> None:
 async def test_default_storage_path_used(monkeypatch: pytest.MonkeyPatch) -> None:
     # We expect the default value to be used
     monkeypatch.delenv('CRAWLEE_STORAGE_DIR', raising=False)
+    service_container.set_configuration(Configuration())
     ms = MemoryStorageClient()
     assert ms.storage_dir == './storage'
 
 
 async def test_storage_path_from_env_var_overrides_default(monkeypatch: pytest.MonkeyPatch) -> None:
     # We expect the env var to override the default value
     monkeypatch.setenv('CRAWLEE_STORAGE_DIR', './env_var_storage_dir')
+    service_container.set_configuration(Configuration())
     ms = MemoryStorageClient()
     assert ms.storage_dir == './env_var_storage_dir'