better service locator

Author: vdusek

src/crawlee/__init__.py

@@ -1,9 +1,10 @@
 from importlib import metadata
 
 from ._request import Request
+from ._service_locator import service_locator
 from ._types import ConcurrencySettings, EnqueueStrategy, HttpHeaders
 from ._utils.globs import Glob
 
 __version__ = metadata.version('crawlee')
 
-__all__ = ['ConcurrencySettings', 'EnqueueStrategy', 'Glob', 'HttpHeaders', 'Request']
+__all__ = ['ConcurrencySettings', 'EnqueueStrategy', 'Glob', 'HttpHeaders', 'Request', 'service_locator']

src/crawlee/_service_locator.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from crawlee._utils.docs import docs_group
+from crawlee.base_storage_client._base_storage_client import BaseStorageClient
+from crawlee.configuration import Configuration
+from crawlee.errors import ServiceConflictError
+from crawlee.events._event_manager import EventManager
+
+
+@docs_group('Classes')
+class ServiceLocator:
+    """Service locator for managing the services used by Crawlee.
+
+    All services are initialized to its default value lazily.
+    """
+
+    def __init__(self) -> None:
+        self._configuration: Configuration | None = None
+        self._event_manager: EventManager | None = None
+        self._storage_client: BaseStorageClient | None = None
+
+        # Flags to check if the services were already set.
+        self._configuration_was_set = False
+        self._event_manager_was_set = False
+        self._storage_client_was_set = False
+
+    def get_configuration(self) -> Configuration:
+        """Get the configuration."""
+        if self._configuration is None:
+            self._configuration = Configuration()
+
+        return self._configuration
+
+    def set_configuration(self, configuration: Configuration) -> None:
+        """Set the configuration.
+
+        Args:
+            configuration: The configuration to set.
+
+        Raises:
+            ServiceConflictError: If the configuration was already set.
+        """
+        if self._configuration_was_set:
+            raise ServiceConflictError(Configuration, configuration, self._configuration)
+
+        self._configuration = configuration
+        self._configuration_was_set = True
+
+    def get_event_manager(self) -> EventManager:
+        """Get the event manager."""
+        if self._event_manager is None:
+            from crawlee.events import LocalEventManager
+
+            self._event_manager = LocalEventManager()
+
+        return self._event_manager
+
+    def set_event_manager(self, event_manager: EventManager) -> None:
+        """Set the event manager.
+
+        Args:
+            event_manager: The event manager to set.
+
+        Raises:
+            ServiceConflictError: If the event manager was already set.
+        """
+        if self._event_manager_was_set:
+            raise ServiceConflictError(EventManager, event_manager, self._event_manager)
+
+        self._event_manager = event_manager
+        self._event_manager_was_set = True
+
+    def get_storage_client(self) -> BaseStorageClient:
+        """Get the storage client."""
+        if self._storage_client is None:
+            from crawlee.memory_storage_client import MemoryStorageClient
+
+            self._storage_client = MemoryStorageClient()
+
+        return self._storage_client
+
+    def set_storage_client(self, storage_client: BaseStorageClient) -> None:
+        """Set the storage client.
+
+        Args:
+            storage_client: The storage client to set.
+
+        Raises:
+            ServiceConflictError: If the storage client was already set.
+        """
+        if self._storage_client_was_set:
+            raise ServiceConflictError(BaseStorageClient, storage_client, self._storage_client)
+
+        self._storage_client = storage_client
+        self._storage_client_was_set = True
+
+
+service_locator = ServiceLocator()

src/crawlee/basic_crawler/_basic_crawler.py

@@ -522,7 +522,7 @@ async def _run_crawler(self) -> None:
 
         async with AsyncExitStack() as exit_stack:
             for context in contexts_to_enter:
-                await exit_stack.enter_async_context(context)
+                await exit_stack.enter_async_context(context)  # type: ignore[arg-type]
 
             await self._autoscaled_pool.run()
 

src/crawlee/configuration.py

@@ -235,11 +235,12 @@ class Configuration(BaseSettings):
     def get_global_configuration(cls) -> Self:
         """Retrieve the global instance of the configuration.
 
-        TODO: Can we remove this?
+        Mostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`
+        instead.
         """
-        from crawlee.service_locator import get_configuration
+        from crawlee import service_locator
 
-        config = get_configuration()
+        config = service_locator.get_configuration()
 
         if not isinstance(config, cls):
             raise TypeError(f'Requested global configuration object of type {cls}, but {config.__class__} was found')

src/crawlee/service_locator.py

@@ -223,8 +223,8 @@ async def test_not_implemented_method(tmp_path: Path) -> None:
 
 async def test_default_storage_path_used(monkeypatch: pytest.MonkeyPatch) -> None:
     # Reset the configuration in service locator
-    service_locator._service_locator._configuration = None
-    service_locator._service_locator._configuration_was_set = False
+    service_locator._configuration = None
+    service_locator._configuration_was_set = False
 
     # Remove the env var for setting the storage directory
     monkeypatch.delenv('CRAWLEE_STORAGE_DIR', raising=False)

tests/unit/_memory_storage_client/test_memory_storage_client.py

@@ -41,14 +41,14 @@ def _prepare_test_env() -> None:
 
         # Reset the flags in the service locator to indicate that no services are explicitly set. This ensures
         # a clean state, as services might have been set during a previous test and not reset properly.
-        service_locator._service_locator._configuration_was_set = False
-        service_locator._service_locator._storage_client_was_set = False
-        service_locator._service_locator._event_manager_was_set = False
+        service_locator._configuration_was_set = False
+        service_locator._storage_client_was_set = False
+        service_locator._event_manager_was_set = False
 
         # Reset the services in the service locator.
-        service_locator._service_locator._configuration = None
-        service_locator._service_locator._event_manager = None
-        service_locator._service_locator._storage_client = None
+        service_locator._configuration = None
+        service_locator._event_manager = None
+        service_locator._storage_client = None
 
         # Clear creation-related caches to ensure no state is carried over between tests.
         monkeypatch.setattr(_creation_management, '_cache_dataset_by_id', {})
@@ -60,9 +60,9 @@ def _prepare_test_env() -> None:
 
         # Verify that the test environment was set up correctly.
         assert os.environ.get('CRAWLEE_STORAGE_DIR') == str(tmp_path)
-        assert service_locator._service_locator.configuration_was_set is False
-        assert service_locator._service_locator.storage_client_was_set is False
-        assert service_locator._service_locator.event_manager_was_set is False
+        assert service_locator._configuration_was_set is False
+        assert service_locator._storage_client_was_set is False
+        assert service_locator._event_manager_was_set is False
 
     return _prepare_test_env
 

tests/unit/conftest.py

@@ -5,10 +5,10 @@
 
 from typing import TYPE_CHECKING
 
+from crawlee import service_locator
 from crawlee.configuration import Configuration
 from crawlee.http_crawler import HttpCrawler, HttpCrawlingContext
 from crawlee.memory_storage_client._memory_storage_client import MemoryStorageClient
-from crawlee.service_locator import get_configuration, set_storage_client
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -20,8 +20,8 @@ def test_global_configuration_works() -> None:
     assert (
         Configuration.get_global_configuration()
         is Configuration.get_global_configuration()
-        is get_configuration()
-        is get_configuration()
+        is service_locator.get_configuration()
+        is service_locator.get_configuration()
     )
 
 
@@ -32,7 +32,7 @@ async def test_storage_not_persisted_when_disabled(tmp_path: Path, httpbin: URL)
         crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
     )
     storage_client = MemoryStorageClient(config)
-    set_storage_client(storage_client)
+    service_locator.set_storage_client(storage_client)
 
     crawler = HttpCrawler()
 
@@ -54,7 +54,7 @@ async def test_storage_persisted_when_enabled(tmp_path: Path, httpbin: URL) -> N
         crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
     )
     storage_client = MemoryStorageClient(config)
-    set_storage_client(storage_client)
+    service_locator.set_storage_client(storage_client)
 
     crawler = HttpCrawler()