Add ServiceConflictError

Author: vdusek

src/crawlee/basic_crawler/_basic_crawler.py

@@ -185,9 +185,8 @@ def __init__(
         concurrency_settings: ConcurrencySettings | None = None,
         request_handler_timeout: timedelta = timedelta(minutes=1),
         statistics: Statistics | None = None,
-        configure_logging: bool = True,
-        max_crawl_depth: int | None = None,
         abort_on_error: bool = False,
+        configure_logging: bool = True,
         _context_pipeline: ContextPipeline[TCrawlingContext] | None = None,
         _additional_context_managers: Sequence[AbstractAsyncContextManager] | None = None,
         _logger: logging.Logger | None = None,
@@ -215,9 +214,8 @@ def __init__(
             concurrency_settings: Settings to fine-tune concurrency levels.
             request_handler_timeout: Maximum duration allowed for a single request handler to run.
             statistics: A custom `Statistics` instance, allowing the use of non-default configuration.
-            configure_logging: If True, the crawler will set up logging infrastructure automatically.
-            max_crawl_depth: Maximum crawl depth. If set, the crawler will stop crawling after reaching this depth.
             abort_on_error: If True, the crawler stops immediately when any request handler error occurs.
+            configure_logging: If True, the crawler will set up logging infrastructure automatically.
             _context_pipeline: Enables extending the request lifecycle and modifying the crawling context.
                 Intended for use by subclasses rather than direct instantiation of `BasicCrawler`.
             _additional_context_managers: Additional context managers used throughout the crawler lifecycle.
@@ -247,6 +245,7 @@ def __init__(
         # Error & failed request handlers
         self._error_handler: ErrorHandler[TCrawlingContext | BasicCrawlingContext] | None = None
         self._failed_request_handler: FailedRequestHandler[TCrawlingContext | BasicCrawlingContext] | None = None
+        self._abort_on_error = abort_on_error
 
         # Context pipeline
         self._context_pipeline = (_context_pipeline or ContextPipeline()).compose(self._check_url_after_redirects)
@@ -305,7 +304,6 @@ def __init__(
         self._has_finished_before = False
 
         self._failed = False
-        self._abort_on_error = abort_on_error
 
     @property
     def log(self) -> logging.Logger:

src/crawlee/errors.py

@@ -14,6 +14,7 @@
     'HttpStatusCodeError',
     'ProxyError',
     'RequestHandlerError',
+    'ServiceConflictError',
     'SessionError',
     'UserDefinedErrorHandlerError',
 ]
@@ -34,6 +35,17 @@ class SessionError(Exception):
     """
 
 
+@docs_group('Errors')
+class ServiceConflictError(Exception):
+    """Raised when attempting to reassign a service in service container that was already configured."""
+
+    def __init__(self, service: type, new_value: object, existing_value: object) -> None:
+        super().__init__(
+            f'Service {service.__name__} has already been set. Existing value: {existing_value}, '
+            f'attempted new value: {new_value}.'
+        )
+
+
 @docs_group('Errors')
 class ProxyError(SessionError):
     """Raised when a proxy is being blocked or malfunctions."""

src/crawlee/service_container.py

@@ -1,14 +1,10 @@
 from __future__ import annotations
 
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
-
 from crawlee._utils.docs import docs_group
-
-if TYPE_CHECKING:
-    from crawlee.base_storage_client._base_storage_client import BaseStorageClient
-    from crawlee.configuration import Configuration
-    from crawlee.events._event_manager import EventManager
+from crawlee.base_storage_client._base_storage_client import BaseStorageClient
+from crawlee.configuration import Configuration
+from crawlee.errors import ServiceConflictError
+from crawlee.events._event_manager import EventManager
 
 __all__ = [
     'get_configuration',
@@ -20,29 +16,33 @@
 ]
 
 
-@dataclass
 class _ServiceLocator:
     """Service locator for managing the services used by Crawlee.
 
     All services are initialized to its default value lazily.
     """
 
-    _configuration: Configuration | None = field(default=None, init=False)
-    _event_manager: EventManager | None = field(default=None, init=False)
-    _storage_client: BaseStorageClient | None = field(default=None, init=False)
+    def __init__(self) -> None:
+        self._configuration: Configuration | None = None
+        self._event_manager: EventManager | None = None
+        self._storage_client: BaseStorageClient | None = None
+
+        # Flags to check if the services were already set.
+        self._configuration_was_set = False
+        self._event_manager_was_set = False
+        self._storage_client_was_set = False
 
     @property
     def configuration(self) -> Configuration:
         if self._configuration is None:
-            from crawlee.configuration import Configuration
-
             self._configuration = Configuration()
 
         return self._configuration
 
     @configuration.setter
     def configuration(self, value: Configuration) -> None:
         self._configuration = value
+        self._configuration_was_set = True
 
     @property
     def storage_client(self) -> BaseStorageClient:
@@ -56,6 +56,7 @@ def storage_client(self) -> BaseStorageClient:
     @storage_client.setter
     def storage_client(self, value: BaseStorageClient) -> None:
         self._storage_client = value
+        self._storage_client_was_set = True
 
     @property
     def event_manager(self) -> EventManager:
@@ -69,6 +70,19 @@ def event_manager(self) -> EventManager:
     @event_manager.setter
     def event_manager(self, value: EventManager) -> None:
         self._event_manager = value
+        self._event_manager_was_set = True
+
+    @property
+    def configuration_was_set(self) -> bool:
+        return self._configuration_was_set
+
+    @property
+    def event_manager_was_set(self) -> bool:
+        return self._event_manager_was_set
+
+    @property
+    def storage_client_was_set(self) -> bool:
+        return self._storage_client_was_set
 
 
 _service_locator = _ServiceLocator()
@@ -81,8 +95,23 @@ def get_configuration() -> Configuration:
 
 
 @docs_group('Functions')
-def set_configuration(configuration: Configuration) -> None:
-    """Set the configuration."""
+def set_configuration(
+    configuration: Configuration,
+    *,
+    force: bool = False,
+) -> None:
+    """Set the configuration.
+
+    Args:
+        configuration: The configuration to set.
+        force: If True, the configuration will be set even if it was already set.
+
+    Raises:
+        ServiceConflictError: If the configuration was already set.
+    """
+    if _service_locator.configuration_was_set and not force:
+        raise ServiceConflictError(Configuration, configuration, _service_locator.configuration)
+
     _service_locator.configuration = configuration
 
 
@@ -93,8 +122,23 @@ def get_event_manager() -> EventManager:
 
 
 @docs_group('Functions')
-def set_event_manager(event_manager: EventManager) -> None:
-    """Set the event manager."""
+def set_event_manager(
+    event_manager: EventManager,
+    *,
+    force: bool = False,
+) -> None:
+    """Set the event manager.
+
+    Args:
+        event_manager: The event manager to set.
+        force: If True, the event manager will be set even if it was already set.
+
+    Raises:
+        ServiceConflictError: If the event manager was already set.
+    """
+    if _service_locator.event_manager_was_set and not force:
+        raise ServiceConflictError(EventManager, event_manager, _service_locator.event_manager)
+
     _service_locator.event_manager = event_manager
 
 
@@ -105,6 +149,21 @@ def get_storage_client() -> BaseStorageClient:
 
 
 @docs_group('Functions')
-def set_storage_client(storage_client: BaseStorageClient) -> None:
-    """Set the storage client."""
+def set_storage_client(
+    storage_client: BaseStorageClient,
+    *,
+    force: bool = False,
+) -> None:
+    """Set the storage client.
+
+    Args:
+        storage_client: The storage client to set.
+        force: If True, the storage client will be set even if it was already set.
+
+    Raises:
+        ServiceConflictError: If the storage client was already set.
+    """
+    if _service_locator.storage_client_was_set and not force:
+        raise ServiceConflictError(BaseStorageClient, storage_client, _service_locator.storage_client)
+
     _service_locator.storage_client = storage_client

tests/unit/_memory_storage_client/test_memory_storage_client.py

@@ -224,15 +224,15 @@ async def test_not_implemented_method(tmp_path: Path) -> None:
 async def test_default_storage_path_used(monkeypatch: pytest.MonkeyPatch) -> None:
     # We expect the default value to be used
     monkeypatch.delenv('CRAWLEE_STORAGE_DIR', raising=False)
-    service_container.set_configuration(Configuration())
+    service_container.set_configuration(Configuration(), force=True)
     ms = MemoryStorageClient()
     assert ms.storage_dir == './storage'
 
 
 async def test_storage_path_from_env_var_overrides_default(monkeypatch: pytest.MonkeyPatch) -> None:
     # We expect the env var to override the default value
     monkeypatch.setenv('CRAWLEE_STORAGE_DIR', './env_var_storage_dir')
-    service_container.set_configuration(Configuration())
+    service_container.set_configuration(Configuration(), force=True)
     ms = MemoryStorageClient()
     assert ms.storage_dir == './env_var_storage_dir'
 

tests/unit/basic_crawler/test_basic_crawler.py

@@ -14,7 +14,7 @@
 import httpx
 import pytest
 
-from crawlee import ConcurrencySettings, EnqueueStrategy, Glob
+from crawlee import ConcurrencySettings, EnqueueStrategy, Glob, service_container
 from crawlee._request import BaseRequestData, Request
 from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs, HttpHeaders
 from crawlee.basic_crawler import BasicCrawler
@@ -797,14 +797,15 @@ async def handler(context: BasicCrawlingContext) -> None:
 
 
 async def test_respects_no_persist_storage() -> None:
-    configuration = Configuration(persist_storage=False)
-    crawler = BasicCrawler(configuration=configuration)
+    config = Configuration(persist_storage=False)
+    service_container.set_configuration(config, force=True)
+    crawler = BasicCrawler()
 
     @crawler.router.default_handler
     async def handler(context: BasicCrawlingContext) -> None:
         await context.push_data({'something': 'something'})
 
-    datasets_path = Path(configuration.storage_dir) / 'datasets' / 'default'
+    datasets_path = Path(config.storage_dir) / 'datasets' / 'default'
     assert not datasets_path.exists() or list(datasets_path.iterdir()) == []
 
 

tests/unit/conftest.py

@@ -11,6 +11,7 @@
 from yarl import URL
 
 from crawlee import service_container
+from crawlee.configuration import Configuration
 from crawlee.events._local_event_manager import LocalEventManager
 from crawlee.memory_storage_client import MemoryStorageClient
 from crawlee.proxy_configuration import ProxyInfo
@@ -28,9 +29,9 @@ def reset() -> None:
         monkeypatch.setenv('CRAWLEE_STORAGE_DIR', str(tmp_path))
 
         # Reset services in crawlee.service_container
-        service_container.set_configuration(Configuration())
-        service_container.set_storage_client(MemoryStorageClient())
-        service_container.set_event_manager(LocalEventManager())
+        service_container.set_configuration(Configuration(), force=True)
+        service_container.set_storage_client(MemoryStorageClient(), force=True)
+        service_container.set_event_manager(LocalEventManager(), force=True)
 
         # Clear creation-related caches to ensure no state is carried over between tests
         monkeypatch.setattr(_creation_management, '_cache_dataset_by_id', {})

tests/unit/test_configuration.py

@@ -24,12 +24,13 @@ def test_global_configuration_works() -> None:
 
 
 async def test_storage_not_persisted_when_disabled(tmp_path: Path) -> None:
-    configuration = Configuration(
+    config = Configuration(
         persist_storage=False,
         write_metadata=False,
         crawlee_storage_dir=str(tmp_path),  # type: ignore
     )
-    set_storage_client(MemoryStorageClient(configuration=configuration))
+    storage_client = MemoryStorageClient(config)
+    set_storage_client(storage_client, force=True)
 
     crawler = HttpCrawler()
 
@@ -45,12 +46,13 @@ async def default_handler(context: HttpCrawlingContext) -> None:
 
 
 async def test_storage_persisted_when_enabled(tmp_path: Path) -> None:
-    configuration = Configuration(
+    config = Configuration(
         persist_storage=True,
         write_metadata=True,
         crawlee_storage_dir=str(tmp_path),  # type: ignore
     )
-    set_storage_client(MemoryStorageClient(configuration=configuration))
+    storage_client = MemoryStorageClient(config)
+    set_storage_client(storage_client, force=True)
 
     crawler = HttpCrawler()
 

tests/unit/test_service_container.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import pytest
+
+from crawlee.configuration import Configuration
+from crawlee.errors import ServiceConflictError
+from crawlee.events import LocalEventManager
+from crawlee.memory_storage_client import MemoryStorageClient
+from crawlee.service_container import (
+    get_configuration,
+    get_event_manager,
+    get_storage_client,
+    set_configuration,
+    set_event_manager,
+    set_storage_client,
+)
+
+
+def test_configuration() -> None:
+    default_config = Configuration()
+    config = get_configuration()
+    assert config == default_config
+
+    custom_config = Configuration(default_browser_path='custom_path')
+
+    with pytest.raises(ServiceConflictError, match='Configuration has already been set.'):
+        set_configuration(custom_config)
+
+    set_configuration(custom_config, force=True)
+    config = get_configuration()
+    assert config == custom_config
+
+
+def test_event_manager() -> None:
+    default_event_manager = get_event_manager()
+    assert isinstance(default_event_manager, LocalEventManager)
+
+    custom_event_manager = LocalEventManager()
+
+    with pytest.raises(ServiceConflictError, match='EventManager has already been set.'):
+        set_event_manager(custom_event_manager)
+
+    set_event_manager(custom_event_manager, force=True)
+    event_manager = get_event_manager()
+    assert event_manager == custom_event_manager
+
+
+def test_storage_client() -> None:
+    default_storage_client = get_storage_client()
+    assert isinstance(default_storage_client, MemoryStorageClient)
+
+    custom_storage_client = MemoryStorageClient()
+
+    with pytest.raises(ServiceConflictError, match='StorageClient has already been set.'):
+        set_storage_client(custom_storage_client)
+
+    set_storage_client(custom_storage_client, force=True)
+    storage_client = get_storage_client()
+    assert storage_client == custom_storage_client