service_container -> service_locator

vdusek · vdusek · commit f1a0c5960abe · 2024-12-05T12:07:42.000+01:00
diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py
@@ -10,7 +10,7 @@
 import psutil
 from sortedcontainers import SortedList
 
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee._autoscaling.types import ClientSnapshot, CpuSnapshot, EventLoopSnapshot, MemorySnapshot, Snapshot
 from crawlee._utils.byte_size import ByteSize
 from crawlee._utils.context import ensure_context
@@ -134,7 +134,7 @@ async def __aenter__(self) -> Snapshotter:
             raise RuntimeError(f'The {self.__class__.__name__} is already active.')
 
         self._active = True
-        event_manager = service_container.get_event_manager()
+        event_manager = service_locator.get_event_manager()
         event_manager.on(event=Event.SYSTEM_INFO, listener=self._snapshot_cpu)
         event_manager.on(event=Event.SYSTEM_INFO, listener=self._snapshot_memory)
         self._snapshot_event_loop_task.start()
@@ -158,7 +158,7 @@ async def __aexit__(
         if not self._active:
             raise RuntimeError(f'The {self.__class__.__name__} is not active.')
 
-        event_manager = service_container.get_event_manager()
+        event_manager = service_locator.get_event_manager()
         event_manager.off(event=Event.SYSTEM_INFO, listener=self._snapshot_cpu)
         event_manager.off(event=Event.SYSTEM_INFO, listener=self._snapshot_memory)
         await self._snapshot_event_loop_task.stop()
diff --git a/src/crawlee/_log_config.py b/src/crawlee/_log_config.py
@@ -9,7 +9,7 @@
 from colorama import Fore, Style, just_fix_windows_console
 from typing_extensions import assert_never
 
-from crawlee import service_container
+from crawlee import service_locator
 
 just_fix_windows_console()
 
@@ -35,7 +35,7 @@
 
 
 def get_configured_log_level() -> int:
-    config = service_container.get_configuration()
+    config = service_locator.get_configuration()
 
     verbose_logging_requested = 'verbose_log' in config.model_fields_set and config.verbose_log
 
diff --git a/src/crawlee/basic_crawler/_basic_crawler.py b/src/crawlee/basic_crawler/_basic_crawler.py
@@ -18,7 +18,7 @@
 from tldextract import TLDExtract
 from typing_extensions import NotRequired, TypedDict, TypeVar, Unpack, assert_never
 
-from crawlee import EnqueueStrategy, Glob, service_container
+from crawlee import EnqueueStrategy, Glob, service_locator
 from crawlee._autoscaling import AutoscaledPool
 from crawlee._autoscaling.snapshotter import Snapshotter
 from crawlee._autoscaling.system_status import SystemStatus
@@ -238,13 +238,13 @@ def __init__(
                 Intended for use by subclasses rather than direct instantiation of `BasicCrawler`.
         """
         if configuration:
-            service_container.set_configuration(configuration)
+            service_locator.set_configuration(configuration)
         if storage_client:
-            service_container.set_storage_client(storage_client)
+            service_locator.set_storage_client(storage_client)
         if event_manager:
-            service_container.set_event_manager(event_manager)
+            service_locator.set_event_manager(event_manager)
 
-        config = service_container.get_configuration()
+        config = service_locator.get_configuration()
 
         # Core components
         self._request_provider = request_provider
@@ -504,7 +504,7 @@ def sigint_handler() -> None:
         return final_statistics
 
     async def _run_crawler(self) -> None:
-        event_manager = service_container.get_event_manager()
+        event_manager = service_locator.get_event_manager()
 
         # Collect the context managers to be entered. Context managers that are already active are excluded,
         # as they were likely entered by the caller, who will also be responsible for exiting them.
diff --git a/src/crawlee/configuration.py b/src/crawlee/configuration.py
@@ -237,7 +237,7 @@ def get_global_configuration(cls) -> Self:
 
         TODO: Can we remove this?
         """
-        from crawlee.service_container import get_configuration
+        from crawlee.service_locator import get_configuration
 
         config = get_configuration()
 
diff --git a/src/crawlee/memory_storage_client/_memory_storage_client.py b/src/crawlee/memory_storage_client/_memory_storage_client.py
@@ -10,7 +10,7 @@
 
 from typing_extensions import override
 
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee._utils.docs import docs_group
 from crawlee.base_storage_client import BaseStorageClient
 from crawlee.memory_storage_client._dataset_client import DatasetClient
@@ -80,7 +80,7 @@ def __init__(
             default_key_value_store_id: The default key-value store ID.
             default_dataset_id: The default dataset ID.
         """
-        config = configuration or service_container.get_configuration()
+        config = configuration or service_locator.get_configuration()
 
         # Set the internal attributes.
         self._write_metadata = write_metadata or config.write_metadata
diff --git a/src/crawlee/service_locator.py b/src/crawlee/service_locator.py
diff --git a/src/crawlee/sessions/_session_pool.py b/src/crawlee/sessions/_session_pool.py
@@ -6,7 +6,7 @@
 from logging import getLogger
 from typing import TYPE_CHECKING, Callable, Literal, overload
 
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee._utils.context import ensure_context
 from crawlee._utils.docs import docs_group
 from crawlee.events._types import Event, EventPersistStateData
@@ -55,7 +55,7 @@ def __init__(
             persist_state_key: The key under which the session pool's state is stored in the `KeyValueStore`.
         """
         if event_manager:
-            service_container.set_event_manager(event_manager)
+            service_locator.set_event_manager(event_manager)
 
         self._max_pool_size = max_pool_size
         self._session_settings = create_session_settings or {}
@@ -110,7 +110,7 @@ async def __aenter__(self) -> SessionPool:
         self._active = True
 
         if self._persistence_enabled:
-            event_manager = service_container.get_event_manager()
+            event_manager = service_locator.get_event_manager()
             self._kvs = await KeyValueStore.open(name=self._persist_state_kvs_name)
 
             # Attempt to restore the previously persisted state.
@@ -143,7 +143,7 @@ async def __aexit__(
             raise RuntimeError(f'The {self.__class__.__name__} is not active.')
 
         if self._persistence_enabled:
-            event_manager = service_container.get_event_manager()
+            event_manager = service_locator.get_event_manager()
             # Remove the event listener for state persistence.
             event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_state)
 
diff --git a/src/crawlee/statistics/_statistics.py b/src/crawlee/statistics/_statistics.py
@@ -8,7 +8,7 @@
 
 from typing_extensions import Self, TypeVar
 
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee._utils.context import ensure_context
 from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
@@ -78,7 +78,7 @@ def __init__(
         state_model: type[TStatisticsState] = cast(Any, StatisticsState),  # noqa: B008 - in an ideal world, TStatisticsState would be inferred from this argument, but I haven't managed to do that
     ) -> None:
         if event_manager:
-            service_container.set_event_manager(event_manager)
+            service_locator.set_event_manager(event_manager)
 
         self._id = Statistics.__next_id
         Statistics.__next_id += 1
@@ -132,7 +132,7 @@ async def __aenter__(self) -> Self:
             self._key_value_store = await KeyValueStore.open(name=self._persist_state_kvs_name)
 
         await self._maybe_load_statistics()
-        event_manager = service_container.get_event_manager()
+        event_manager = service_locator.get_event_manager()
         event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_state)
         self._periodic_logger.start()
 
@@ -153,7 +153,7 @@ async def __aexit__(
             raise RuntimeError(f'The {self.__class__.__name__} is not active.')
 
         self.state.crawler_finished_at = datetime.now(timezone.utc)
-        event_manager = service_container.get_event_manager()
+        event_manager = service_locator.get_event_manager()
         event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_state)
         await self._periodic_logger.stop()
         await self._persist_state(event_data=EventPersistStateData(is_migrating=False))
diff --git a/src/crawlee/storages/_dataset.py b/src/crawlee/storages/_dataset.py
@@ -8,7 +8,7 @@
 
 from typing_extensions import NotRequired, Required, Unpack, override
 
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee._utils.byte_size import ByteSize
 from crawlee._utils.docs import docs_group
 from crawlee._utils.file import json_dumps
@@ -223,8 +223,8 @@ async def open(
     ) -> Dataset:
         from crawlee.storages._creation_management import open_storage
 
-        configuration = configuration or service_container.get_configuration()
-        storage_client = storage_client or service_container.get_storage_client()
+        configuration = configuration or service_locator.get_configuration()
+        storage_client = storage_client or service_locator.get_storage_client()
 
         return await open_storage(
             storage_class=cls,
diff --git a/src/crawlee/storages/_key_value_store.py b/src/crawlee/storages/_key_value_store.py
@@ -5,7 +5,7 @@
 
 from typing_extensions import override
 
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee._utils.docs import docs_group
 from crawlee.base_storage_client import BaseStorageClient, KeyValueStoreKeyInfo, KeyValueStoreMetadata
 from crawlee.storages._base_storage import BaseStorage
@@ -86,8 +86,8 @@ async def open(
     ) -> KeyValueStore:
         from crawlee.storages._creation_management import open_storage
 
-        configuration = configuration or service_container.get_configuration()
-        storage_client = storage_client or service_container.get_storage_client()
+        configuration = configuration or service_locator.get_configuration()
+        storage_client = storage_client or service_locator.get_storage_client()
 
         return await open_storage(
             storage_class=cls,
diff --git a/src/crawlee/storages/_request_queue.py b/src/crawlee/storages/_request_queue.py
@@ -9,7 +9,7 @@
 
 from typing_extensions import override
 
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.docs import docs_group
 from crawlee._utils.lru_cache import LRUCache
@@ -105,8 +105,8 @@ class RequestQueue(BaseStorage, RequestProvider):
     """Expected delay for storage to achieve consistency, guiding the timing of subsequent read operations."""
 
     def __init__(self, id: str, name: str | None, storage_client: BaseStorageClient) -> None:
-        config = service_container.get_configuration()
-        event_manager = service_container.get_event_manager()
+        config = service_locator.get_configuration()
+        event_manager = service_locator.get_event_manager()
 
         self._id = id
         self._name = name
@@ -157,8 +157,8 @@ async def open(
     ) -> RequestQueue:
         from crawlee.storages._creation_management import open_storage
 
-        configuration = configuration or service_container.get_configuration()
-        storage_client = storage_client or service_container.get_storage_client()
+        configuration = configuration or service_locator.get_configuration()
+        storage_client = storage_client or service_locator.get_storage_client()
 
         return await open_storage(
             storage_class=cls,
diff --git a/tests/unit/_autoscaling/test_snapshotter.py b/tests/unit/_autoscaling/test_snapshotter.py
@@ -7,7 +7,7 @@
 
 import pytest
 
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee._autoscaling import Snapshotter
 from crawlee._autoscaling.types import CpuSnapshot, EventLoopSnapshot, Snapshot
 from crawlee._utils.byte_size import ByteSize
@@ -248,7 +248,7 @@ def create_event_data(creation_time: datetime) -> EventSystemInfoData:
         )
 
     async with (
-        service_container.get_event_manager() as event_manager,
+        service_locator.get_event_manager() as event_manager,
         Snapshotter(available_memory_ratio=0.25) as snapshotter,
     ):
         event_manager.emit(event=Event.SYSTEM_INFO, event_data=create_event_data(time_new))
diff --git a/tests/unit/_memory_storage_client/test_memory_storage_client.py b/tests/unit/_memory_storage_client/test_memory_storage_client.py
@@ -8,7 +8,7 @@
 
 import pytest
 
-from crawlee import Request, service_container
+from crawlee import Request, service_locator
 from crawlee._consts import METADATA_FILENAME
 from crawlee.configuration import Configuration
 from crawlee.memory_storage_client import MemoryStorageClient
@@ -224,15 +224,15 @@ async def test_not_implemented_method(tmp_path: Path) -> None:
 async def test_default_storage_path_used(monkeypatch: pytest.MonkeyPatch) -> None:
     # We expect the default value to be used
     monkeypatch.delenv('CRAWLEE_STORAGE_DIR', raising=False)
-    service_container.set_configuration(Configuration())
+    service_locator.set_configuration(Configuration())
     ms = MemoryStorageClient()
     assert ms.storage_dir == './storage'
 
 
 async def test_storage_path_from_env_var_overrides_default(monkeypatch: pytest.MonkeyPatch) -> None:
     # We expect the env var to override the default value
     monkeypatch.setenv('CRAWLEE_STORAGE_DIR', './env_var_storage_dir')
-    service_container.set_configuration(Configuration())
+    service_locator.set_configuration(Configuration())
     ms = MemoryStorageClient()
     assert ms.storage_dir == './env_var_storage_dir'
 
diff --git a/tests/unit/_memory_storage_client/test_memory_storage_e2e.py b/tests/unit/_memory_storage_client/test_memory_storage_e2e.py
@@ -5,7 +5,7 @@
 
 import pytest
 
-from crawlee import Request, service_container
+from crawlee import Request, service_locator
 from crawlee.storages._key_value_store import KeyValueStore
 from crawlee.storages._request_queue import RequestQueue
 
@@ -22,7 +22,7 @@ async def test_actor_memory_storage_client_key_value_store_e2e(
     # Configure purging env var
     monkeypatch.setenv('CRAWLEE_PURGE_ON_START', f'{int(purge_on_start)}')
     # Store old storage client so we have the object reference for comparison
-    old_client = service_container.get_storage_client()
+    old_client = service_locator.get_storage_client()
 
     old_default_kvs = await KeyValueStore.open()
     old_non_default_kvs = await KeyValueStore.open(name='non-default')
@@ -35,7 +35,7 @@ async def test_actor_memory_storage_client_key_value_store_e2e(
     prepare_test_env()
 
     # Check if we're using a different memory storage instance
-    assert old_client is not service_container.get_storage_client()
+    assert old_client is not service_locator.get_storage_client()
     default_kvs = await KeyValueStore.open()
     assert default_kvs is not old_default_kvs
     non_default_kvs = await KeyValueStore.open(name='non-default')
diff --git a/tests/unit/basic_crawler/test_basic_crawler.py b/tests/unit/basic_crawler/test_basic_crawler.py
@@ -14,7 +14,7 @@
 import httpx
 import pytest
 
-from crawlee import ConcurrencySettings, EnqueueStrategy, Glob, service_container
+from crawlee import ConcurrencySettings, EnqueueStrategy, Glob, service_locator
 from crawlee._request import BaseRequestData, Request
 from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs, HttpHeaders
 from crawlee.basic_crawler import BasicCrawler
@@ -798,7 +798,7 @@ async def handler(context: BasicCrawlingContext) -> None:
 
 async def test_respects_no_persist_storage() -> None:
     config = Configuration(persist_storage=False)
-    service_container.set_configuration(config)
+    service_locator.set_configuration(config)
     crawler = BasicCrawler()
 
     @crawler.router.default_handler
diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py
@@ -10,7 +10,7 @@
 from proxy import Proxy
 from yarl import URL
 
-from crawlee import service_container
+from crawlee import service_locator
 from crawlee.configuration import Configuration
 from crawlee.events._local_event_manager import LocalEventManager
 from crawlee.memory_storage_client import MemoryStorageClient
@@ -42,14 +42,14 @@ def _prepare_test_env() -> None:
         monkeypatch.setenv('CRAWLEE_STORAGE_DIR', str(tmp_path))
 
         # Initialize services in the service container with default values.
-        service_container.set_configuration(Configuration())
-        service_container.set_storage_client(MemoryStorageClient())
-        service_container.set_event_manager(LocalEventManager())
+        service_locator.set_configuration(Configuration())
+        service_locator.set_storage_client(MemoryStorageClient())
+        service_locator.set_event_manager(LocalEventManager())
 
         # Reset the global state flags in the service locator.
-        service_container._service_locator._configuration_was_set = False
-        service_container._service_locator._storage_client_was_set = False
-        service_container._service_locator._event_manager_was_set = False
+        service_locator._service_locator._configuration_was_set = False
+        service_locator._service_locator._storage_client_was_set = False
+        service_locator._service_locator._event_manager_was_set = False
 
         # Clear creation-related caches to ensure no state is carried over between tests.
         monkeypatch.setattr(_creation_management, '_cache_dataset_by_id', {})
@@ -61,9 +61,9 @@ def _prepare_test_env() -> None:
 
         # Verify that the test environment was set up correctly.
         assert os.environ.get('CRAWLEE_STORAGE_DIR') == str(tmp_path)
-        assert service_container._service_locator.configuration_was_set is False
-        assert service_container._service_locator.storage_client_was_set is False
-        assert service_container._service_locator.event_manager_was_set is False
+        assert service_locator._service_locator.configuration_was_set is False
+        assert service_locator._service_locator.storage_client_was_set is False
+        assert service_locator._service_locator.event_manager_was_set is False
 
     return _prepare_test_env
 
@@ -81,9 +81,9 @@ def cleanup_test_env() -> Callable[[], None]:
 
     def _cleanup_test_env() -> None:
         # Reset the flags in the service locator to indicate no services are explicitly set.
-        service_container._service_locator._configuration_was_set = False
-        service_container._service_locator._storage_client_was_set = False
-        service_container._service_locator._event_manager_was_set = False
+        service_locator._service_locator._configuration_was_set = False
+        service_locator._service_locator._storage_client_was_set = False
+        service_locator._service_locator._event_manager_was_set = False
 
     return _cleanup_test_env
 
diff --git a/tests/unit/sessions/test_session_pool.py b/tests/unit/sessions/test_session_pool.py
diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py
diff --git a/tests/unit/test_service_locator.py b/tests/unit/test_service_locator.py
