refactor!: Refactor service container and other related components

Author: vdusek

src/crawlee/_log_config.py

@@ -4,13 +4,12 @@
 import logging
 import sys
 import textwrap
-from typing import TYPE_CHECKING, Any
+from typing import Any
 
 from colorama import Fore, Style, just_fix_windows_console
 from typing_extensions import assert_never
 
-if TYPE_CHECKING:
-    from crawlee.configuration import Configuration
+from crawlee import service_container
 
 just_fix_windows_console()
 
@@ -35,35 +34,32 @@
 _LOG_MESSAGE_INDENT = ' ' * 6
 
 
-def get_configured_log_level(configuration: Configuration) -> int:
-    verbose_logging_requested = 'verbose_log' in configuration.model_fields_set and configuration.verbose_log
+def get_configured_log_level() -> int:
+    config = service_container.get_configuration()
 
-    if 'log_level' in configuration.model_fields_set:
-        if configuration.log_level == 'DEBUG':
+    verbose_logging_requested = 'verbose_log' in config.model_fields_set and config.verbose_log
+
+    if 'log_level' in config.model_fields_set:
+        if config.log_level == 'DEBUG':
             return logging.DEBUG
-        if configuration.log_level == 'INFO':
+        if config.log_level == 'INFO':
             return logging.INFO
-        if configuration.log_level == 'WARNING':
+        if config.log_level == 'WARNING':
             return logging.WARNING
-        if configuration.log_level == 'ERROR':
+        if config.log_level == 'ERROR':
             return logging.ERROR
-        if configuration.log_level == 'CRITICAL':
+        if config.log_level == 'CRITICAL':
             return logging.CRITICAL
 
-        assert_never(configuration.log_level)
+        assert_never(config.log_level)
 
     if sys.flags.dev_mode or verbose_logging_requested:
         return logging.DEBUG
 
     return logging.INFO
 
 
-def configure_logger(
-    logger: logging.Logger,
-    configuration: Configuration,
-    *,
-    remove_old_handlers: bool = False,
-) -> None:
+def configure_logger(logger: logging.Logger, *, remove_old_handlers: bool = False) -> None:
     handler = logging.StreamHandler()
     handler.setFormatter(CrawleeLogFormatter())
 
@@ -72,7 +68,7 @@ def configure_logger(
             logger.removeHandler(old_handler)
 
     logger.addHandler(handler)
-    logger.setLevel(get_configured_log_level(configuration))
+    logger.setLevel(get_configured_log_level())
 
 
 class CrawleeLogFormatter(logging.Formatter):

src/crawlee/basic_crawler/_basic_crawler.py

@@ -50,7 +50,6 @@
 
     from crawlee._types import ConcurrencySettings, HttpMethod, JsonSerializable
     from crawlee.base_storage_client._models import DatasetItemsListPage
-    from crawlee.configuration import Configuration
     from crawlee.events._event_manager import EventManager
     from crawlee.http_clients import BaseHttpClient, HttpResponse
     from crawlee.proxy_configuration import ProxyConfiguration, ProxyInfo
@@ -95,9 +94,6 @@ class BasicCrawlerOptions(TypedDict, Generic[TCrawlingContext]):
     """Maximum number of session rotations per request. The crawler rotates the session if a proxy error occurs
     or if the website blocks the request."""
 
-    configuration: NotRequired[Configuration]
-    """Crawler configuration."""
-
     request_handler_timeout: NotRequired[timedelta]
     """Maximum duration allowed for a single request handler to run."""
 
@@ -172,7 +168,6 @@ def __init__(
         max_request_retries: int = 3,
         max_requests_per_crawl: int | None = None,
         max_session_rotations: int = 10,
-        configuration: Configuration | None = None,
         request_handler_timeout: timedelta = timedelta(minutes=1),
         session_pool: SessionPool | None = None,
         use_session_pool: bool = True,
@@ -200,7 +195,6 @@ def __init__(
                 this value.
             max_session_rotations: Maximum number of session rotations per request. The crawler rotates the session
                 if a proxy error occurs or if the website blocks the request.
-            configuration: Crawler configuration.
             request_handler_timeout: Maximum duration allowed for a single request handler to run.
             use_session_pool: Enable the use of a session pool for managing sessions during crawling.
             session_pool: A custom `SessionPool` instance, allowing the use of non-default configuration.
@@ -235,12 +229,13 @@ def __init__(
         self._max_session_rotations = max_session_rotations
 
         self._request_provider = request_provider
-        self._configuration = configuration or service_container.get_configuration()
+
+        config = service_container.get_configuration()
 
         self._request_handler_timeout = request_handler_timeout
         self._internal_timeout = (
-            self._configuration.internal_timeout
-            if self._configuration.internal_timeout is not None
+            config.internal_timeout
+            if config.internal_timeout is not None
             else max(2 * request_handler_timeout, timedelta(minutes=5))
         )
 
@@ -249,10 +244,8 @@ def __init__(
         self._event_manager = event_manager or service_container.get_event_manager()
         self._snapshotter = Snapshotter(
             self._event_manager,
-            max_memory_size=ByteSize.from_mb(self._configuration.memory_mbytes)
-            if self._configuration.memory_mbytes
-            else None,
-            available_memory_ratio=self._configuration.available_memory_ratio,
+            max_memory_size=ByteSize.from_mb(config.memory_mbytes) if config.memory_mbytes else None,
+            available_memory_ratio=config.available_memory_ratio,
         )
         self._pool = AutoscaledPool(
             system_status=SystemStatus(self._snapshotter),
@@ -269,13 +262,11 @@ def __init__(
 
         if configure_logging:
             root_logger = logging.getLogger()
-            configure_logger(root_logger, self._configuration, remove_old_handlers=True)
+            configure_logger(root_logger, remove_old_handlers=True)
 
             # Silence HTTPX logger
             httpx_logger = logging.getLogger('httpx')
-            httpx_logger.setLevel(
-                logging.DEBUG if get_configured_log_level(self._configuration) <= logging.DEBUG else logging.WARNING
-            )
+            httpx_logger.setLevel(logging.DEBUG if get_configured_log_level() <= logging.DEBUG else logging.WARNING)
 
         if not _logger:
             _logger = logging.getLogger(__name__)
@@ -360,7 +351,7 @@ async def get_request_provider(
     ) -> RequestProvider:
         """Return the configured request provider. If none is configured, open and return the default request queue."""
         if not self._request_provider:
-            self._request_provider = await RequestQueue.open(id=id, name=name, configuration=self._configuration)
+            self._request_provider = await RequestQueue.open(id=id, name=name)
 
         return self._request_provider
 
@@ -371,7 +362,7 @@ async def get_dataset(
         name: str | None = None,
     ) -> Dataset:
         """Return the dataset with the given ID or name. If none is provided, return the default dataset."""
-        return await Dataset.open(id=id, name=name, configuration=self._configuration)
+        return await Dataset.open(id=id, name=name)
 
     async def get_key_value_store(
         self,
@@ -380,7 +371,7 @@ async def get_key_value_store(
         name: str | None = None,
     ) -> KeyValueStore:
         """Return the key-value store with the given ID or name. If none is provided, return the default KVS."""
-        return await KeyValueStore.open(id=id, name=name, configuration=self._configuration)
+        return await KeyValueStore.open(id=id, name=name)
 
     def error_handler(
         self, handler: ErrorHandler[TCrawlingContext | BasicCrawlingContext]
@@ -425,7 +416,7 @@ async def run(
             request_provider = await self.get_request_provider()
             if purge_request_queue and isinstance(request_provider, RequestQueue):
                 await request_provider.drop()
-                self._request_provider = await RequestQueue.open(configuration=self._configuration)
+                self._request_provider = await RequestQueue.open()
 
         if requests is not None:
             await self.add_requests(requests)

src/crawlee/configuration.py

@@ -229,20 +229,3 @@ class Configuration(BaseSettings):
         ),
     ] = False
     """This setting is currently unused. For more details, see https://github.com/apify/crawlee-python/issues/670."""
-
-    @classmethod
-    def get_global_configuration(cls) -> Self:
-        """Retrieve the global instance of the configuration."""
-        from crawlee import service_container
-
-        if service_container.get_configuration_if_set() is None:
-            service_container.set_configuration(cls())
-
-        global_instance = service_container.get_configuration()
-
-        if not isinstance(global_instance, cls):
-            raise TypeError(
-                f'Requested global configuration object of type {cls}, but {global_instance.__class__} was found'
-            )
-
-        return global_instance

src/crawlee/errors.py

@@ -16,7 +16,6 @@
     'RequestHandlerError',
     'SessionError',
     'UserDefinedErrorHandlerError',
-    'ServiceConflictError',
 ]
 
 TCrawlingContext = TypeVar('TCrawlingContext', bound=BasicCrawlingContext, default=BasicCrawlingContext)
@@ -89,13 +88,3 @@ def __init__(self, wrapped_exception: Exception, crawling_context: BasicCrawling
 @docs_group('Errors')
 class ContextPipelineInterruptedError(Exception):
     """May be thrown in the initialization phase of a middleware to signal that the request should not be processed."""
-
-
-@docs_group('Errors')
-class ServiceConflictError(RuntimeError):
-    """Thrown when a service container is getting reconfigured."""
-
-    def __init__(self, service_name: str, new_value: object, old_value: object) -> None:
-        super().__init__(
-            f"Service '{service_name}' was already set (existing value is '{old_value}', new value is '{new_value}')."
-        )

src/crawlee/memory_storage_client/_creation_management.py

@@ -10,6 +10,7 @@
 from logging import getLogger
 from typing import TYPE_CHECKING
 
+from crawlee import service_container
 from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.data_processing import maybe_parse_body
 from crawlee._utils.file import json_dumps
@@ -405,17 +406,17 @@ def _determine_storage_path(
     from crawlee.memory_storage_client._request_queue_client import RequestQueueClient
     from crawlee.storages._creation_management import _get_default_storage_id
 
-    configuration = memory_storage_client._configuration  # noqa: SLF001
+    config = service_container.get_configuration()
 
     if issubclass(resource_client_class, DatasetClient):
         storages_dir = memory_storage_client.datasets_directory
-        default_id = _get_default_storage_id(configuration, Dataset)
+        default_id = _get_default_storage_id(config, Dataset)
     elif issubclass(resource_client_class, KeyValueStoreClient):
         storages_dir = memory_storage_client.key_value_stores_directory
-        default_id = _get_default_storage_id(configuration, KeyValueStore)
+        default_id = _get_default_storage_id(config, KeyValueStore)
     elif issubclass(resource_client_class, RequestQueueClient):
         storages_dir = memory_storage_client.request_queues_directory
-        default_id = _get_default_storage_id(configuration, RequestQueue)
+        default_id = _get_default_storage_id(config, RequestQueue)
     else:
         raise TypeError('Invalid resource client class.')
 

src/crawlee/memory_storage_client/_memory_storage_client.py

@@ -10,9 +10,9 @@
 
 from typing_extensions import override
 
+from crawlee import service_container
 from crawlee._utils.docs import docs_group
 from crawlee.base_storage_client import BaseStorageClient
-from crawlee.configuration import Configuration
 from crawlee.memory_storage_client._dataset_client import DatasetClient
 from crawlee.memory_storage_client._dataset_collection_client import DatasetCollectionClient
 from crawlee.memory_storage_client._key_value_store_client import KeyValueStoreClient
@@ -45,13 +45,34 @@ class MemoryStorageClient(BaseStorageClient):
     _TEMPORARY_DIR_NAME = '__CRAWLEE_TEMPORARY'
     """Name of the directory used to temporarily store files during purges."""
 
-    def __init__(self, configuration: Configuration | None = None) -> None:
+    def __init__(
+        self,
+        write_metadata: bool | None = None,
+        persist_storage: bool | None = None,
+        storage_dir: str | None = None,
+        default_request_queue_id: str | None = None,
+        default_key_value_store_id: str | None = None,
+        default_dataset_id: str | None = None,
+    ) -> None:
         """A default constructor.
 
         Args:
-            configuration: Configuration object to use. If None, a default instance will be created.
+            write_metadata: Whether to write metadata to the storage.
+            persist_storage: Whether to persist the storage.
+            storage_dir: Path to the storage directory.
+            default_request_queue_id: The default request queue ID.
+            default_key_value_store_id: The default key-value store ID.
+            default_dataset_id: The default dataset ID.
         """
-        self._explicit_configuration = configuration
+        config = service_container.get_configuration()
+
+        # Set the internal attributes.
+        self._write_metadata = write_metadata or config.write_metadata
+        self._persist_storage = persist_storage or config.persist_storage
+        self._storage_dir = storage_dir or config.storage_dir
+        self._default_request_queue_id = default_request_queue_id or config.default_request_queue_id
+        self._default_key_value_store_id = default_key_value_store_id or config.default_key_value_store_id
+        self._default_dataset_id = default_dataset_id or config.default_dataset_id
 
         self.datasets_handled: list[DatasetClient] = []
         self.key_value_stores_handled: list[KeyValueStoreClient] = []
@@ -60,24 +81,20 @@ def __init__(self, configuration: Configuration | None = None) -> None:
         self._purged_on_start = False  # Indicates whether a purge was already performed on this instance.
         self._purge_lock = asyncio.Lock()
 
-    @property
-    def _configuration(self) -> Configuration:
-        return self._explicit_configuration or Configuration.get_global_configuration()
-
     @property
     def write_metadata(self) -> bool:
         """Whether to write metadata to the storage."""
-        return self._configuration.write_metadata
+        return self._write_metadata
 
     @property
     def persist_storage(self) -> bool:
         """Whether to persist the storage."""
-        return self._configuration.persist_storage
+        return self._persist_storage
 
     @property
     def storage_dir(self) -> str:
         """Path to the storage directory."""
-        return self._configuration.storage_dir
+        return self._storage_dir
 
     @property
     def datasets_directory(self) -> str:
@@ -197,14 +214,14 @@ async def _purge_default_storages(self) -> None:
                     self._TEMPORARY_DIR_NAME
                 ) or key_value_store_folder.name.startswith('__OLD'):
                     await self._batch_remove_files(key_value_store_folder.path)
-                elif key_value_store_folder.name == self._configuration.default_key_value_store_id:
+                elif key_value_store_folder.name == self._default_key_value_store_id:
                     await self._handle_default_key_value_store(key_value_store_folder.path)
 
         # Datasets
         if await asyncio.to_thread(os.path.exists, self.datasets_directory):
             dataset_folders = await asyncio.to_thread(os.scandir, self.datasets_directory)
             for dataset_folder in dataset_folders:
-                if dataset_folder.name == self._configuration.default_dataset_id or dataset_folder.name.startswith(
+                if dataset_folder.name == self._default_dataset_id or dataset_folder.name.startswith(
                     self._TEMPORARY_DIR_NAME
                 ):
                     await self._batch_remove_files(dataset_folder.path)
@@ -213,9 +230,8 @@ async def _purge_default_storages(self) -> None:
         if await asyncio.to_thread(os.path.exists, self.request_queues_directory):
             request_queue_folders = await asyncio.to_thread(os.scandir, self.request_queues_directory)
             for request_queue_folder in request_queue_folders:
-                if (
-                    request_queue_folder.name == self._configuration.default_request_queue_id
-                    or request_queue_folder.name.startswith(self._TEMPORARY_DIR_NAME)
+                if request_queue_folder.name == self._default_request_queue_id or request_queue_folder.name.startswith(
+                    self._TEMPORARY_DIR_NAME
                 ):
                     await self._batch_remove_files(request_queue_folder.path)