same approach for event manager and storage client

Author: vdusek

src/crawlee/_autoscaling/snapshotter.py

@@ -10,13 +10,8 @@
 import psutil
 from sortedcontainers import SortedList
 
-from crawlee._autoscaling.types import (
-    ClientSnapshot,
-    CpuSnapshot,
-    EventLoopSnapshot,
-    MemorySnapshot,
-    Snapshot,
-)
+from crawlee import service_container
+from crawlee._autoscaling.types import ClientSnapshot, CpuSnapshot, EventLoopSnapshot, MemorySnapshot, Snapshot
 from crawlee._utils.byte_size import ByteSize
 from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
@@ -25,8 +20,6 @@
 if TYPE_CHECKING:
     from types import TracebackType
 
-    from crawlee.events import EventManager
-
 logger = getLogger(__name__)
 
 T = TypeVar('T')
@@ -44,7 +37,6 @@ class Snapshotter:
 
     def __init__(
         self,
-        event_manager: EventManager,
         *,
         event_loop_snapshot_interval: timedelta = timedelta(milliseconds=500),
         client_snapshot_interval: timedelta = timedelta(milliseconds=1000),
@@ -62,8 +54,6 @@ def __init__(
         """A default constructor.
 
         Args:
-            event_manager: The event manager used to emit system info events. From data provided by this event
-              the CPU and memory usage are read.
             event_loop_snapshot_interval: The interval at which the event loop is sampled.
             client_snapshot_interval: The interval at which the client is sampled.
             max_used_cpu_ratio: Sets the ratio, defining the maximum CPU usage. When the CPU usage is higher than
@@ -89,7 +79,8 @@ def __init__(
         if available_memory_ratio is None and max_memory_size is None:
             raise ValueError('At least one of `available_memory_ratio` or `max_memory_size` must be specified')
 
-        self._event_manager = event_manager
+        self._event_manager = service_container.get_event_manager()
+
         self._event_loop_snapshot_interval = event_loop_snapshot_interval
         self._client_snapshot_interval = client_snapshot_interval
         self._max_event_loop_delay = max_event_loop_delay

src/crawlee/basic_crawler/_basic_crawler.py

@@ -50,7 +50,6 @@
 
     from crawlee._types import ConcurrencySettings, HttpMethod, JsonSerializable
     from crawlee.base_storage_client._models import DatasetItemsListPage
-    from crawlee.events._event_manager import EventManager
     from crawlee.http_clients import BaseHttpClient, HttpResponse
     from crawlee.proxy_configuration import ProxyConfiguration, ProxyInfo
     from crawlee.sessions import Session
@@ -112,9 +111,6 @@ class BasicCrawlerOptions(TypedDict, Generic[TCrawlingContext]):
     statistics: NotRequired[Statistics[StatisticsState]]
     """A custom `Statistics` instance, allowing the use of non-default configuration."""
 
-    event_manager: NotRequired[EventManager]
-    """A custom `EventManager` instance, allowing the use of non-default configuration."""
-
     configure_logging: NotRequired[bool]
     """If True, the crawler will set up logging infrastructure automatically."""
 
@@ -174,7 +170,6 @@ def __init__(
         retry_on_blocked: bool = True,
         proxy_configuration: ProxyConfiguration | None = None,
         statistics: Statistics | None = None,
-        event_manager: EventManager | None = None,
         configure_logging: bool = True,
         max_crawl_depth: int | None = None,
         _context_pipeline: ContextPipeline[TCrawlingContext] | None = None,
@@ -201,7 +196,6 @@ def __init__(
             retry_on_blocked: If True, the crawler attempts to bypass bot protections automatically.
             proxy_configuration: HTTP proxy configuration used when making requests.
             statistics: A custom `Statistics` instance, allowing the use of non-default configuration.
-            event_manager: A custom `EventManager` instance, allowing the use of non-default configuration.
             configure_logging: If True, the crawler will set up logging infrastructure automatically.
             max_crawl_depth: Maximum crawl depth. If set, the crawler will stop crawling after reaching this depth.
             _context_pipeline: Enables extending the request lifecycle and modifying the crawling context.
@@ -241,9 +235,8 @@ def __init__(
 
         self._tld_extractor = TLDExtract(cache_dir=tempfile.TemporaryDirectory().name)
 
-        self._event_manager = event_manager or service_container.get_event_manager()
+        self._event_manager = service_container.get_event_manager()
         self._snapshotter = Snapshotter(
-            self._event_manager,
             max_memory_size=ByteSize.from_mb(config.memory_mbytes) if config.memory_mbytes else None,
             available_memory_ratio=config.available_memory_ratio,
         )

src/crawlee/configuration.py

@@ -5,7 +5,9 @@
 
 from pydantic import AliasChoices, BeforeValidator, Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
+from typing_extensions import Self
 
+from crawlee import service_container
 from crawlee._utils.docs import docs_group
 from crawlee._utils.models import timedelta_ms
 
@@ -229,3 +231,16 @@ class Configuration(BaseSettings):
         ),
     ] = False
     """This setting is currently unused. For more details, see https://github.com/apify/crawlee-python/issues/670."""
+
+    @classmethod
+    def get_global_configuration(cls) -> Self:
+        """Retrieve the global instance of the configuration.
+
+        Mostly for the backward compatibility.
+        """
+        config = service_container.get_configuration()
+
+        if not isinstance(config, cls):
+            raise TypeError(f'Requested configuration of type {cls}, but got {config.__class__} instead.')
+
+        return config

src/crawlee/sessions/_session_pool.py

@@ -6,6 +6,7 @@
 from logging import getLogger
 from typing import TYPE_CHECKING, Callable, Literal, overload
 
+from crawlee import service_container
 from crawlee._utils.docs import docs_group
 from crawlee.events._types import Event, EventPersistStateData
 from crawlee.sessions import Session
@@ -15,8 +16,6 @@
 if TYPE_CHECKING:
     from types import TracebackType
 
-    from crawlee.events import EventManager
-
 logger = getLogger(__name__)
 
 CreateSessionFunctionType = Callable[[], Session]
@@ -32,7 +31,6 @@ def __init__(
         max_pool_size: int = 1000,
         create_session_settings: dict | None = None,
         create_session_function: CreateSessionFunctionType | None = None,
-        event_manager: EventManager | None = None,
         persistence_enabled: bool = False,
         persist_state_kvs_name: str = 'default',
         persist_state_key: str = 'CRAWLEE_SESSION_POOL_STATE',
@@ -46,7 +44,6 @@ def __init__(
                 be used. Do not set it if you are providing a `create_session_function`.
             create_session_function: A callable to create new session instances. If None, a default session settings
                 will be used. Do not set it if you are providing `create_session_settings`.
-            event_manager: The event manager to handle events like persist state.
             persistence_enabled: Flag to enable or disable state persistence of the pool. If it is enabled, make sure
                 to provide an event manager to handle the events.
             persist_state_kvs_name: The name of the `KeyValueStore` used for state persistence.
@@ -55,7 +52,7 @@ def __init__(
         self._max_pool_size = max_pool_size
         self._session_settings = create_session_settings or {}
         self._create_session_function = create_session_function
-        self._event_manager = event_manager
+        self._event_manager = service_container.get_event_manager()
         self._persistence_enabled = persistence_enabled
         self._persist_state_kvs_name = persist_state_kvs_name
         self._persist_state_key = persist_state_key

src/crawlee/statistics/_statistics.py

@@ -8,7 +8,7 @@
 
 from typing_extensions import Self, TypeVar
 
-import crawlee.service_container
+from crawlee import service_container
 from crawlee._utils.docs import docs_group
 from crawlee._utils.recurring_task import RecurringTask
 from crawlee.events._types import Event, EventPersistStateData
@@ -19,8 +19,6 @@
 if TYPE_CHECKING:
     from types import TracebackType
 
-    from crawlee.events import EventManager
-
 TStatisticsState = TypeVar('TStatisticsState', bound=StatisticsState, default=StatisticsState)
 
 logger = getLogger(__name__)
@@ -66,7 +64,6 @@ class Statistics(Generic[TStatisticsState]):
     def __init__(
         self,
         *,
-        event_manager: EventManager | None = None,
         persistence_enabled: bool = False,
         persist_state_kvs_name: str = 'default',
         persist_state_key: str | None = None,
@@ -86,9 +83,7 @@ def __init__(
 
         self.error_tracker = ErrorTracker()
         self.error_tracker_retry = ErrorTracker()
-
-        self._events = event_manager or crawlee.service_container.get_event_manager()
-
+        self._event_manager = service_container.get_event_manager()
         self._requests_in_progress = dict[str, RequestProcessingRecord]()
 
         if persist_state_key is None:
@@ -114,7 +109,7 @@ async def __aenter__(self) -> Self:
             self._key_value_store = await KeyValueStore.open(name=self._persist_state_kvs_name)
 
         await self._maybe_load_statistics()
-        self._events.on(event=Event.PERSIST_STATE, listener=self._persist_state)
+        self._event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_state)
 
         self._periodic_logger.start()
 
@@ -128,7 +123,7 @@ async def __aexit__(
     ) -> None:
         """Stop collecting statistics."""
         self.state.crawler_finished_at = datetime.now(timezone.utc)
-        self._events.off(event=Event.PERSIST_STATE, listener=self._persist_state)
+        self._event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_state)
         await self._periodic_logger.stop()
         await self._persist_state(event_data=EventPersistStateData(is_migrating=False))
 

src/crawlee/storages/_creation_management.py

@@ -122,13 +122,12 @@ def _get_default_storage_id(configuration: Configuration, storage_class: type[TR
 async def open_storage(
     *,
     storage_class: type[TResource],
-    storage_client: BaseStorageClient | None = None,
     id: str | None = None,
     name: str | None = None,
 ) -> TResource:
     """Open either a new storage or restore an existing one and return it."""
     config = service_container.get_configuration()
-    storage_client = storage_client or service_container.get_storage_client()
+    storage_client = service_container.get_storage_client()
 
     # Try to restore the storage from cache by name
     if name:
@@ -175,7 +174,6 @@ async def open_storage(
                 id=storage_info.id,
                 name=storage_info.name,
                 client=storage_client,
-                event_manager=service_container.get_event_manager(),
             )
         else:
             storage = storage_class(

src/crawlee/storages/_dataset.py

@@ -8,6 +8,7 @@
 
 from typing_extensions import NotRequired, Required, Unpack, override
 
+from crawlee import service_container
 from crawlee._utils.byte_size import ByteSize
 from crawlee._utils.docs import docs_group
 from crawlee._utils.file import json_dumps
@@ -19,7 +20,6 @@
     from collections.abc import Callable
 
     from crawlee._types import JsonSerializable, PushDataKwargs
-    from crawlee.base_storage_client import BaseStorageClient
     from crawlee.base_storage_client._models import DatasetItemsListPage
 
 
@@ -192,18 +192,14 @@ class Dataset(BaseStorage):
     _EFFECTIVE_LIMIT_SIZE = _MAX_PAYLOAD_SIZE - (_MAX_PAYLOAD_SIZE * _SAFETY_BUFFER_PERCENT)
     """Calculated payload limit considering safety buffer."""
 
-    def __init__(
-        self,
-        id: str,
-        name: str | None,
-        client: BaseStorageClient,
-    ) -> None:
+    def __init__(self, id: str, name: str | None) -> None:
         self._id = id
         self._name = name
 
         # Get resource clients from storage client
-        self._resource_client = client.dataset(self._id)
-        self._resource_collection_client = client.datasets()
+        storage_client = service_container.get_storage_client()
+        self._resource_client = storage_client.dataset(self._id)
+        self._resource_collection_client = storage_client.datasets()
 
     @override
     @property
@@ -217,21 +213,10 @@ def name(self) -> str | None:
 
     @override
     @classmethod
-    async def open(
-        cls,
-        *,
-        id: str | None = None,
-        name: str | None = None,
-        storage_client: BaseStorageClient | None = None,
-    ) -> Dataset:
+    async def open(cls, *, id: str | None = None, name: str | None = None) -> Dataset:
         from crawlee.storages._creation_management import open_storage
 
-        return await open_storage(
-            storage_class=cls,
-            id=id,
-            name=name,
-            storage_client=storage_client,
-        )
+        return await open_storage(storage_class=cls, id=id, name=name)
 
     @override
     async def drop(self) -> None:

src/crawlee/storages/_key_value_store.py

@@ -1,16 +1,14 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, AsyncIterator, TypeVar, overload
+from typing import Any, AsyncIterator, TypeVar, overload
 
 from typing_extensions import override
 
+from crawlee import service_container
 from crawlee._utils.docs import docs_group
 from crawlee.base_storage_client._models import KeyValueStoreKeyInfo, KeyValueStoreMetadata
 from crawlee.storages._base_storage import BaseStorage
 
-if TYPE_CHECKING:
-    from crawlee.base_storage_client import BaseStorageClient
-
 T = TypeVar('T')
 
 
@@ -49,17 +47,13 @@ class KeyValueStore(BaseStorage):
     ```
     """
 
-    def __init__(
-        self,
-        id: str,
-        name: str | None,
-        client: BaseStorageClient,
-    ) -> None:
+    def __init__(self, id: str, name: str | None) -> None:
         self._id = id
         self._name = name
 
         # Get resource clients from storage client
-        self._resource_client = client.key_value_store(self._id)
+        storage_client = service_container.get_storage_client()
+        self._resource_client = storage_client.key_value_store(self._id)
 
     @override
     @property
@@ -77,21 +71,10 @@ async def get_info(self) -> KeyValueStoreMetadata | None:
 
     @override
     @classmethod
-    async def open(
-        cls,
-        *,
-        id: str | None = None,
-        name: str | None = None,
-        storage_client: BaseStorageClient | None = None,
-    ) -> KeyValueStore:
+    async def open(cls, *, id: str | None = None, name: str | None = None) -> KeyValueStore:
         from crawlee.storages._creation_management import open_storage
 
-        return await open_storage(
-            storage_class=cls,
-            id=id,
-            name=name,
-            storage_client=storage_client,
-        )
+        return await open_storage(storage_class=cls, id=id, name=name)
 
     @override
     async def drop(self) -> None: