refactor!: Refactor storage creation and caching, configuration and services

docs/guides/code_examples/service_locator/service_storage_configuration.py

@@ -1,7 +1,9 @@
 import asyncio
 from datetime import timedelta
 
+from crawlee import service_locator
 from crawlee.configuration import Configuration
+from crawlee.storage_clients import MemoryStorageClient
 from crawlee.storages import Dataset
 
 
@@ -11,10 +13,16 @@ async def main() -> None:
         headless=False,
         persist_state_interval=timedelta(seconds=30),
     )
+    # Set the custom configuration as the global default configuration.
+    service_locator.set_configuration(configuration)
 
-    # Pass the configuration to the dataset (or other storage) when opening it.
-    dataset = await Dataset.open(
-        configuration=configuration,
+    # Use the global defaults when creating the dataset (or other storage).
+    dataset_1 = await Dataset.open()
+
+    # Or set explicitly specific configuration if
+    # you do not want to rely on global defaults.
+    dataset_2 = await Dataset.open(
+        storage_client=MemoryStorageClient(), configuration=configuration
     )
 
 

docs/upgrading/upgrading_to_v1.md

@@ -189,6 +189,96 @@ The interface for custom storage clients has been simplified:
 - Collection storage clients have been removed.
 - The number of methods that have to be implemented have been reduced.
 
+## ServiceLocator changes
+
+### ServiceLocator is stricter with registering services
+You can register the services just once, and you can no longer override already registered services.
+
+**Before (v0.6):**
+```python
+from crawlee import service_locator
+from crawlee.storage_clients import MemoryStorageClient
+
+service_locator.set_storage_client(MemoryStorageClient())
+service_locator.set_storage_client(MemoryStorageClient())
+```
+**Now (v1.0):**
+
+```python
+from crawlee import service_locator
+from crawlee.storage_clients import MemoryStorageClient
+
+service_locator.set_storage_client(MemoryStorageClient())
+service_locator.set_storage_client(MemoryStorageClient())  # Raises an error
+```
+
+### BasicCrawler has its own instance of ServiceLocator to track its own services
+Explicitly passed services to the crawler can be different the global ones accessible in `crawlee.service_locator`. `BasicCrawler` no longer causes the global services in `service_locator` to be set to the crawler's explicitly passed services.
+
+**Before (v0.6):**
+```python
+from crawlee import service_locator
+from crawlee.crawlers import BasicCrawler
+from crawlee.storage_clients import MemoryStorageClient
+from crawlee.storages import Dataset
+
+
+async def main() -> None:
+    custom_storage_client = MemoryStorageClient()
+    crawler = BasicCrawler(storage_client=custom_storage_client)
+
+    assert service_locator.get_storage_client() is custom_storage_client
+    assert await crawler.get_dataset() is await Dataset.open()
+```
+**Now (v1.0):**
+
+```python
+from crawlee import service_locator
+from crawlee.crawlers import BasicCrawler
+from crawlee.storage_clients import MemoryStorageClient
+from crawlee.storages import Dataset
+
+
+async def main() -> None:
+    custom_storage_client = MemoryStorageClient()
+    crawler = BasicCrawler(storage_client=custom_storage_client)
+
+    assert service_locator.get_storage_client() is not custom_storage_client
+    assert await crawler.get_dataset() is not await Dataset.open()
+```
+### There can be two crawlers with different services at the same time
+
+**Now (v1.0):**
+
+```python
+from crawlee.crawlers import BasicCrawler
+from crawlee.storage_clients import MemoryStorageClient, FileSystemStorageClient
+from crawlee.configuration import Configuration
+from crawlee.events import LocalEventManager
+
+custom_configuration_1 = Configuration()
+custom_event_manager_1 = LocalEventManager.from_config(custom_configuration_1)
+custom_storage_client_1 = MemoryStorageClient()
+
+custom_configuration_2 = Configuration()
+custom_event_manager_2 = LocalEventManager.from_config(custom_configuration_2)
+custom_storage_client_2 = FileSystemStorageClient()
+
+crawler_1 = BasicCrawler(
+    configuration=custom_configuration_1,
+    event_manager=custom_event_manager_1,
+    storage_client=custom_storage_client_1,
+)
+
+crawler_2 = BasicCrawler(
+    configuration=custom_configuration_2,
+    event_manager=custom_event_manager_2,
+    storage_client=custom_storage_client_2,
+  )
+
+# use crawlers without runtime crash...
+```
+
 ## Other smaller updates
 
 There are more smaller updates.

src/crawlee/_autoscaling/snapshotter.py

@@ -113,7 +113,7 @@ def from_config(cls, config: Configuration | None = None) -> Snapshotter:
         Args:
             config: The `Configuration` instance. Uses the global (default) one if not provided.
         """
-        config = service_locator.get_configuration()
+        config = config or service_locator.get_configuration()
 
         # Compute the maximum memory size based on the provided configuration. If `memory_mbytes` is provided,
         # it uses that value. Otherwise, it calculates the `max_memory_size` as a proportion of the system's

src/crawlee/_service_locator.py

@@ -11,6 +11,10 @@
 if TYPE_CHECKING:
     from crawlee.storages._storage_instance_manager import StorageInstanceManager
 
+from logging import getLogger
+
+logger = getLogger(__name__)
+
 
 @docs_group('Configuration')
 class ServiceLocator:
@@ -19,23 +23,24 @@ class ServiceLocator:
     All services are initialized to its default value lazily.
     """
 
-    def __init__(self) -> None:
-        self._configuration: Configuration | None = None
-        self._event_manager: EventManager | None = None
-        self._storage_client: StorageClient | None = None
-        self._storage_instance_manager: StorageInstanceManager | None = None
+    global_storage_instance_manager: StorageInstanceManager | None = None
 
-        # Flags to check if the services were already set.
-        self._configuration_was_retrieved = False
-        self._event_manager_was_retrieved = False
-        self._storage_client_was_retrieved = False
+    def __init__(
+        self,
+        configuration: Configuration | None = None,
+        event_manager: EventManager | None = None,
+        storage_client: StorageClient | None = None,
+    ) -> None:
+        self._configuration = configuration
+        self._event_manager = event_manager
+        self._storage_client = storage_client
 
     def get_configuration(self) -> Configuration:
         """Get the configuration."""
         if self._configuration is None:
+            logger.warning('No configuration set, implicitly creating and using default Configuration.')
             self._configuration = Configuration()
 
-        self._configuration_was_retrieved = True
         return self._configuration
 
     def set_configuration(self, configuration: Configuration) -> None:
@@ -47,21 +52,25 @@ def set_configuration(self, configuration: Configuration) -> None:
         Raises:
             ServiceConflictError: If the configuration has already been retrieved before.
         """
-        if self._configuration_was_retrieved:
+        if self._configuration is configuration:
+            # Same instance, no need to anything
+            return
+        if self._configuration:
             raise ServiceConflictError(Configuration, configuration, self._configuration)
 
         self._configuration = configuration
 
     def get_event_manager(self) -> EventManager:
         """Get the event manager."""
         if self._event_manager is None:
-            self._event_manager = (
-                LocalEventManager().from_config(config=self._configuration)
-                if self._configuration
-                else LocalEventManager.from_config()
-            )
+            logger.warning('No event manager set, implicitly creating and using default LocalEventManager.')
+            if self._configuration is None:
+                logger.warning(
+                    'Implicit creation of event manager will implicitly set configuration as side effect. '
+                    'It is advised to explicitly first set the configuration instead.'
+                )
+            self._event_manager = LocalEventManager().from_config(config=self._configuration)
 
-        self._event_manager_was_retrieved = True
         return self._event_manager
 
     def set_event_manager(self, event_manager: EventManager) -> None:
@@ -73,17 +82,25 @@ def set_event_manager(self, event_manager: EventManager) -> None:
         Raises:
             ServiceConflictError: If the event manager has already been retrieved before.
         """
-        if self._event_manager_was_retrieved:
+        if self._event_manager is event_manager:
+            # Same instance, no need to anything
+            return
+        if self._event_manager:
             raise ServiceConflictError(EventManager, event_manager, self._event_manager)
 
         self._event_manager = event_manager
 
     def get_storage_client(self) -> StorageClient:
         """Get the storage client."""
         if self._storage_client is None:
+            logger.warning('No storage client set, implicitly creating and using default FileSystemStorageClient.')
+            if self._configuration is None:
+                logger.warning(
+                    'Implicit creation of storage client will implicitly set configuration as side effect. '
+                    'It is advised to explicitly first set the configuration instead.'
+                )
             self._storage_client = FileSystemStorageClient()
 
-        self._storage_client_was_retrieved = True
         return self._storage_client
 
     def set_storage_client(self, storage_client: StorageClient) -> None:
@@ -95,21 +112,24 @@ def set_storage_client(self, storage_client: StorageClient) -> None:
         Raises:
             ServiceConflictError: If the storage client has already been retrieved before.
         """
-        if self._storage_client_was_retrieved:
+        if self._storage_client is storage_client:
+            # Same instance, no need to anything
+            return
+        if self._storage_client:
             raise ServiceConflictError(StorageClient, storage_client, self._storage_client)
 
         self._storage_client = storage_client
 
     @property
     def storage_instance_manager(self) -> StorageInstanceManager:
-        """Get the storage instance manager."""
-        if self._storage_instance_manager is None:
+        """Get the storage instance manager. It is global manager shared by all instances of ServiceLocator."""
+        if ServiceLocator.global_storage_instance_manager is None:
             # Import here to avoid circular imports.
             from crawlee.storages._storage_instance_manager import StorageInstanceManager  # noqa: PLC0415
 
-            self._storage_instance_manager = StorageInstanceManager()
+            ServiceLocator.global_storage_instance_manager = StorageInstanceManager()
 
-        return self._storage_instance_manager
+        return ServiceLocator.global_storage_instance_manager
 
 
 service_locator = ServiceLocator()