add memory storage from_config constructor

Author: vdusek

docs/guides/code/request_storage/purge_explicitly_example.py

@@ -4,7 +4,7 @@
 
 
 async def main() -> None:
-    storage_client = MemoryStorageClient()
+    storage_client = MemoryStorageClient.from_config()
     # highlight-next-line
     await storage_client.purge_on_start()
 

src/crawlee/_service_locator.py

@@ -75,7 +75,7 @@ def get_storage_client(self) -> BaseStorageClient:
         if self._storage_client is None:
             from crawlee.memory_storage_client import MemoryStorageClient
 
-            self._storage_client = MemoryStorageClient()
+            self._storage_client = MemoryStorageClient.from_config()
 
         return self._storage_client
 

src/crawlee/memory_storage_client/_memory_storage_client.py

@@ -10,9 +10,9 @@
 
 from typing_extensions import override
 
-from crawlee import service_locator
 from crawlee._utils.docs import docs_group
 from crawlee.base_storage_client import BaseStorageClient
+from crawlee.configuration import Configuration
 from crawlee.memory_storage_client._dataset_client import DatasetClient
 from crawlee.memory_storage_client._dataset_collection_client import DatasetCollectionClient
 from crawlee.memory_storage_client._key_value_store_client import KeyValueStoreClient
@@ -22,7 +22,7 @@
 
 if TYPE_CHECKING:
     from crawlee.base_storage_client._types import ResourceClient
-    from crawlee.configuration import Configuration
+
 
 TResourceClient = TypeVar('TResourceClient', DatasetClient, KeyValueStoreClient, RequestQueueClient)
 
@@ -57,38 +57,31 @@ class MemoryStorageClient(BaseStorageClient):
 
     def __init__(
         self,
-        configuration: Configuration | None = None,
         *,
-        write_metadata: bool | None = None,
-        persist_storage: bool | None = None,
-        storage_dir: str | None = None,
-        default_request_queue_id: str | None = None,
-        default_key_value_store_id: str | None = None,
-        default_dataset_id: str | None = None,
+        write_metadata: bool,
+        persist_storage: bool,
+        storage_dir: str,
+        default_request_queue_id: str,
+        default_key_value_store_id: str,
+        default_dataset_id: str,
     ) -> None:
         """A default constructor.
 
-        All parameters are optional and can be set either directly or via the configuration object. The defaults
-        are taken from the configuration object.
-
         Args:
-            configuration: The configuration object.
             write_metadata: Whether to write metadata to the storage.
             persist_storage: Whether to persist the storage.
             storage_dir: Path to the storage directory.
             default_request_queue_id: The default request queue ID.
             default_key_value_store_id: The default key-value store ID.
             default_dataset_id: The default dataset ID.
         """
-        config = configuration or service_locator.get_configuration()
-
         # Set the internal attributes.
-        self._write_metadata = write_metadata or config.write_metadata
-        self._persist_storage = persist_storage or config.persist_storage
-        self._storage_dir = storage_dir or config.storage_dir
-        self._default_request_queue_id = default_request_queue_id or config.default_request_queue_id
-        self._default_key_value_store_id = default_key_value_store_id or config.default_key_value_store_id
-        self._default_dataset_id = default_dataset_id or config.default_dataset_id
+        self._write_metadata = write_metadata
+        self._persist_storage = persist_storage
+        self._storage_dir = storage_dir
+        self._default_request_queue_id = default_request_queue_id
+        self._default_key_value_store_id = default_key_value_store_id
+        self._default_dataset_id = default_dataset_id
 
         self.datasets_handled: list[DatasetClient] = []
         self.key_value_stores_handled: list[KeyValueStoreClient] = []
@@ -97,6 +90,26 @@ def __init__(
         self._purged_on_start = False  # Indicates whether a purge was already performed on this instance.
         self._purge_lock = asyncio.Lock()
 
+    @classmethod
+    def from_config(cls, config: Configuration | None = None) -> MemoryStorageClient:
+        """Create a new instance based on the provided configuration.
+
+        All the memory storage client parameters are taken from the configuration object.
+
+        Args:
+            config: The configuration object.
+        """
+        config = config or Configuration.get_global_configuration()
+
+        return cls(
+            write_metadata=config.write_metadata,
+            persist_storage=config.persist_storage,
+            storage_dir=config.storage_dir,
+            default_request_queue_id=config.default_request_queue_id,
+            default_key_value_store_id=config.default_key_value_store_id,
+            default_dataset_id=config.default_dataset_id,
+        )
+
     @property
     def write_metadata(self) -> bool:
         """Whether to write metadata to the storage."""

tests/unit/_memory_storage_client/test_memory_storage_client.py

@@ -17,13 +17,13 @@
 async def test_write_metadata(tmp_path: Path) -> None:
     dataset_name = 'test'
     dataset_no_metadata_name = 'test-no-metadata'
-    ms = MemoryStorageClient(
+    ms = MemoryStorageClient.from_config(
         Configuration(
             crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
             write_metadata=True,
         ),
     )
-    ms_no_metadata = MemoryStorageClient(
+    ms_no_metadata = MemoryStorageClient.from_config(
         Configuration(
             crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
             write_metadata=False,
@@ -48,7 +48,7 @@ async def test_write_metadata(tmp_path: Path) -> None:
     ],
 )
 async def test_persist_storage(persist_storage: bool, tmp_path: Path) -> None:  # noqa: FBT001
-    ms = MemoryStorageClient(
+    ms = MemoryStorageClient.from_config(
         Configuration(
             crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
             persist_storage=persist_storage,
@@ -82,18 +82,20 @@ async def test_persist_storage(persist_storage: bool, tmp_path: Path) -> None:
 
 def test_persist_storage_set_to_false_via_string_env_var(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     monkeypatch.setenv('CRAWLEE_PERSIST_STORAGE', 'false')
-    ms = MemoryStorageClient(Configuration(crawlee_storage_dir=str(tmp_path)))  # type: ignore[call-arg]
+    ms = MemoryStorageClient.from_config(
+        Configuration(crawlee_storage_dir=str(tmp_path)),  # type: ignore[call-arg]
+    )
     assert ms.persist_storage is False
 
 
 def test_persist_storage_set_to_false_via_numeric_env_var(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     monkeypatch.setenv('CRAWLEE_PERSIST_STORAGE', '0')
-    ms = MemoryStorageClient(Configuration(crawlee_storage_dir=str(tmp_path)))  # type: ignore[call-arg]
+    ms = MemoryStorageClient.from_config(Configuration(crawlee_storage_dir=str(tmp_path)))  # type: ignore[call-arg]
     assert ms.persist_storage is False
 
 
 def test_persist_storage_true_via_constructor_arg(tmp_path: Path) -> None:
-    ms = MemoryStorageClient(
+    ms = MemoryStorageClient.from_config(
         Configuration(
             crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
             persist_storage=True,
@@ -104,20 +106,24 @@ def test_persist_storage_true_via_constructor_arg(tmp_path: Path) -> None:
 
 def test_default_write_metadata_behavior(tmp_path: Path) -> None:
     # Default behavior
-    ms = MemoryStorageClient(Configuration(crawlee_storage_dir=str(tmp_path)))  # type: ignore[call-arg]
+    ms = MemoryStorageClient.from_config(
+        Configuration(crawlee_storage_dir=str(tmp_path)),  # type: ignore[call-arg]
+    )
     assert ms.write_metadata is True
 
 
 def test_write_metadata_set_to_false_via_env_var(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     # Test if env var changes write_metadata to False
     monkeypatch.setenv('CRAWLEE_WRITE_METADATA', 'false')
-    ms = MemoryStorageClient(Configuration(crawlee_storage_dir=str(tmp_path)))  # type: ignore[call-arg]
+    ms = MemoryStorageClient.from_config(
+        Configuration(crawlee_storage_dir=str(tmp_path)),  # type: ignore[call-arg]
+    )
     assert ms.write_metadata is False
 
 
 def test_write_metadata_false_via_constructor_arg_overrides_env_var(tmp_path: Path) -> None:
     # Test if constructor arg takes precedence over env var value
-    ms = MemoryStorageClient(
+    ms = MemoryStorageClient.from_config(
         Configuration(
             write_metadata=False,
             crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
@@ -127,7 +133,7 @@ def test_write_metadata_false_via_constructor_arg_overrides_env_var(tmp_path: Pa
 
 
 async def test_purge_datasets(tmp_path: Path) -> None:
-    ms = MemoryStorageClient(
+    ms = MemoryStorageClient.from_config(
         Configuration(
             write_metadata=True,
             crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
@@ -150,7 +156,7 @@ async def test_purge_datasets(tmp_path: Path) -> None:
 
 
 async def test_purge_key_value_stores(tmp_path: Path) -> None:
-    ms = MemoryStorageClient(
+    ms = MemoryStorageClient.from_config(
         Configuration(
             write_metadata=True,
             crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
@@ -185,7 +191,7 @@ async def test_purge_key_value_stores(tmp_path: Path) -> None:
 
 
 async def test_purge_request_queues(tmp_path: Path) -> None:
-    ms = MemoryStorageClient(
+    ms = MemoryStorageClient.from_config(
         Configuration(
             write_metadata=True,
             crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
@@ -207,7 +213,7 @@ async def test_purge_request_queues(tmp_path: Path) -> None:
 
 
 async def test_not_implemented_method(tmp_path: Path) -> None:
-    ms = MemoryStorageClient(
+    ms = MemoryStorageClient.from_config(
         Configuration(
             write_metadata=True,
             crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
@@ -230,21 +236,21 @@ async def test_default_storage_path_used(monkeypatch: pytest.MonkeyPatch) -> Non
     monkeypatch.delenv('CRAWLEE_STORAGE_DIR', raising=False)
 
     # Initialize the service locator with default configuration
-    msc = MemoryStorageClient()
+    msc = MemoryStorageClient.from_config()
     assert msc.storage_dir == './storage'
 
 
 async def test_storage_path_from_env_var_overrides_default(monkeypatch: pytest.MonkeyPatch) -> None:
     # We expect the env var to override the default value
     monkeypatch.setenv('CRAWLEE_STORAGE_DIR', './env_var_storage_dir')
     service_locator.set_configuration(Configuration())
-    ms = MemoryStorageClient()
+    ms = MemoryStorageClient.from_config()
     assert ms.storage_dir == './env_var_storage_dir'
 
 
 async def test_parametrized_storage_path_overrides_env_var() -> None:
     # We expect the parametrized value to be used
-    ms = MemoryStorageClient(
+    ms = MemoryStorageClient.from_config(
         Configuration(crawlee_storage_dir='./parametrized_storage_dir'),  # type: ignore[call-arg]
     )
     assert ms.storage_dir == './parametrized_storage_dir'

tests/unit/conftest.py

@@ -11,6 +11,7 @@
 from yarl import URL
 
 from crawlee import service_locator
+from crawlee.configuration import Configuration
 from crawlee.memory_storage_client import MemoryStorageClient
 from crawlee.proxy_configuration import ProxyInfo
 from crawlee.storages import _creation_management
@@ -164,8 +165,10 @@ async def disabled_proxy(proxy_info: ProxyInfo) -> AsyncGenerator[ProxyInfo, Non
 @pytest.fixture
 def memory_storage_client(tmp_path: Path) -> MemoryStorageClient:
     """A fixture for testing the memory storage client and its resource clients."""
-    return MemoryStorageClient(
-        write_metadata=True,
+    config = Configuration(
         persist_storage=True,
-        storage_dir=str(tmp_path),
+        write_metadata=True,
+        crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
     )
+
+    return MemoryStorageClient.from_config(config)

tests/unit/test_configuration.py

@@ -31,7 +31,7 @@ async def test_storage_not_persisted_when_disabled(tmp_path: Path, httpbin: URL)
         write_metadata=False,
         crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
     )
-    storage_client = MemoryStorageClient(config)
+    storage_client = MemoryStorageClient.from_config(config)
     service_locator.set_storage_client(storage_client)
 
     crawler = HttpCrawler()
@@ -53,7 +53,7 @@ async def test_storage_persisted_when_enabled(tmp_path: Path, httpbin: URL) -> N
         write_metadata=True,
         crawlee_storage_dir=str(tmp_path),  # type: ignore[call-arg]
     )
-    storage_client = MemoryStorageClient(config)
+    storage_client = MemoryStorageClient.from_config(config)
     service_locator.set_storage_client(storage_client)
 
     crawler = HttpCrawler()

tests/unit/test_service_locator.py

@@ -40,7 +40,7 @@ def test_storage_client() -> None:
     default_storage_client = service_locator.get_storage_client()
     assert isinstance(default_storage_client, MemoryStorageClient)
 
-    custom_storage_client = MemoryStorageClient()
+    custom_storage_client = MemoryStorageClient.from_config()
     service_locator.set_storage_client(custom_storage_client)
     storage_client = service_locator.get_storage_client()
     assert storage_client == custom_storage_client