purge only unnamed storages

Author: vdusek

src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -47,14 +47,17 @@ class FileSystemDatasetClient(DatasetClient):
     _STORAGE_SUBDIR = 'datasets'
     """The name of the subdirectory where datasets are stored."""
 
+    _STORAGE_SUBSUBDIR_DEFAULT = 'default'
+    """The name of the subdirectory for the default dataset."""
+
     _ITEM_FILENAME_DIGITS = 9
     """Number of digits used for the dataset item file names (e.g., 000000019.json)."""
 
     def __init__(
         self,
         *,
         id: str,
-        name: str,
+        name: str | None,
         created_at: datetime,
         accessed_at: datetime,
         modified_at: datetime,
@@ -88,6 +91,9 @@ def metadata(self) -> DatasetMetadata:
     @property
     def path_to_dataset(self) -> Path:
         """The full path to the dataset directory."""
+        if self.metadata.name is None:
+            return self._storage_dir / self._STORAGE_SUBDIR / self._STORAGE_SUBSUBDIR_DEFAULT
+
         return self._storage_dir / self._STORAGE_SUBDIR / self.metadata.name
 
     @property
@@ -149,8 +155,9 @@ async def open(
 
         # Get a new instance by name.
         else:
-            name = name or configuration.default_dataset_id
-            dataset_path = dataset_base_path / name
+            dataset_path = (
+                dataset_base_path / cls._STORAGE_SUBSUBDIR_DEFAULT if name is None else dataset_base_path / name
+            )
             metadata_path = dataset_path / METADATA_FILENAME
 
             # If the dataset directory exists, reconstruct the client from the metadata file.

src/crawlee/storage_clients/_file_system/_key_value_store_client.py

@@ -49,11 +49,14 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
     _STORAGE_SUBDIR = 'key_value_stores'
     """The name of the subdirectory where key-value stores are stored."""
 
+    _STORAGE_SUBSUBDIR_DEFAULT = 'default'
+    """The name of the subdirectory for the default key-value store."""
+
     def __init__(
         self,
         *,
         id: str,
-        name: str,
+        name: str | None,
         created_at: datetime,
         accessed_at: datetime,
         modified_at: datetime,
@@ -85,6 +88,9 @@ def metadata(self) -> KeyValueStoreMetadata:
     @property
     def path_to_kvs(self) -> Path:
         """The full path to the key-value store directory."""
+        if self.metadata.name is None:
+            return self._storage_dir / self._STORAGE_SUBDIR / self._STORAGE_SUBSUBDIR_DEFAULT
+
         return self._storage_dir / self._STORAGE_SUBDIR / self.metadata.name
 
     @property
@@ -145,9 +151,9 @@ async def open(
 
         # Get a new instance by name.
         else:
-            name = name or configuration.default_key_value_store_id
-
-            kvs_path = storage_dir / cls._STORAGE_SUBDIR / name
+            kvs_path = (
+                kvs_base_path / cls._STORAGE_SUBSUBDIR_DEFAULT if name is None else kvs_base_path / name
+            )
             metadata_path = kvs_path / METADATA_FILENAME
 
             # If the key-value store directory exists, reconstruct the client from the metadata file.

src/crawlee/storage_clients/_file_system/_request_queue_client.py

@@ -48,11 +48,14 @@ class FileSystemRequestQueueClient(RequestQueueClient):
     _STORAGE_SUBDIR = 'request_queues'
     """The name of the subdirectory where request queues are stored."""
 
+    _STORAGE_SUBSUBDIR_DEFAULT = 'default'
+    """The name of the subdirectory for the default request queue."""
+
     def __init__(
         self,
         *,
         id: str,
-        name: str,
+        name: str | None,
         created_at: datetime,
         accessed_at: datetime,
         modified_at: datetime,
@@ -100,6 +103,9 @@ def metadata(self) -> RequestQueueMetadata:
     @property
     def path_to_rq(self) -> Path:
         """The full path to the request queue directory."""
+        if self.metadata.name is None:
+            return self._storage_dir / self._STORAGE_SUBDIR / self._STORAGE_SUBSUBDIR_DEFAULT
+
         return self._storage_dir / self._STORAGE_SUBDIR / self.metadata.name
 
     @property
@@ -165,9 +171,9 @@ async def open(
 
         # Get a new instance by name.
         else:
-            name = name or configuration.default_request_queue_id
-
-            rq_path = storage_dir / cls._STORAGE_SUBDIR / name
+            rq_path = (
+                rq_base_path / cls._STORAGE_SUBSUBDIR_DEFAULT if name is None else rq_base_path / name
+            )
             metadata_path = rq_path / METADATA_FILENAME
 
             # If the RQ directory exists, reconstruct the client from the metadata file.

src/crawlee/storage_clients/_file_system/_storage_client.py

@@ -24,7 +24,7 @@ async def open_dataset_client(
         configuration = configuration or Configuration.get_global_configuration()
         client = await FileSystemDatasetClient.open(id=id, name=name, configuration=configuration)
 
-        if configuration.purge_on_start:
+        if configuration.purge_on_start and client.metadata.name is None:
             await client.purge()
 
         return client
@@ -40,7 +40,7 @@ async def open_key_value_store_client(
         configuration = configuration or Configuration.get_global_configuration()
         client = await FileSystemKeyValueStoreClient.open(id=id, name=name, configuration=configuration)
 
-        if configuration.purge_on_start:
+        if configuration.purge_on_start and client.metadata.name is None:
             await client.purge()
 
         return client
@@ -56,7 +56,7 @@ async def open_request_queue_client(
         configuration = configuration or Configuration.get_global_configuration()
         client = await FileSystemRequestQueueClient.open(id=id, name=name, configuration=configuration)
 
-        if configuration.purge_on_start:
+        if configuration.purge_on_start and client.metadata.name is None:
             await client.purge()
 
         return client

src/crawlee/storage_clients/_memory/_dataset_client.py

@@ -35,7 +35,7 @@ def __init__(
         self,
         *,
         id: str,
-        name: str,
+        name: str | None,
         created_at: datetime,
         accessed_at: datetime,
         modified_at: datetime,
@@ -71,8 +71,6 @@ async def open(
         name: str | None,
         configuration: Configuration,
     ) -> MemoryDatasetClient:
-        name = name or configuration.default_dataset_id
-
         # Otherwise create a new dataset
         dataset_id = id or crypto_random_object_id()
         now = datetime.now(timezone.utc)

src/crawlee/storage_clients/_memory/_key_value_store_client.py

@@ -36,7 +36,7 @@ def __init__(
         self,
         *,
         id: str,
-        name: str,
+        name: str | None,
         created_at: datetime,
         accessed_at: datetime,
         modified_at: datetime,
@@ -70,8 +70,6 @@ async def open(
         name: str | None,
         configuration: Configuration,
     ) -> MemoryKeyValueStoreClient:
-        name = name or configuration.default_key_value_store_id
-
         # Otherwise create a new key-value store
         store_id = id or crypto_random_object_id()
         now = datetime.now(timezone.utc)

src/crawlee/storage_clients/_memory/_request_queue_client.py

@@ -39,7 +39,7 @@ def __init__(
         self,
         *,
         id: str,
-        name: str,
+        name: str | None,
         created_at: datetime,
         accessed_at: datetime,
         modified_at: datetime,
@@ -86,8 +86,6 @@ async def open(
         name: str | None,
         configuration: Configuration,
     ) -> MemoryRequestQueueClient:
-        name = name or configuration.default_request_queue_id
-
         # Otherwise create a new queue
         queue_id = id or crypto_random_object_id()
         now = datetime.now(timezone.utc)

src/crawlee/storage_clients/_memory/_storage_client.py

@@ -24,7 +24,7 @@ async def open_dataset_client(
         configuration = configuration or Configuration.get_global_configuration()
         client = await MemoryDatasetClient.open(id=id, name=name, configuration=configuration)
 
-        if configuration.purge_on_start:
+        if configuration.purge_on_start and client.metadata.name is None:
             await client.purge()
 
         return client
@@ -40,7 +40,7 @@ async def open_key_value_store_client(
         configuration = configuration or Configuration.get_global_configuration()
         client = await MemoryKeyValueStoreClient.open(id=id, name=name, configuration=configuration)
 
-        if configuration.purge_on_start:
+        if configuration.purge_on_start and client.metadata.name is None:
             await client.purge()
 
         return client
@@ -56,7 +56,7 @@ async def open_request_queue_client(
         configuration = configuration or Configuration.get_global_configuration()
         client = await MemoryRequestQueueClient.open(id=id, name=name, configuration=configuration)
 
-        if configuration.purge_on_start:
+        if configuration.purge_on_start and client.metadata.name is None:
             await client.purge()
 
         return client

src/crawlee/storage_clients/models.py

@@ -28,7 +28,7 @@ class StorageMetadata(BaseModel):
     id: Annotated[str, Field(alias='id')]
     """The unique identifier of the storage."""
 
-    name: Annotated[str, Field(alias='name', default='default')]
+    name: Annotated[str | None, Field(alias='name', default=None)]
     """The name of the storage."""
 
     accessed_at: Annotated[datetime, Field(alias='accessedAt')]

tests/unit/storage_clients/_file_system/test_fs_dataset_client.py

@@ -70,7 +70,6 @@ async def test_dataset_client_purge_on_start(configuration: Configuration) -> No
 
     # Create dataset and add data
     dataset_client1 = await FileSystemStorageClient().open_dataset_client(
-        name='test-purge-dataset',
         configuration=configuration,
     )
     await dataset_client1.push_data({'item': 'initial data'})
@@ -81,7 +80,6 @@ async def test_dataset_client_purge_on_start(configuration: Configuration) -> No
 
     # Reopen
     dataset_client2 = await FileSystemStorageClient().open_dataset_client(
-        name='test-purge-dataset',
         configuration=configuration,
     )