First step - get rid of collections and unused methods

Author: vdusek

src/crawlee/storage_clients/_base/__init__.py

@@ -1,11 +1,8 @@
 from ._dataset_client import DatasetClient
-from ._dataset_collection_client import DatasetCollectionClient
 from ._key_value_store_client import KeyValueStoreClient
-from ._key_value_store_collection_client import KeyValueStoreCollectionClient
 from ._request_queue_client import RequestQueueClient
-from ._request_queue_collection_client import RequestQueueCollectionClient
 from ._storage_client import StorageClient
-from ._types import ResourceClient, ResourceCollectionClient
+from ._types import ResourceClient
 
 __all__ = [
     'DatasetClient',
@@ -15,6 +12,5 @@
     'RequestQueueClient',
     'RequestQueueCollectionClient',
     'ResourceClient',
-    'ResourceCollectionClient',
     'StorageClient',
 ]

src/crawlee/storage_clients/_base/_dataset_client.py

@@ -35,24 +35,17 @@ async def get(self) -> DatasetMetadata | None:
         """
 
     @abstractmethod
-    async def update(
-        self,
-        *,
-        name: str | None = None,
-    ) -> DatasetMetadata:
-        """Update the dataset metadata.
+    async def delete(self) -> None:
+        """Permanently delete the dataset managed by this client."""
 
-        Args:
-            name: New new name for the dataset.
+    @abstractmethod
+    async def push_items(self, items: JsonSerializable) -> None:
+        """Push items to the dataset.
 
-        Returns:
-            An object reflecting the updated dataset metadata.
+        Args:
+            items: The items which to push in the dataset. They must be JSON serializable.
         """
 
-    @abstractmethod
-    async def delete(self) -> None:
-        """Permanently delete the dataset managed by this client."""
-
     @abstractmethod
     async def list_items(
         self,
@@ -221,11 +214,3 @@ async def stream_items(
         Yields:
             The dataset items in a streaming response.
         """
-
-    @abstractmethod
-    async def push_items(self, items: JsonSerializable) -> None:
-        """Push items to the dataset.
-
-        Args:
-            items: The items which to push in the dataset. They must be JSON serializable.
-        """

src/crawlee/storage_clients/_base/_dataset_collection_client.py

@@ -29,21 +29,6 @@ async def get(self) -> KeyValueStoreMetadata | None:
             An object containing the key-value store's details, or None if the key-value store does not exist.
         """
 
-    @abstractmethod
-    async def update(
-        self,
-        *,
-        name: str | None = None,
-    ) -> KeyValueStoreMetadata:
-        """Update the key-value store metadata.
-
-        Args:
-            name: New new name for the key-value store.
-
-        Returns:
-            An object reflecting the updated key-value store metadata.
-        """
-
     @abstractmethod
     async def delete(self) -> None:
         """Permanently delete the key-value store managed by this client."""

src/crawlee/storage_clients/_base/_key_value_store_client.py

@@ -35,21 +35,6 @@ async def get(self) -> RequestQueueMetadata | None:
             An object containing the request queue's details, or None if the request queue does not exist.
         """
 
-    @abstractmethod
-    async def update(
-        self,
-        *,
-        name: str | None = None,
-    ) -> RequestQueueMetadata:
-        """Update the request queue metadata.
-
-        Args:
-            name: New new name for the request queue.
-
-        Returns:
-            An object reflecting the updated request queue metadata.
-        """
-
     @abstractmethod
     async def delete(self) -> None:
         """Permanently delete the request queue managed by this client."""
@@ -150,6 +135,14 @@ async def delete_request(self, request_id: str) -> None:
             request_id: ID of the request to delete.
         """
 
+    @abstractmethod
+    async def batch_delete_requests(self, requests: list[Request]) -> BatchRequestsOperationResponse:
+        """Delete given requests from the queue.
+
+        Args:
+            requests: The requests to delete from the queue.
+        """
+
     @abstractmethod
     async def prolong_request_lock(
         self,
@@ -179,11 +172,3 @@ async def delete_request_lock(
             request_id: ID of the request to delete the lock.
             forefront: Whether to put the request in the beginning or the end of the queue after the lock is deleted.
         """
-
-    @abstractmethod
-    async def batch_delete_requests(self, requests: list[Request]) -> BatchRequestsOperationResponse:
-        """Delete given requests from the queue.
-
-        Args:
-            requests: The requests to delete from the queue.
-        """

src/crawlee/storage_clients/_base/_key_value_store_collection_client.py

@@ -9,11 +9,8 @@
 
 if TYPE_CHECKING:
     from ._dataset_client import DatasetClient
-    from ._dataset_collection_client import DatasetCollectionClient
     from ._key_value_store_client import KeyValueStoreClient
-    from ._key_value_store_collection_client import KeyValueStoreCollectionClient
     from ._request_queue_client import RequestQueueClient
-    from ._request_queue_collection_client import RequestQueueCollectionClient
 
 
 @docs_group('Abstract classes')
@@ -28,26 +25,14 @@ class StorageClient(ABC):
     def dataset(self, id: str) -> DatasetClient:
         """Get a subclient for a specific dataset by its ID."""
 
-    @abstractmethod
-    def datasets(self) -> DatasetCollectionClient:
-        """Get a subclient for dataset collection operations."""
-
     @abstractmethod
     def key_value_store(self, id: str) -> KeyValueStoreClient:
         """Get a subclient for a specific key-value store by its ID."""
 
-    @abstractmethod
-    def key_value_stores(self) -> KeyValueStoreCollectionClient:
-        """Get a subclient for key-value store collection operations."""
-
     @abstractmethod
     def request_queue(self, id: str) -> RequestQueueClient:
         """Get a subclient for a specific request queue by its ID."""
 
-    @abstractmethod
-    def request_queues(self) -> RequestQueueCollectionClient:
-        """Get a subclient for request queue collection operations."""
-
     @abstractmethod
     async def purge_on_start(self) -> None:
         """Perform a purge of the default storages.

src/crawlee/storage_clients/_base/_request_queue_client.py

@@ -3,20 +3,11 @@
 from typing import Union
 
 from ._dataset_client import DatasetClient
-from ._dataset_collection_client import DatasetCollectionClient
 from ._key_value_store_client import KeyValueStoreClient
-from ._key_value_store_collection_client import KeyValueStoreCollectionClient
 from ._request_queue_client import RequestQueueClient
-from ._request_queue_collection_client import RequestQueueCollectionClient
 
 ResourceClient = Union[
     DatasetClient,
     KeyValueStoreClient,
     RequestQueueClient,
 ]
-
-ResourceCollectionClient = Union[
-    DatasetCollectionClient,
-    KeyValueStoreCollectionClient,
-    RequestQueueCollectionClient,
-]

src/crawlee/storage_clients/_base/_request_queue_collection_client.py

@@ -1,17 +1,11 @@
 from ._dataset_client import DatasetClient
-from ._dataset_collection_client import DatasetCollectionClient
 from ._key_value_store_client import KeyValueStoreClient
-from ._key_value_store_collection_client import KeyValueStoreCollectionClient
 from ._memory_storage_client import MemoryStorageClient
 from ._request_queue_client import RequestQueueClient
-from ._request_queue_collection_client import RequestQueueCollectionClient
 
 __all__ = [
     'DatasetClient',
-    'DatasetCollectionClient',
     'KeyValueStoreClient',
-    'KeyValueStoreCollectionClient',
     'MemoryStorageClient',
     'RequestQueueClient',
-    'RequestQueueCollectionClient',
 ]