inherit from Storage class and RQ init

Author: vdusek

src/crawlee/storage_clients/_base/_request_queue_client.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 from abc import ABC, abstractmethod
+from datetime import datetime
 from typing import TYPE_CHECKING
 
 from crawlee._utils.docs import docs_group
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
+    from datetime import datetime
 
     from crawlee.storage_clients.models import (
         BatchRequestsOperationResponse,
@@ -15,7 +17,6 @@
         Request,
         RequestQueueHead,
         RequestQueueHeadWithLocks,
-        RequestQueueMetadata,
     )
 
 
@@ -27,17 +28,67 @@ class RequestQueueClient(ABC):
     client, like a memory storage client.
     """
 
+    @property
     @abstractmethod
-    async def get(self) -> RequestQueueMetadata | None:
-        """Get metadata about the request queue being managed by this client.
+    def id(self) -> str:
+        """The ID of the dataset."""
 
-        Returns:
-            An object containing the request queue's details, or None if the request queue does not exist.
-        """
+    @property
+    @abstractmethod
+    def name(self) -> str | None:
+        """The name of the dataset."""
+
+    @property
+    @abstractmethod
+    def created_at(self) -> datetime:
+        """The time at which the dataset was created."""
+
+    @property
+    @abstractmethod
+    def accessed_at(self) -> datetime:
+        """The time at which the dataset was last accessed."""
+
+    @property
+    @abstractmethod
+    def modified_at(self) -> datetime:
+        """The time at which the dataset was last modified."""
 
+    @property
     @abstractmethod
-    async def delete(self) -> None:
-        """Permanently delete the request queue managed by this client."""
+    def had_multiple_clients(self) -> bool:
+        """TODO."""
+
+    @property
+    @abstractmethod
+    def handled_request_count(self) -> int:
+        """TODO."""
+
+    @property
+    @abstractmethod
+    def pending_request_count(self) -> int:
+        """TODO."""
+
+    @property
+    @abstractmethod
+    def stats(self) -> dict:
+        """TODO."""
+
+    @property
+    @abstractmethod
+    def total_request_count(self) -> int:
+        """TODO."""
+
+    @property
+    @abstractmethod
+    def resource_directory(self) -> str:
+        """TODO."""
+
+    @abstractmethod
+    async def drop(self) -> None:
+        """Drop the whole request queue and remove all its values.
+
+        The backend method for the `RequestQueue.drop` call.
+        """
 
     @abstractmethod
     async def list_head(self, *, limit: int | None = None) -> RequestQueueHead:

src/crawlee/storages/_base.py

@@ -6,7 +6,7 @@
 if TYPE_CHECKING:
     from crawlee.configuration import Configuration
     from crawlee.storage_clients._base import StorageClient
-    from crawlee.storage_clients.models import StorageMetadata
+    from crawlee.storage_clients.models import DatasetMetadata, KeyValueStoreMetadata, RequestQueueMetadata
 
 
 class Storage(ABC):
@@ -24,13 +24,8 @@ def name(self) -> str | None:
 
     @property
     @abstractmethod
-    def storage_object(self) -> StorageMetadata:
-        """Get the full storage object."""
-
-    @storage_object.setter
-    @abstractmethod
-    def storage_object(self, storage_object: StorageMetadata) -> None:
-        """Set the full storage object."""
+    def metadata(self) -> DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata:
+        """Get the storage metadata."""
 
     @classmethod
     @abstractmethod

src/crawlee/storages/_creation_management.py

@@ -5,11 +5,14 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Literal
 
+from typing_extensions import override
+
 from crawlee import service_locator
 from crawlee._utils.docs import docs_group
 from crawlee._utils.file import export_csv_to_stream, export_json_to_stream
 from crawlee.storage_clients.models import DatasetMetadata
 
+from ._base import Storage
 from ._key_value_store import KeyValueStore
 
 if TYPE_CHECKING:
@@ -28,7 +31,6 @@
 logger = logging.getLogger(__name__)
 
 # TODO:
-# - inherit from storage class
 # - caching / memoization of both datasets & dataset clients
 
 # Properties:
@@ -56,7 +58,7 @@
 
 
 @docs_group('Classes')
-class Dataset:
+class Dataset(Storage):
     """Dataset is an append-only structured storage, ideal for tabular data similar to database tables.
 
     The `Dataset` class is designed to store structured data, where each entry (row) maintains consistent attributes
@@ -98,14 +100,17 @@ def __init__(self, client: DatasetClient) -> None:
         """
         self._client = client
 
+    @override
     @property
     def id(self) -> str:
         return self._client.id
 
+    @override
     @property
     def name(self) -> str | None:
         return self._client.name
 
+    @override
     @property
     def metadata(self) -> DatasetMetadata:
         return DatasetMetadata(
@@ -117,6 +122,7 @@ def metadata(self) -> DatasetMetadata:
             item_count=self._client.item_count,
         )
 
+    @override
     @classmethod
     async def open(
         cls,
@@ -145,6 +151,7 @@ async def open(
 
         return cls(client)
 
+    @override
     async def drop(self) -> None:
         await self._client.drop()