Memory storage clients and their tests

Author: vdusek

src/crawlee/_utils/file.py

@@ -103,6 +103,31 @@ async def json_dumps(obj: Any) -> str:
     return await asyncio.to_thread(json.dumps, obj, ensure_ascii=False, indent=2, default=str)
 
 
+def infer_mime_type(value: Any) -> str:
+    """Infer the MIME content type from the value.
+
+    Args:
+        value: The value to infer the content type from.
+
+    Returns:
+        The inferred MIME content type.
+    """
+    # If the value is bytes (or bytearray), return binary content type.
+    if isinstance(value, (bytes, bytearray)):
+        return 'application/octet-stream'
+
+    # If the value is a dict or list, assume JSON.
+    if isinstance(value, (dict, list)):
+        return 'application/json; charset=utf-8'
+
+    # If the value is a string, assume plain text.
+    if isinstance(value, str):
+        return 'text/plain; charset=utf-8'
+
+    # Default fallback.
+    return 'application/octet-stream'
+
+
 async def export_json_to_stream(
     iterator: AsyncIterator[dict],
     dst: TextIO,

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -714,9 +714,9 @@ async def export_data(
         dst = path.open('w', newline='')
 
         if path.suffix == '.csv':
-            await export_csv_to_stream(dataset.iterate(), dst)
+            await export_csv_to_stream(dataset.iterate_items(), dst)
         elif path.suffix == '.json':
-            await export_json_to_stream(dataset.iterate(), dst)
+            await export_json_to_stream(dataset.iterate_items(), dst)
         else:
             raise ValueError(f'Unsupported file extension: {path.suffix}')
 

src/crawlee/storage_clients/_base/_dataset_client.py

@@ -14,81 +14,74 @@
     from crawlee.storage_clients.models import DatasetItemsListPage
 
 
-# Properties:
-# - id
-# - name
-# - created_at
-# - accessed_at
-# - modified_at
-# - item_count
-
-# Methods:
-# - open
-# - drop
-# - push_data
-# - get_data
-# - iterate
-
-
 @docs_group('Abstract classes')
 class DatasetClient(ABC):
-    """An abstract class for dataset resource clients.
+    """An abstract class for dataset storage clients.
+
+    Dataset clients provide an interface for accessing and manipulating dataset storage. They handle
+    operations like adding and getting dataset items across different storage backends.
 
-    These clients are specific to the type of resource they manage and operate under a designated storage
-    client, like a memory storage client.
+    Storage clients are specific to the type of storage they manage (`Dataset`, `KeyValueStore`,
+    `RequestQueue`), and can operate with various storage systems including memory, file system,
+    databases, and cloud storage solutions.
+
+    This abstract class defines the interface that all specific dataset clients must implement.
     """
 
     @property
     @abstractmethod
     def id(self) -> str:
-        """The ID of the dataset."""
+        """The ID of the dataet, a unique identifier, typically a UUID or similar value."""
 
     @property
     @abstractmethod
     def name(self) -> str | None:
-        """The name of the dataset."""
+        """The optional human-readable name of the dataset."""
 
     @property
     @abstractmethod
     def created_at(self) -> datetime:
-        """The time at which the dataset was created."""
+        """Timestamp when the dataset was first created, remains unchanged."""
 
     @property
     @abstractmethod
     def accessed_at(self) -> datetime:
-        """The time at which the dataset was last accessed."""
+        """Timestamp of last access to the dataset, updated on read or write operations."""
 
     @property
     @abstractmethod
     def modified_at(self) -> datetime:
-        """The time at which the dataset was last modified."""
+        """Timestamp of last modification of the dataset, updated when new data are added."""
 
     @property
     @abstractmethod
     def item_count(self) -> int:
-        """The number of items in the dataset."""
+        """Total count of data items stored in the dataset."""
 
     @classmethod
     @abstractmethod
     async def open(
         cls,
         *,
-        id: str | None,
-        name: str | None,
-        storage_dir: Path,
+        id: str | None = None,
+        name: str | None = None,
+        storage_dir: Path | None = None,
     ) -> DatasetClient:
         """Open existing or create a new dataset client.
 
-        If a dataset with the given name already exists, the appropriate dataset client is returned.
+        If a dataset with the given name or ID already exists, the appropriate dataset client is returned.
         Otherwise, a new dataset is created and client for it is returned.
 
+        The backend method for the `Dataset.open` call.
+
         Args:
-            id: The ID of the dataset.
-            name: The name of the dataset.
-            storage_dir: The path to the storage directory. If the client persists data, it should use this directory.
+            id: The ID of the dataset. If not provided, an ID may be generated.
+            name: The name of the dataset. If not provided a default name may be used.
+            storage_dir: The path to the storage directory. If the client persists data,
+                it should use this directory. May be ignored by non-persistent implementations.
 
         Returns:
-            A dataset client.
+            A dataset client instance.
         """
 
     @abstractmethod
@@ -99,7 +92,7 @@ async def drop(self) -> None:
         """
 
     @abstractmethod
-    async def push_data(self, *, data: list[Any] | dict[str, Any]) -> None:
+    async def push_data(self, data: list[Any] | dict[str, Any]) -> None:
         """Push data to the dataset.
 
         The backend method for the `Dataset.push_data` call.
@@ -121,13 +114,13 @@ async def get_data(
         flatten: list[str] | None = None,
         view: str | None = None,
     ) -> DatasetItemsListPage:
-        """Get data from the dataset.
+        """Get data from the dataset with various filtering options.
 
         The backend method for the `Dataset.get_data` call.
         """
 
     @abstractmethod
-    async def iterate(
+    async def iterate_items(
         self,
         *,
         offset: int = 0,
@@ -140,9 +133,9 @@ async def iterate(
         skip_empty: bool = False,
         skip_hidden: bool = False,
     ) -> AsyncIterator[dict]:
-        """Iterate over the dataset.
+        """Iterate over the dataset items with filtering options.
 
-        The backend method for the `Dataset.iterate` call.
+        The backend method for the `Dataset.iterate_items` call.
         """
         # This syntax is to make mypy properly work with abstract AsyncIterator.
         # https://mypy.readthedocs.io/en/stable/more_types.html#asynchronous-iterators

src/crawlee/storage_clients/_base/_key_value_store_client.py

@@ -10,82 +10,73 @@
     from datetime import datetime
     from pathlib import Path
 
-    from crawlee.storage_clients.models import (
-        KeyValueStoreRecord,
-        KeyValueStoreRecordMetadata,
-    )
-
-# Properties:
-# - id
-# - name
-# - created_at
-# - accessed_at
-# - modified_at
-
-# Methods:
-# - open
-# - drop
-# - get_value
-# - set_value
-# - delete_value
-# - iterate_keys
-# - get_public_url
+    from crawlee.storage_clients.models import KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
 
 @docs_group('Abstract classes')
 class KeyValueStoreClient(ABC):
-    """An abstract class for key-value store (KVS) resource clients.
+    """An abstract class for key-value store (KVS) storage clients.
 
-    These clients are specific to the type of resource they manage and operate under a designated storage
-    client, like a memory storage client.
+    Key-value stores clients provide an interface for accessing and manipulating KVS storage. They handle
+    operations like getting, setting, deleting KVS values across different storage backends.
+
+    Storage clients are specific to the type of storage they manage (`Dataset`, `KeyValueStore`,
+    `RequestQueue`), and can operate with various storage systems including memory, file system,
+    databases, and cloud storage solutions.
+
+    This abstract class defines the interface that all specific KVS clients must implement.
     """
 
     @property
     @abstractmethod
     def id(self) -> str:
-        """The ID of the key-value store."""
+        """The unique identifier of the key-value store (typically a UUID)."""
 
     @property
     @abstractmethod
     def name(self) -> str | None:
-        """The name of the key-value store."""
+        """The optional human-readable name for the KVS."""
 
     @property
     @abstractmethod
     def created_at(self) -> datetime:
-        """The time at which the key-value store was created."""
+        """Timestamp when the KVS was first created, remains unchanged."""
 
     @property
     @abstractmethod
     def accessed_at(self) -> datetime:
-        """The time at which the key-value store was last accessed."""
+        """Timestamp of last access to the KVS, updated on read or write operations."""
 
     @property
     @abstractmethod
     def modified_at(self) -> datetime:
-        """The time at which the key-value store was last modified."""
+        """Timestamp of last modification of the KVS, updated when new data are added, updated or deleted."""
 
     @classmethod
     @abstractmethod
     async def open(
         cls,
         *,
-        id: str | None,
-        name: str | None,
-        storage_dir: Path,
+        id: str | None = None,
+        name: str | None = None,
+        storage_dir: Path | None = None,
     ) -> KeyValueStoreClient:
         """Open existing or create a new key-value store client.
 
-        If a key-value store with the given name already exists, the appropriate key-value store client is returned.
-        Otherwise, a new key-value store is created and client for it is returned.
+        If a key-value store with the given name or ID already exists, the appropriate
+        key-value store client is returned. Otherwise, a new key-value store is created
+        and a client for it is returned.
+
+        The backend method for the `KeyValueStoreClient.open` call.
 
         Args:
-            id: The ID of the key-value store.
-            name: The name of the key-value store.
-            storage_dir: The path to the storage directory. If the client persists data, it should use this directory.
+            id: The ID of the key-value store. If not provided, an ID may be generated.
+            name: The name of the key-value store. If not provided a default name may be used.
+            storage_dir: The path to the storage directory. If the client persists data,
+                it should use this directory. May be ignored by non-persistent implementations.
 
         Returns:
-            A key-value store client.
+            A key-value store client instance.
         """
 
     @abstractmethod