push data type

Author: vdusek

src/crawlee/_types.py

@@ -188,7 +188,7 @@ class PushDataKwargs(TypedDict):
 
 
 class PushDataFunctionCall(PushDataKwargs):
-    data: JsonSerializable
+    data: list[dict[str, Any]] | dict[str, Any]
     dataset_id: str | None
     dataset_name: str | None
 
@@ -269,7 +269,7 @@ async def add_requests(
 
     async def push_data(
         self,
-        data: JsonSerializable,
+        data: list[dict[str, Any]] | dict[str, Any],
         dataset_id: str | None = None,
         dataset_name: str | None = None,
         **kwargs: Unpack[PushDataKwargs],
@@ -514,7 +514,7 @@ class PushDataFunction(Protocol):
 
     def __call__(
         self,
-        data: JsonSerializable,
+        data: list[dict[str, Any]] | dict[str, Any],
         dataset_id: str | None = None,
         dataset_name: str | None = None,
         **kwargs: Unpack[PushDataKwargs],

src/crawlee/crawlers/_basic/_basic_crawler.py

@@ -758,7 +758,7 @@ async def export_data(
 
     async def _push_data(
         self,
-        data: JsonSerializable,
+        data: list[dict[str, Any]] | dict[str, Any],
         dataset_id: str | None = None,
         dataset_name: str | None = None,
         **kwargs: Unpack[PushDataKwargs],

src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -216,7 +216,7 @@ async def purge(self) -> None:
             )
 
     @override
-    async def push_data(self, data: list[Any] | dict[str, Any]) -> None:
+    async def push_data(self, data: list[dict[str, Any]] | dict[str, Any]) -> None:
         new_item_count = self.metadata.item_count
 
         # If data is a list, push each item individually.

src/crawlee/storage_clients/_file_system/_request_queue_client.py

@@ -171,7 +171,7 @@ async def open(
             metadata_path = rq_path / METADATA_FILENAME
 
             # If the RQ directory exists, reconstruct the client from the metadata file.
-            if rq_path.exists() and not configuration.purge_on_start:
+            if rq_path.exists():
                 # If metadata file is missing, raise an error.
                 if not metadata_path.exists():
                     raise ValueError(f'Metadata file not found for request queue "{name}"')
@@ -204,10 +204,6 @@ async def open(
 
             # Otherwise, create a new dataset client.
             else:
-                # If purge_on_start is true and the directory exists, remove it
-                if configuration.purge_on_start and rq_path.exists():
-                    await asyncio.to_thread(shutil.rmtree, rq_path)
-
                 now = datetime.now(timezone.utc)
                 client = cls(
                     id=crypto_random_object_id(),

src/crawlee/storage_clients/_memory/_dataset_client.py

@@ -109,7 +109,7 @@ async def purge(self) -> None:
         )
 
     @override
-    async def push_data(self, data: list[Any] | dict[str, Any]) -> None:
+    async def push_data(self, data: list[dict[str, Any]] | dict[str, Any]) -> None:
         new_item_count = self.metadata.item_count
 
         if isinstance(data, list):

src/crawlee/storage_clients/_memory/_request_queue_client.py

@@ -2,7 +2,7 @@
 
 from datetime import datetime, timezone
 from logging import getLogger
-from typing import TYPE_CHECKING, ClassVar
+from typing import TYPE_CHECKING
 
 from typing_extensions import override
 

src/crawlee/storages/_dataset.py

@@ -158,7 +158,7 @@ async def drop(self) -> None:
     async def purge(self) -> None:
         await self._client.purge()
 
-    async def push_data(self, data: list[Any] | dict[str, Any]) -> None:
+    async def push_data(self, data: list[dict[str, Any]] | dict[str, Any]) -> None:
         """Store an object or an array of objects to the dataset.
 
         The size of the data is limited by the receiving API and therefore `push_data()` will only

tests/unit/crawlers/_basic/test_basic_crawler.py

@@ -589,7 +589,7 @@ async def test_context_push_and_get_data_handler_error() -> None:
 
     @crawler.router.default_handler
     async def handler(context: BasicCrawlingContext) -> None:
-        await context.push_data('{"b": 2}')
+        await context.push_data({'b': 2})
         raise RuntimeError('Watch me crash')
 
     stats = await crawler.run(['https://a.com'])

tests/unit/crawlers/_http/test_http_crawler.py

@@ -291,7 +291,6 @@ async def request_handler(context: HttpCrawlingContext) -> None:
     assert responses[0]['data'] == '', 'Response raw data should be empty when only form data is sent.'
 
 
-# @pytest.mark.skip(reason='TODO: broken, freezing')
 async def test_sending_payload_as_json(http_client: HttpClient, server_url: URL) -> None:
     crawler = HttpCrawler(http_client=http_client)
     responses = []

tests/unit/storages/test_dataset.py

@@ -14,6 +14,7 @@
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
     from pathlib import Path
+    from typing import Any
 
     from crawlee.storage_clients import StorageClient
 
@@ -328,7 +329,7 @@ async def test_list_items(dataset: Dataset) -> None:
 async def test_list_items_with_options(dataset: Dataset) -> None:
     """Test that list_items respects filtering options."""
     # Add some items
-    items = [
+    items: list[dict[str, Any]] = [
         {'id': 1, 'name': 'Item 1'},
         {'id': 2, 'name': 'Item 2'},
         {'id': 3},  # Item with missing 'name' field