Fix

Author: vdusek

src/apify/storage_clients/_apify/_dataset_client.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import warnings
+from datetime import datetime
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
@@ -65,7 +66,18 @@ def __init__(
     @override
     async def get_metadata(self) -> DatasetMetadata:
         metadata = await self._api_client.get()
-        return DatasetMetadata.model_validate(metadata)
+
+        if metadata is None:
+            raise ValueError('Failed to retrieve dataset metadata.')
+
+        return DatasetMetadata(
+            id=metadata.id,
+            name=metadata.name,
+            created_at=datetime.fromisoformat(metadata.created_at.replace('Z', '+00:00')),
+            modified_at=datetime.fromisoformat(metadata.modified_at.replace('Z', '+00:00')),
+            accessed_at=datetime.fromisoformat(metadata.accessed_at.replace('Z', '+00:00')),
+            item_count=int(metadata.item_count),
+        )
 
     @classmethod
     async def open(

src/apify/storage_clients/_apify/_key_value_store_client.py

@@ -2,6 +2,7 @@
 
 import asyncio
 import warnings
+from datetime import datetime
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
@@ -54,7 +55,18 @@ def __init__(
     @override
     async def get_metadata(self) -> ApifyKeyValueStoreMetadata:
         metadata = await self._api_client.get()
-        return ApifyKeyValueStoreMetadata.model_validate(metadata)
+
+        if metadata is None:
+            raise ValueError('Failed to retrieve dataset metadata.')
+
+        return ApifyKeyValueStoreMetadata(
+            id=metadata.id,
+            name=metadata.name,
+            created_at=datetime.fromisoformat(metadata.created_at.replace('Z', '+00:00')),
+            modified_at=datetime.fromisoformat(metadata.modified_at.replace('Z', '+00:00')),
+            accessed_at=datetime.fromisoformat(metadata.accessed_at.replace('Z', '+00:00')),
+            url_signing_secret_key=metadata.url_signing_secret_key,
+        )
 
     @classmethod
     async def open(

src/apify/storage_clients/_apify/_request_queue_client.py

@@ -78,32 +78,32 @@ async def get_metadata(self) -> ApifyRequestQueueMetadata:
         Returns:
             Request queue metadata with accurate counts and timestamps, combining API data with local estimates.
         """
-        response = await self._api_client.get()
+        metadata = await self._api_client.get()
 
-        if response is None:
+        if metadata is None:
             raise ValueError('Failed to fetch request queue metadata from the API.')
 
-        total_request_count = int(response.total_request_count)
-        handled_request_count = int(response.handled_request_count)
-        pending_request_count = int(response.pending_request_count)
-        created_at = datetime.fromisoformat(response.created_at.replace('Z', '+00:00'))
-        modified_at = datetime.fromisoformat(response.modified_at.replace('Z', '+00:00'))
-        accessed_at = datetime.fromisoformat(response.accessed_at.replace('Z', '+00:00'))
+        total_request_count = int(metadata.total_request_count)
+        handled_request_count = int(metadata.handled_request_count)
+        pending_request_count = int(metadata.pending_request_count)
+        created_at = datetime.fromisoformat(metadata.created_at.replace('Z', '+00:00'))
+        modified_at = datetime.fromisoformat(metadata.modified_at.replace('Z', '+00:00'))
+        accessed_at = datetime.fromisoformat(metadata.accessed_at.replace('Z', '+00:00'))
 
         # Enhance API response with local estimations to account for propagation delays (API data can be delayed
         # by a few seconds, while local estimates are immediately accurate).
         return ApifyRequestQueueMetadata(
-            id=response.id,
-            name=response.name,
+            id=metadata.id,
+            name=metadata.name,
             total_request_count=max(total_request_count, self._implementation.metadata.total_request_count),
             handled_request_count=max(handled_request_count, self._implementation.metadata.handled_request_count),
             pending_request_count=pending_request_count,
             created_at=min(created_at, self._implementation.metadata.created_at),
             modified_at=max(modified_at, self._implementation.metadata.modified_at),
             accessed_at=max(accessed_at, self._implementation.metadata.accessed_at),
-            had_multiple_clients=response.had_multiple_clients or self._implementation.metadata.had_multiple_clients,
+            had_multiple_clients=metadata.had_multiple_clients or self._implementation.metadata.had_multiple_clients,
             stats=RequestQueueStats.model_validate(
-                response.stats.model_dump(by_alias=True) if response.stats else {},
+                metadata.stats.model_dump(by_alias=True) if metadata.stats else {},
                 by_alias=True,
             ),
         )

src/apify/storage_clients/_apify/_request_queue_shared_client.py

@@ -121,18 +121,17 @@ async def add_batch_of_requests(
 
         if new_requests:
             # Prepare requests for API by converting to dictionaries.
-            requests_dict = [
-                request.model_dump(
-                    by_alias=True,
-                )
-                for request in new_requests
-            ]
+            requests_dict = [request.model_dump(by_alias=True) for request in new_requests]
 
             # Send requests to API.
-            api_response = AddRequestsResponse.model_validate(
-                await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront)
+            batch_response = await self._api_client.batch_add_requests(
+                requests=requests_dict,
+                forefront=forefront,
             )
 
+            batch_response_dict = batch_response.model_dump(by_alias=True)
+            api_response = AddRequestsResponse.model_validate(batch_response_dict)
+
             # Add the locally known already present processed requests based on the local cache.
             api_response.processed_requests.extend(already_present_requests)
 
@@ -312,7 +311,8 @@ async def _get_request_by_id(self, request_id: str) -> Request | None:
         if response is None:
             return None
 
-        return Request.model_validate(response)
+        response_dict = response.model_dump(by_alias=True)
+        return Request.model_validate(response_dict)
 
     async def _ensure_head_is_non_empty(self) -> None:
         """Ensure that the queue head has requests if they are available in the queue."""
@@ -473,7 +473,8 @@ async def _list_head(
             # After adding new requests to the forefront, any existing leftover locked request is kept in the end.
             self._queue_head.append(leftover_id)
 
-        return RequestQueueHead.model_validate(list_and_lost_data)
+        list_and_lost_dict = list_and_lost_data.model_dump(by_alias=True)
+        return RequestQueueHead.model_validate(list_and_lost_dict)
 
     def _cache_request(
         self,

src/apify/storage_clients/_apify/_request_queue_single_client.py

@@ -147,22 +147,20 @@ async def add_batch_of_requests(
 
         if new_requests:
             # Prepare requests for API by converting to dictionaries.
-            requests_dict = [
-                request.model_dump(
-                    by_alias=True,
-                )
-                for request in new_requests
-            ]
+            requests_dict = [request.model_dump(by_alias=True) for request in new_requests]
 
             # Send requests to API.
-            api_response = AddRequestsResponse.model_validate(
-                await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront)
-            )
+            batch_response = await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront)
+            batch_response_dict = batch_response.model_dump(by_alias=True)
+            api_response = AddRequestsResponse.model_validate(batch_response_dict)
+
             # Add the locally known already present processed requests based on the local cache.
             api_response.processed_requests.extend(already_present_requests)
+
             # Remove unprocessed requests from the cache
             for unprocessed_request in api_response.unprocessed_requests:
-                self._requests_cache.pop(unique_key_to_request_id(unprocessed_request.unique_key), None)
+                request_id = unique_key_to_request_id(unprocessed_request.unique_key)
+                self._requests_cache.pop(request_id, None)
 
         else:
             api_response = AddRequestsResponse(
@@ -329,7 +327,8 @@ async def _get_request_by_id(self, id: str) -> Request | None:
         if response is None:
             return None
 
-        request = Request.model_validate(response)
+        response_dict = response.model_dump(by_alias=True)
+        request = Request.model_validate(response_dict)
 
         # Updated local caches
         if id in self._requests_in_progress:

tests/unit/conftest.py

@@ -62,6 +62,7 @@ def prepare_test_env(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Callabl
     def _prepare_test_env() -> None:
         if hasattr(apify._actor.Actor, '__wrapped__'):
             delattr(apify._actor.Actor, '__wrapped__')
+
         apify._actor.Actor._is_initialized = False
 
         # Set the environment variable for the local storage directory to the temporary path.