update metadata fixes

Author: vdusek

src/apify/storage_clients/__init__.py

@@ -1,7 +1,7 @@
 from crawlee.storage_clients import MemoryStorageClient
 
 from ._apify import ApifyStorageClient
-from ._file_system import FileSystemStorageClient
+from ._file_system import ApifyFileSystemStorageClient as FileSystemStorageClient
 
 __all__ = [
     'ApifyStorageClient',

src/apify/storage_clients/_apify/_key_value_store_client.py

@@ -149,9 +149,7 @@ async def drop(self) -> None:
     @override
     async def get_value(self, key: str) -> KeyValueStoreRecord | None:
         response = await self._api_client.get_record(key)
-        record = KeyValueStoreRecord.model_validate(response) if response else None
-        await self._update_metadata()
-        return record
+        return KeyValueStoreRecord.model_validate(response) if response else None
 
     @override
     async def set_value(self, key: str, value: Any, content_type: str | None = None) -> None:
@@ -161,13 +159,11 @@ async def set_value(self, key: str, value: Any, content_type: str | None = None)
                 value=value,
                 content_type=content_type,
             )
-            await self._update_metadata()
 
     @override
     async def delete_value(self, key: str) -> None:
         async with self._lock:
             await self._api_client.delete_record(key=key)
-            await self._update_metadata()
 
     @override
     async def iterate_keys(
@@ -202,8 +198,6 @@ async def iterate_keys(
 
             exclusive_start_key = list_key_page.next_exclusive_start_key
 
-        await self._update_metadata()
-
     @override
     async def record_exists(self, key: str) -> bool:
         return await self._api_client.record_exists(key=key)
@@ -231,8 +225,3 @@ async def get_public_url(self, key: str) -> str:
                 public_url = public_url.with_query(signature=create_hmac_signature(url_signing_secret_key, key))
 
         return str(public_url)
-
-    async def _update_metadata(self) -> None:
-        """Update the key-value store metadata with current information."""
-        metadata = await self._api_client.get()
-        self._metadata = KeyValueStoreMetadata.model_validate(metadata)

src/apify/storage_clients/_apify/_request_queue_client.py

@@ -202,9 +202,6 @@ async def add_batch_of_requests(
         # Send requests to API
         response = await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront)
 
-        # Update metadata after adding requests
-        await self._update_metadata()
-
         return AddRequestsResponse.model_validate(response)
 
     @override
@@ -218,7 +215,6 @@ async def get_request(self, request_id: str) -> Request | None:
             The request or None if not found.
         """
         response = await self._api_client.get_request(request_id)
-        await self._update_metadata()
 
         if response is None:
             return None
@@ -295,9 +291,6 @@ async def mark_request_as_handled(self, request: Request) -> ProcessedRequest |
                 forefront=False,
                 hydrated_request=request,
             )
-
-            # Update metadata after marking request as handled
-            await self._update_metadata()
         except Exception as exc:
             logger.debug(f'Error marking request {request.id} as handled: {exc!s}')
             return None
@@ -346,9 +339,6 @@ async def reclaim_request(
                 await self._delete_request_lock(request.id, forefront=forefront)
             except Exception as err:
                 logger.debug(f'Failed to delete request lock for request {request.id}', exc_info=err)
-
-            # Update metadata after reclaiming request
-            await self._update_metadata()
         except Exception as exc:
             logger.debug(f'Error reclaiming request {request.id}: {exc!s}')
             return None
@@ -648,8 +638,3 @@ def _cache_request(
             lock_expires_at=None,
             forefront=forefront,
         )
-
-    async def _update_metadata(self) -> None:
-        """Update the request queue metadata with current information."""
-        metadata = await self._api_client.get()
-        self._metadata = RequestQueueMetadata.model_validate(metadata)

src/apify/storage_clients/_file_system/__init__.py

@@ -1 +1,2 @@
-from ._storage_client import ApifyFileSystemStorageClient as FileSystemStorageClient
+from ._key_value_store_client import ApifyFileSystemKeyValueStoreClient
+from ._storage_client import ApifyFileSystemStorageClient

tests/unit/storage_clients/test_file_system.py

@@ -5,27 +5,25 @@
 from crawlee._consts import METADATA_FILENAME
 
 from apify import Configuration
-from apify.storage_clients._file_system._key_value_store_client import ApifyFileSystemKeyValueStoreClient
+from apify.storage_clients._file_system import ApifyFileSystemKeyValueStoreClient
 
 
 async def test_purge_preserves_input_file_and_metadata() -> None:
     """Test that purge() preserves INPUT.json and metadata files but removes other files."""
     # Get the global configuration (storage directory is set by test fixtures)
-    config = Configuration.get_global_configuration()
+    configuration = Configuration.get_global_configuration()
 
-    # Create the key-value store client
-    kvs_client = await ApifyFileSystemKeyValueStoreClient.open(
+    kvs_storage_client = await ApifyFileSystemKeyValueStoreClient.open(
         id=None,
         name='test-kvs',
-        configuration=config,
+        configuration=configuration,
     )
 
     # Create some test files in the KVS directory
-    kvs_path = kvs_client.path_to_kvs
+    kvs_path = kvs_storage_client.path_to_kvs
 
     # Create various files
-    kvs_input_filename = f'{config.input_key}.json'
-    input_file = kvs_path / kvs_input_filename
+    input_file = kvs_path / f'{configuration.input_key}.json'
     metadata_file = kvs_path / METADATA_FILENAME
     regular_file1 = kvs_path / 'regular_file1.json'
     regular_file2 = kvs_path / 'another_file.txt'
@@ -42,8 +40,10 @@ async def test_purge_preserves_input_file_and_metadata() -> None:
     assert regular_file2.exists()
 
     # Purge the key-value store
-    await kvs_client.purge()  # Verify INPUT.json and metadata are preserved
-    assert input_file.exists(), f'{kvs_input_filename} should be preserved during purge'
+    await kvs_storage_client.purge()
+
+    # Verify INPUT.json and metadata are preserved
+    assert input_file.exists(), f'{configuration.input_key} should be preserved during purge'
     assert metadata_file.exists(), f'{METADATA_FILENAME} should be preserved during purge'
 
     # Verify other files are deleted