add async metadata getters

Author: vdusek

docs/03_concepts/code/conditional_actor_charge.py

@@ -6,7 +6,8 @@ async def main() -> None:
         # Check the dataset because there might already be items
         # if the run migrated or was restarted
         default_dataset = await Actor.open_dataset()
-        charged_items = default_dataset.metadata.item_count
+        metadata = await default_dataset.get_metadata()
+        charged_items = metadata.item_count
 
         # highlight-start
         if Actor.get_charging_manager().get_pricing_info().is_pay_per_event:

pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
     "apify-client>=1.12.0",
     "apify-shared>=1.3.0",
     "cachetools>=5.5.0",
-    "crawlee@git+https://github.com/apify/crawlee-python.git@new-storage-clients",
+    "crawlee@git+https://github.com/apify/crawlee-python.git@9dfac4b8afb8027979d85947f0db303f384b7158",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
     # TODO: ensure compatibility with the latest version of lazy-object-proxy

src/apify/storage_clients/_apify/_dataset_client.py

@@ -37,7 +37,6 @@ class ApifyDatasetClient(DatasetClient):
     def __init__(
         self,
         *,
-        metadata: DatasetMetadata,
         api_client: DatasetClientAsync,
         api_public_base_url: str,
         lock: asyncio.Lock,
@@ -46,8 +45,6 @@ def __init__(
 
         Preferably use the `ApifyDatasetClient.open` class method to create a new instance.
         """
-        self._metadata = metadata
-
         self._api_client = api_client
         """The Apify dataset client for API operations."""
 
@@ -57,10 +54,10 @@ def __init__(
         self._lock = lock
         """A lock to ensure that only one operation is performed at a time."""
 
-    @property
     @override
-    def metadata(self) -> DatasetMetadata:
-        return self._metadata
+    async def get_metadata(self) -> DatasetMetadata:
+        metadata = await self._api_client.get()
+        return DatasetMetadata.model_validate(metadata)
 
     @classmethod
     async def open(
@@ -138,11 +135,7 @@ async def open(
         # Get the client for the specific storage by ID.
         apify_dataset_client = apify_client_async.dataset(dataset_id=id)
 
-        # Fetch its metadata.
-        metadata = DatasetMetadata.model_validate(await apify_dataset_client.get())
-
         return cls(
-            metadata=metadata,
             api_client=apify_dataset_client,
             api_public_base_url=api_public_base_url,
             lock=asyncio.Lock(),
@@ -178,8 +171,6 @@ async def payloads_generator() -> AsyncIterator[str]:
                 items = await self._check_and_serialize(data)
                 await self._api_client.push_items(items=items)
 
-            await self._update_metadata()
-
     @override
     async def get_data(
         self,
@@ -209,9 +200,7 @@ async def get_data(
             flatten=flatten,
             view=view,
         )
-        result = DatasetItemsListPage.model_validate(vars(response))
-        await self._update_metadata()
-        return result
+        return DatasetItemsListPage.model_validate(vars(response))
 
     @override
     async def iterate_items(
@@ -240,13 +229,6 @@ async def iterate_items(
         ):
             yield item
 
-        await self._update_metadata()
-
-    async def _update_metadata(self) -> None:
-        """Update the dataset metadata file with current information."""
-        metadata = await self._api_client.get()
-        self._metadata = DatasetMetadata.model_validate(metadata)
-
     @classmethod
     async def _check_and_serialize(cls, item: JsonSerializable, index: int | None = None) -> str:
         """Serialize a given item to JSON, checks its serializability and size against a limit.

src/apify/storage_clients/_apify/_key_value_store_client.py

@@ -29,7 +29,6 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
     def __init__(
         self,
         *,
-        metadata: KeyValueStoreMetadata,
         api_client: KeyValueStoreClientAsync,
         api_public_base_url: str,
         lock: asyncio.Lock,
@@ -38,8 +37,6 @@ def __init__(
 
         Preferably use the `ApifyKeyValueStoreClient.open` class method to create a new instance.
         """
-        self._metadata = metadata
-
         self._api_client = api_client
         """The Apify KVS client for API operations."""
 
@@ -49,10 +46,10 @@ def __init__(
         self._lock = lock
         """A lock to ensure that only one operation is performed at a time."""
 
-    @property
     @override
-    def metadata(self) -> KeyValueStoreMetadata:
-        return self._metadata
+    async def get_metadata(self) -> KeyValueStoreMetadata:
+        metadata = await self._api_client.get()
+        return KeyValueStoreMetadata.model_validate(metadata)
 
     @classmethod
     async def open(
@@ -130,11 +127,7 @@ async def open(
         # Get the client for the specific storage by ID.
         apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
 
-        # Fetch its metadata.
-        metadata = KeyValueStoreMetadata.model_validate(await apify_kvs_client.get())
-
         return cls(
-            metadata=metadata,
             api_client=apify_kvs_client,
             api_public_base_url=api_public_base_url,
             lock=asyncio.Lock(),
@@ -229,9 +222,10 @@ async def get_public_url(self, key: str) -> str:
         public_url = (
             URL(self._api_public_base_url) / 'v2' / 'key-value-stores' / self._api_client.resource_id / 'records' / key
         )
+        metadata = await self.get_metadata()
 
-        if self.metadata.model_extra is not None:
-            url_signing_secret_key = self.metadata.model_extra.get('urlSigningSecretKey')
+        if metadata.model_extra is not None:
+            url_signing_secret_key = metadata.model_extra.get('urlSigningSecretKey')
             if url_signing_secret_key is not None:
                 public_url = public_url.with_query(signature=create_hmac_signature(url_signing_secret_key, key))
 

src/apify/storage_clients/_apify/_request_queue_client.py

@@ -70,10 +70,10 @@ def __init__(
         self._should_check_for_forefront_requests = False
         """Whether to check for forefront requests in the next list_head call."""
 
-    @property
     @override
-    def metadata(self) -> RequestQueueMetadata:
-        return self._metadata
+    async def get_metadata(self) -> RequestQueueMetadata:
+        metadata = await self._api_client.get()
+        return RequestQueueMetadata.model_validate(metadata)
 
     @classmethod
     async def open(

tests/integration/test_actor_dataset.py

@@ -104,8 +104,9 @@ async def main() -> None:
             dataset_by_name_2 = await Actor.open_dataset(name=dataset_name)
             assert dataset_by_name_1 is dataset_by_name_2
 
-            dataset_by_id_1 = await Actor.open_dataset(id=dataset_by_name_1.metadata.id)
-            dataset_by_id_2 = await Actor.open_dataset(id=dataset_by_name_1.metadata.id)
+            dataset_1_metadata = await dataset_by_name_1.get_metadata()
+            dataset_by_id_1 = await Actor.open_dataset(id=dataset_1_metadata.id)
+            dataset_by_id_2 = await Actor.open_dataset(id=dataset_1_metadata.id)
             assert dataset_by_id_1 is dataset_by_name_1
             assert dataset_by_id_2 is dataset_by_id_1
 
@@ -129,7 +130,7 @@ async def test_force_cloud(
 
     async with Actor:
         dataset = await Actor.open_dataset(name=dataset_name, force_cloud=True)
-        dataset_id = dataset.metadata.id
+        dataset_id = (await dataset.get_metadata()).id
 
         await dataset.push_data(dataset_item)
 

tests/integration/test_actor_key_value_store.py

@@ -45,8 +45,9 @@ async def main() -> None:
             kvs_by_name_2 = await Actor.open_key_value_store(name=kvs_name)
             assert kvs_by_name_1 is kvs_by_name_2
 
-            kvs_by_id_1 = await Actor.open_key_value_store(id=kvs_by_name_1.metadata.id)
-            kvs_by_id_2 = await Actor.open_key_value_store(id=kvs_by_name_1.metadata.id)
+            kvs_1_metadata = await kvs_by_name_1.get_metadata()
+            kvs_by_id_1 = await Actor.open_key_value_store(id=kvs_1_metadata.id)
+            kvs_by_id_2 = await Actor.open_key_value_store(id=kvs_1_metadata.id)
             assert kvs_by_id_1 is kvs_by_name_1
             assert kvs_by_id_2 is kvs_by_id_1
 
@@ -69,7 +70,7 @@ async def test_force_cloud(
 
     async with Actor:
         key_value_store = await Actor.open_key_value_store(name=key_value_store_name, force_cloud=True)
-        key_value_store_id = key_value_store.metadata.id
+        key_value_store_id = (await key_value_store.get_metadata()).id
 
         await key_value_store.set_value('foo', 'bar')
 
@@ -209,9 +210,10 @@ async def main() -> None:
             record_key = 'public-record-key'
 
             kvs = await Actor.open_key_value_store()
-            assert kvs.metadata.model_extra is not None
+            metadata = await kvs.get_metadata()
+            assert metadata.model_extra is not None
 
-            url_signing_secret_key = kvs.metadata.model_extra.get('urlSigningSecretKey')
+            url_signing_secret_key = metadata.model_extra.get('urlSigningSecretKey')
             assert url_signing_secret_key is not None
 
             await kvs.set_value(record_key, {'exposedData': 'test'}, 'application/json')

tests/integration/test_actor_request_queue.py

@@ -45,8 +45,9 @@ async def main() -> None:
             rq_by_name_2 = await Actor.open_request_queue(name=rq_name)
             assert rq_by_name_1 is rq_by_name_2
 
-            rq_by_id_1 = await Actor.open_request_queue(id=rq_by_name_1.metadata.id)
-            rq_by_id_2 = await Actor.open_request_queue(id=rq_by_name_1.metadata.id)
+            rq_1_metadata = await rq_by_name_1.get_metadata()
+            rq_by_id_1 = await Actor.open_request_queue(id=rq_1_metadata.id)
+            rq_by_id_2 = await Actor.open_request_queue(id=rq_1_metadata.id)
             assert rq_by_id_1 is rq_by_name_1
             assert rq_by_id_2 is rq_by_id_1
 
@@ -69,7 +70,7 @@ async def test_force_cloud(
 
     async with Actor:
         request_queue = await Actor.open_request_queue(name=request_queue_name, force_cloud=True)
-        request_queue_id = request_queue.metadata.id
+        request_queue_id = (await request_queue.get_metadata()).id
 
         request_info = await request_queue.add_request(Request.from_url('http://example.com'))
 

tests/unit/actor/test_actor_dataset.py

@@ -21,8 +21,9 @@ async def test_open_dataset_returns_same_references() -> None:
         dataset_by_name_2 = await Actor.open_dataset(name=dataset_name)
         assert dataset_by_name_1 is dataset_by_name_2
 
-        dataset_by_id_1 = await Actor.open_dataset(id=dataset_by_name_1.metadata.id)
-        dataset_by_id_2 = await Actor.open_dataset(id=dataset_by_name_1.metadata.id)
+        dataset_1_metadata = await dataset_by_name_1.get_metadata()
+        dataset_by_id_1 = await Actor.open_dataset(id=dataset_1_metadata.id)
+        dataset_by_id_2 = await Actor.open_dataset(id=dataset_1_metadata.id)
 
         assert dataset_by_id_1 is dataset_by_name_1
         assert dataset_by_id_2 is dataset_by_id_1

tests/unit/actor/test_actor_key_value_store.py

@@ -21,8 +21,9 @@ async def test_open_returns_same_references() -> None:
         kvs_by_name_2 = await Actor.open_key_value_store(name=kvs_name)
         assert kvs_by_name_1 is kvs_by_name_2
 
-        kvs_by_id_1 = await Actor.open_key_value_store(id=kvs_by_name_1.metadata.id)
-        kvs_by_id_2 = await Actor.open_key_value_store(id=kvs_by_name_1.metadata.id)
+        kvs_1_metadata = await kvs_by_name_1.get_metadata()
+        kvs_by_id_1 = await Actor.open_key_value_store(id=kvs_1_metadata.id)
+        kvs_by_id_2 = await Actor.open_key_value_store(id=kvs_1_metadata.id)
         assert kvs_by_id_1 is kvs_by_name_1
         assert kvs_by_id_2 is kvs_by_id_1