Apify storage client fixes

Author: vdusek

pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
     "apify-client>=1.12.0",
     "apify-shared>=1.3.0",
     "cachetools>=5.5.0",
-    "crawlee@git+https://github.com/apify/crawlee-python.git@0c4cfc9ada06e35f63213e6a937c4e85defcbecf",
+    "crawlee@git+https://github.com/apify/crawlee-python.git@master",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
     # TODO: ensure compatibility with the latest version of lazy-object-proxy

src/apify/storage_clients/_apify/_dataset_client.py

@@ -105,9 +105,6 @@ async def open(
                 f'(api_public_base_url={api_public_base_url}).'
             )
 
-        if id and name:
-            raise ValueError('Only one of "id" or "name" can be specified, not both.')
-
         # Create Apify client with the provided token and API URL.
         apify_client_async = ApifyClientAsync(
             token=token,
@@ -118,23 +115,40 @@ async def open(
         )
         apify_datasets_client = apify_client_async.datasets()
 
+        # If both id and name are provided, raise an error.
+        if id and name:
+            raise ValueError('Only one of "id" or "name" can be specified, not both.')
+
+        # If id is provided, get the storage by ID.
+        if id and name is None:
+            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
+
         # If name is provided, get or create the storage by name.
-        if name is not None and id is None:
+        if name and id is None:
             id = DatasetMetadata.model_validate(
                 await apify_datasets_client.get_or_create(name=name),
             ).id
+            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
 
         # If both id and name are None, try to get the default storage ID from environment variables.
         if id is None and name is None:
-            id = getattr(configuration, 'default_dataset_id', None)
+            id = configuration.default_dataset_id
+            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
 
-        if id is None:
-            raise ValueError(
-                'Either "id" or "name" must be provided, or the storage ID must be set in environment variable.'
-            )
+        # Fetch its metadata.
+        metadata = await apify_dataset_client.get()
+
+        # If metadata is None, it means the storage does not exist, so we create it.
+        if metadata is None:
+            id = DatasetMetadata.model_validate(
+                await apify_datasets_client.get_or_create(),
+            ).id
+            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
 
-        # Get the client for the specific storage by ID.
-        apify_dataset_client = apify_client_async.dataset(dataset_id=id)
+        # Verify that the storage exists by fetching its metadata again.
+        metadata = await apify_dataset_client.get()
+        if metadata is None:
+            raise ValueError(f'Opening dataset with id={id} and name={name} failed.')
 
         return cls(
             api_client=apify_dataset_client,

src/apify/storage_clients/_apify/_key_value_store_client.py

@@ -97,9 +97,6 @@ async def open(
                 f'(api_public_base_url={api_public_base_url}).'
             )
 
-        if id and name:
-            raise ValueError('Only one of "id" or "name" can be specified, not both.')
-
         # Create Apify client with the provided token and API URL.
         apify_client_async = ApifyClientAsync(
             token=token,
@@ -110,23 +107,40 @@ async def open(
         )
         apify_kvss_client = apify_client_async.key_value_stores()
 
+        # If both id and name are provided, raise an error.
+        if id and name:
+            raise ValueError('Only one of "id" or "name" can be specified, not both.')
+
+        # If id is provided, get the storage by ID.
+        if id and name is None:
+            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
+
         # If name is provided, get or create the storage by name.
-        if name is not None and id is None:
+        if name and id is None:
             id = ApifyKeyValueStoreMetadata.model_validate(
                 await apify_kvss_client.get_or_create(name=name),
             ).id
+            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
 
         # If both id and name are None, try to get the default storage ID from environment variables.
         if id is None and name is None:
-            id = getattr(configuration, 'default_key_value_store_id', None)
+            id = configuration.default_key_value_store_id
+            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
 
-        if id is None:
-            raise ValueError(
-                'Either "id" or "name" must be provided, or the storage ID must be set in environment variable.'
-            )
+        # Fetch its metadata.
+        metadata = await apify_kvs_client.get()
+
+        # If metadata is None, it means the storage does not exist, so we create it.
+        if metadata is None:
+            id = ApifyKeyValueStoreMetadata.model_validate(
+                await apify_kvss_client.get_or_create(),
+            ).id
+            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
 
-        # Get the client for the specific storage by ID.
-        apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
+        # Verify that the storage exists by fetching its metadata again.
+        metadata = await apify_kvs_client.get()
+        if metadata is None:
+            raise ValueError(f'Opening key-value store with id={id} and name={name} failed.')
 
         return cls(
             api_client=apify_kvs_client,

src/apify/storage_clients/_apify/_request_queue_client.py

@@ -39,7 +39,6 @@ class ApifyRequestQueueClient(RequestQueueClient):
     def __init__(
         self,
         *,
-        metadata: RequestQueueMetadata,
         api_client: RequestQueueClientAsync,
         api_public_base_url: str,
         lock: asyncio.Lock,
@@ -48,8 +47,6 @@ def __init__(
 
         Preferably use the `ApifyRequestQueueClient.open` class method to create a new instance.
         """
-        self._metadata = metadata
-
         self._api_client = api_client
         """The Apify request queue client for API operations."""
 
@@ -122,9 +119,6 @@ async def open(
                 f'(api_public_base_url={api_public_base_url}).'
             )
 
-        if id and name:
-            raise ValueError('Only one of "id" or "name" can be specified, not both.')
-
         # Create Apify client with the provided token and API URL.
         apify_client_async = ApifyClientAsync(
             token=token,
@@ -135,29 +129,42 @@ async def open(
         )
         apify_rqs_client = apify_client_async.request_queues()
 
+        # If both id and name are provided, raise an error.
+        if id and name:
+            raise ValueError('Only one of "id" or "name" can be specified, not both.')
+
+        # If id is provided, get the storage by ID.
+        if id and name is None:
+            apify_rq_client = apify_client_async.request_queue(request_queue_id=id)
+
         # If name is provided, get or create the storage by name.
-        if name is not None and id is None:
+        if name and id is None:
             id = RequestQueueMetadata.model_validate(
                 await apify_rqs_client.get_or_create(name=name),
             ).id
+            apify_rq_client = apify_client_async.request_queue(request_queue_id=id)
 
         # If both id and name are None, try to get the default storage ID from environment variables.
         if id is None and name is None:
-            id = getattr(configuration, 'default_request_queue_id', None)
+            id = configuration.default_request_queue_id
+            apify_rq_client = apify_client_async.request_queue(request_queue_id=id)
 
-        if id is None:
-            raise ValueError(
-                'Either "id" or "name" must be provided, or the storage ID must be set in environment variable.'
-            )
+        # Fetch its metadata.
+        metadata = await apify_rq_client.get()
 
-        # Get the client for the specific storage by ID.
-        apify_rq_client = apify_client_async.request_queue(request_queue_id=id)
+        # If metadata is None, it means the storage does not exist, so we create it.
+        if metadata is None:
+            id = RequestQueueMetadata.model_validate(
+                await apify_rqs_client.get_or_create(),
+            ).id
+            apify_rq_client = apify_client_async.request_queue(request_queue_id=id)
 
-        # Fetch its metadata.
-        metadata = RequestQueueMetadata.model_validate(await apify_rq_client.get())
+        # Verify that the storage exists by fetching its metadata again.
+        metadata = await apify_rq_client.get()
+        if metadata is None:
+            raise ValueError(f'Opening request queue with id={id} and name={name} failed.')
 
         return cls(
-            metadata=metadata,
             api_client=apify_rq_client,
             api_public_base_url=api_public_base_url,
             lock=asyncio.Lock(),
@@ -477,10 +484,12 @@ async def _list_head(
                 if cached_request and cached_request.hydrated:
                     items.append(cached_request.hydrated)
 
+            metadata = await self.get_metadata()
+
             return RequestQueueHead(
                 limit=limit,
-                had_multiple_clients=self._metadata.had_multiple_clients,
-                queue_modified_at=self._metadata.modified_at,
+                had_multiple_clients=metadata.had_multiple_clients,
+                queue_modified_at=metadata.modified_at,
                 items=items,
                 queue_has_locked_requests=self._queue_has_locked_requests,
                 lock_time=lock_time,