Adapt to Crawlee v1 (p2)

Author: vdusek

pyproject.toml

@@ -36,6 +36,7 @@ keywords = [
 dependencies = [
     "apify-client>=1.9.2",
     "apify-shared>=1.3.0",
+    "cachetools>=5.5.0",
     "crawlee@git+https://github.com/apify/crawlee-python.git@new-storage-clients",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
@@ -72,7 +73,8 @@ dev = [
     "pytest~=8.4.0",
     "respx~=0.22.0",
     "ruff~=0.11.0",
-    "setuptools",             # setuptools are used by pytest but not explicitly required
+    "setuptools", # setuptools are used by pytest but not explicitly required
+    "types-cachetools>=6.0.0.20250525",
 ]
 
 [tool.hatch.build.targets.wheel]

src/apify/apify_storage_client/_dataset_client.py

@@ -15,8 +15,7 @@
     from datetime import datetime
 
     from apify_client.clients import DatasetClientAsync
-
-    from apify import Configuration
+    from crawlee.configuration import Configuration
 
 logger = getLogger(__name__)
 
@@ -68,8 +67,8 @@ async def open(
         name: str | None,
         configuration: Configuration,
     ) -> ApifyDatasetClient:
-        token = configuration.token
-        api_url = configuration.api_base_url
+        token = getattr(configuration, 'token', None)
+        api_url = getattr(configuration, 'api_base_url', 'https://api.apify.com')
 
         # Otherwise, create a new one.
         apify_client_async = ApifyClientAsync(
@@ -100,7 +99,8 @@ async def open(
 
     @override
     async def purge(self) -> None:
-        # TODO: better
+        # TODO: better?
+        # https://github.com/apify/apify-sdk-python/issues/469
         async with self._lock:
             await self._api_client.delete()
 

src/apify/apify_storage_client/_key_value_store_client.py

@@ -19,8 +19,7 @@
     from datetime import datetime
 
     from apify_client.clients import KeyValueStoreClientAsync
-
-    from apify import Configuration
+    from crawlee.configuration import Configuration
 
 logger = getLogger(__name__)
 
@@ -70,8 +69,8 @@ async def open(
         name: str | None,
         configuration: Configuration,
     ) -> ApifyKeyValueStoreClient:
-        token = configuration.token
-        api_url = configuration.api_base_url
+        token = getattr(configuration, 'token', None)
+        api_url = getattr(configuration, 'api_base_url', 'https://api.apify.com')
 
         # Otherwise, create a new one.
         apify_client_async = ApifyClientAsync(
@@ -101,7 +100,8 @@ async def open(
 
     @override
     async def purge(self) -> None:
-        # TODO: better
+        # TODO: better?
+        # https://github.com/apify/apify-sdk-python/issues/469
         async with self._lock:
             await self._api_client.delete()
 
@@ -147,7 +147,13 @@ async def iterate_keys(
             list_key_page = KeyValueStoreListKeysPage.model_validate(response)
 
             for item in list_key_page.items:
-                yield item
+                # Convert KeyValueStoreKeyInfo to KeyValueStoreRecordMetadata
+                record_metadata = KeyValueStoreRecordMetadata(
+                    key=item.key,
+                    size=item.size,
+                    content_type='application/octet-stream',  # Content type not available from list_keys
+                )
+                yield record_metadata
                 count += 1
 
                 # If we've reached the limit, stop yielding

src/apify/apify_storage_client/_request_queue_client.py

@@ -21,8 +21,7 @@
     from collections.abc import Sequence
 
     from apify_client.clients import RequestQueueClientAsync
-
-    from apify import Configuration
+    from crawlee.configuration import Configuration
 
 logger = getLogger(__name__)
 
@@ -101,8 +100,8 @@ async def open(
         configuration: Configuration,
     ) -> ApifyRequestQueueClient:
         # Get API credentials
-        token = configuration.token
-        api_url = configuration.api_base_url
+        token = getattr(configuration, 'token', None)
+        api_url = getattr(configuration, 'api_base_url', 'https://api.apify.com')
 
         # Create a new API client
         apify_client_async = ApifyClientAsync(
@@ -139,7 +138,8 @@ async def open(
 
     @override
     async def purge(self) -> None:
-        # TODO: better
+        # TODO: better?
+        # https://github.com/apify/apify-sdk-python/issues/469
         async with self._lock:
             await self._api_client.delete()
 

src/apify/apify_storage_client/_storage_client.py

@@ -14,7 +14,7 @@ class ApifyStorageClient(StorageClient):
     """Apify storage client."""
 
     @override
-    async def open_dataset_client(
+    async def create_dataset_client(
         self,
         *,
         id: str | None = None,
@@ -31,7 +31,7 @@ async def open_dataset_client(
         return client
 
     @override
-    async def open_key_value_store_client(
+    async def create_kvs_client(
         self,
         *,
         id: str | None = None,
@@ -48,7 +48,7 @@ async def open_key_value_store_client(
         return client
 
     @override
-    async def open_request_queue_client(
+    async def create_rq_client(
         self,
         *,
         id: str | None = None,

tests/unit/actor/test_actor_dataset.py

@@ -43,7 +43,7 @@ async def test_open_dataset_uses_env_var(monkeypatch: pytest.MonkeyPatch) -> Non
     async with Actor:
         ddt = await Actor.open_dataset()
         assert ddt.metadata.id == default_dataset_id
-        dataset = await memory_storage_client.open_dataset_client(id=ddt.metadata.id)
+        dataset = await memory_storage_client.create_dataset_client(id=ddt.metadata.id)
         await dataset.drop()
 
 

tests/unit/actor/test_actor_key_value_store.py

@@ -52,7 +52,7 @@ async def test_get_input() -> None:
     input_key = 'INPUT'
     test_input = {'foo': 'bar'}
 
-    kvs_client = await memory_storage_client.open_key_value_store_client()
+    kvs_client = await memory_storage_client.create_kvs_client()
 
     await kvs_client.set_value(
         key=input_key,
@@ -79,7 +79,7 @@ async def test_get_input_with_encrypted_secrets(monkeypatch: pytest.MonkeyPatch)
         'secret': f'{ENCRYPTED_INPUT_VALUE_PREFIX}:{encrypted_secret["encrypted_password"]}:{encrypted_secret["encrypted_value"]}',  # noqa: E501
     }
 
-    kvs_client = await memory_storage_client.open_key_value_store_client()
+    kvs_client = await memory_storage_client.create_kvs_client()
 
     await kvs_client.set_value(
         key=input_key,