Adapt to Crawlee v1 (p2)

Author: vdusek

pyproject.toml

@@ -36,6 +36,7 @@ keywords = [
 dependencies = [
     "apify-client>=1.9.2",
     "apify-shared>=1.3.0",
+    "cachetools>=5.5.0",
     "crawlee@git+https://github.com/apify/crawlee-python.git@new-storage-clients",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
@@ -72,7 +73,8 @@ dev = [
     "pytest~=8.4.0",
     "respx~=0.22.0",
     "ruff~=0.11.0",
-    "setuptools",             # setuptools are used by pytest but not explicitly required
+    "setuptools", # setuptools are used by pytest but not explicitly required
+    "types-cachetools>=6.0.0.20250525",
 ]
 
 [tool.hatch.build.targets.wheel]

src/apify/_actor.py

@@ -33,8 +33,8 @@
 from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager
 from apify._proxy_configuration import ProxyConfiguration
 from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython
-from apify.apify_storage_client import ApifyStorageClient
 from apify.log import _configure_logging, logger
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 if TYPE_CHECKING:

src/apify/_proxy_configuration.py

@@ -21,7 +21,8 @@
 
 if TYPE_CHECKING:
     from apify_client import ApifyClientAsync
-    from crawlee import Request
+
+    from apify import Request
 
 APIFY_PROXY_VALUE_REGEX = re.compile(r'^[\w._~]+$')
 COUNTRY_CODE_REGEX = re.compile(r'^[A-Z]{2}$')

src/apify/scrapy/extensions/_httpcache.py

@@ -13,8 +13,8 @@
 from scrapy.responsetypes import responsetypes
 
 from apify import Configuration
-from apify.apify_storage_client import ApifyStorageClient
 from apify.scrapy._async_thread import AsyncThread
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import KeyValueStore
 
 if TYPE_CHECKING:

src/apify/scrapy/requests.py

@@ -10,9 +10,10 @@
 from scrapy.http.headers import Headers
 from scrapy.utils.request import request_from_dict
 
-from crawlee import Request as ApifyRequest
 from crawlee._types import HttpHeaders
 
+from apify import Request as ApifyRequest
+
 logger = getLogger(__name__)
 
 

src/apify/scrapy/scheduler.py

@@ -11,7 +11,7 @@
 from ._async_thread import AsyncThread
 from .requests import to_apify_request, to_scrapy_request
 from apify import Configuration
-from apify.apify_storage_client import ApifyStorageClient
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import RequestQueue
 
 if TYPE_CHECKING:

src/apify/storage_clients/__init__.py

@@ -0,0 +1,10 @@
+from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient, StorageClient
+
+from ._apify import ApifyStorageClient
+
+__all__ = [
+    'ApifyStorageClient',
+    'FileSystemStorageClient',
+    'MemoryStorageClient',
+    'StorageClient',
+]

src/apify/apify_storage_client/__init__.py renamed to src/apify/storage_clients/_apify/__init__.py

@@ -15,8 +15,7 @@
     from datetime import datetime
 
     from apify_client.clients import DatasetClientAsync
-
-    from apify import Configuration
+    from crawlee.configuration import Configuration
 
 logger = getLogger(__name__)
 
@@ -68,8 +67,13 @@ async def open(
         name: str | None,
         configuration: Configuration,
     ) -> ApifyDatasetClient:
-        token = configuration.token
-        api_url = configuration.api_base_url
+        token = getattr(configuration, 'token', None)
+        if not token:
+            raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
+
+        api_url = getattr(configuration, 'api_base_url', None)
+        if not api_url:
+            raise ValueError(f'Apify storage client requires a valid API URL in Configuration (api_url={api_url}).')
 
         # Otherwise, create a new one.
         apify_client_async = ApifyClientAsync(
@@ -100,7 +104,8 @@ async def open(
 
     @override
     async def purge(self) -> None:
-        # TODO: better
+        # TODO: better?
+        # https://github.com/apify/apify-sdk-python/issues/469
         async with self._lock:
             await self._api_client.delete()
 

src/apify/apify_storage_client/_dataset_client.py renamed to src/apify/storage_clients/_apify/_dataset_client.py

@@ -19,8 +19,7 @@
     from datetime import datetime
 
     from apify_client.clients import KeyValueStoreClientAsync
-
-    from apify import Configuration
+    from crawlee.configuration import Configuration
 
 logger = getLogger(__name__)
 
@@ -70,8 +69,13 @@ async def open(
         name: str | None,
         configuration: Configuration,
     ) -> ApifyKeyValueStoreClient:
-        token = configuration.token
-        api_url = configuration.api_base_url
+        token = getattr(configuration, 'token', None)
+        if not token:
+            raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
+
+        api_url = getattr(configuration, 'api_base_url', None)
+        if not api_url:
+            raise ValueError(f'Apify storage client requires a valid API URL in Configuration (api_url={api_url}).')
 
         # Otherwise, create a new one.
         apify_client_async = ApifyClientAsync(
@@ -101,7 +105,8 @@ async def open(
 
     @override
     async def purge(self) -> None:
-        # TODO: better
+        # TODO: better?
+        # https://github.com/apify/apify-sdk-python/issues/469
         async with self._lock:
             await self._api_client.delete()
 
@@ -147,7 +152,13 @@ async def iterate_keys(
             list_key_page = KeyValueStoreListKeysPage.model_validate(response)
 
             for item in list_key_page.items:
-                yield item
+                # Convert KeyValueStoreKeyInfo to KeyValueStoreRecordMetadata
+                record_metadata = KeyValueStoreRecordMetadata(
+                    key=item.key,
+                    size=item.size,
+                    content_type='application/octet-stream',  # Content type not available from list_keys
+                )
+                yield record_metadata
                 count += 1
 
                 # If we've reached the limit, stop yielding