Adapt to Crawlee v1

Author: vdusek

docs/03_concepts/code/03_dataset_exports.py

@@ -11,14 +11,14 @@ async def main() -> None:
         await dataset.export_to(
             content_type='csv',
             key='data.csv',
-            to_key_value_store_name='my-cool-key-value-store',
+            to_kvs_name='my-cool-key-value-store',
         )
 
         # Export the data as JSON
         await dataset.export_to(
             content_type='json',
             key='data.json',
-            to_key_value_store_name='my-cool-key-value-store',
+            to_kvs_name='my-cool-key-value-store',
         )
 
         # Print the exported records

docs/03_concepts/code/conditional_actor_charge.py

@@ -6,8 +6,7 @@ async def main() -> None:
         # Check the dataset because there might already be items
         # if the run migrated or was restarted
         default_dataset = await Actor.open_dataset()
-        dataset_info = await default_dataset.get_info()
-        charged_items = dataset_info.item_count if dataset_info else 0
+        charged_items = default_dataset.metadata.item_count
 
         # highlight-start
         if Actor.get_charging_manager().get_pricing_info().is_pay_per_event:

pyproject.toml

@@ -36,7 +36,7 @@ keywords = [
 dependencies = [
     "apify-client>=1.9.2",
     "apify-shared>=1.3.0",
-    "crawlee~=0.6.0",
+    "crawlee@git+https://github.com/apify/crawlee-python.git@new-storage-clients",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
     # TODO: ensure compatibility with the latest version of lazy-object-proxy
@@ -78,6 +78,9 @@ dev = [
 [tool.hatch.build.targets.wheel]
 packages = ["src/apify"]
 
+[tool.hatch.metadata]
+allow-direct-references = true
+
 [tool.ruff]
 line-length = 120
 include = ["src/**/*.py", "tests/**/*.py", "docs/**/*.py", "website/**/*.py"]

src/apify/_actor.py

@@ -88,7 +88,7 @@ def __init__(
 
         # Create an instance of the cloud storage client, the local storage client is obtained
         # from the service locator.
-        self._cloud_storage_client = ApifyStorageClient.from_config(config=self._configuration)
+        self._cloud_storage_client = ApifyStorageClient()
 
         # Set the event manager based on whether the Actor is running on the platform or locally.
         self._event_manager = (

src/apify/apify_storage_client/_dataset_client.py

@@ -54,8 +54,8 @@ def __init__(
         self._lock = asyncio.Lock()
         """A lock to ensure that only one operation is performed at a time."""
 
-    @override
     @property
+    @override
     def metadata(self) -> DatasetMetadata:
         return self._metadata
 

src/apify/apify_storage_client/_key_value_store_client.py

@@ -10,8 +10,8 @@
 from apify_client import ApifyClientAsync
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
-from ._models import KeyValueStoreListKeysPage
 
+from ._models import KeyValueStoreListKeysPage
 from apify._crypto import create_hmac_signature
 
 if TYPE_CHECKING:
@@ -56,8 +56,8 @@ def __init__(
         self._lock = asyncio.Lock()
         """A lock to ensure that only one operation is performed at a time."""
 
-    @override
     @property
+    @override
     def metadata(self) -> KeyValueStoreMetadata:
         return self._metadata
 

src/apify/apify_storage_client/_request_queue_client.py

@@ -86,8 +86,8 @@ def __init__(
         self._should_check_for_forefront_requests = False
         """Whether to check for forefront requests in the next list_head call."""
 
-    @override
     @property
+    @override
     def metadata(self) -> RequestQueueMetadata:
         return self._metadata
 

src/apify/scrapy/extensions/_httpcache.py

@@ -51,10 +51,14 @@ def open_spider(self, spider: Spider) -> None:
         kvs_name = get_kvs_name(spider.name)
 
         async def open_kvs() -> KeyValueStore:
-            config = Configuration.get_global_configuration()
-            if config.is_at_home:
-                storage_client = ApifyStorageClient.from_config(config)
-                return await KeyValueStore.open(name=kvs_name, storage_client=storage_client)
+            configuration = Configuration.get_global_configuration()
+            if configuration.is_at_home:
+                storage_client = ApifyStorageClient()
+                return await KeyValueStore.open(
+                    name=kvs_name,
+                    configuration=configuration,
+                    storage_client=storage_client,
+                )
             return await KeyValueStore.open(name=kvs_name)
 
         logger.debug("Starting background thread for cache storage's event loop")

src/apify/scrapy/scheduler.py

@@ -49,10 +49,13 @@ def open(self, spider: Spider) -> Deferred[None] | None:
         self.spider = spider
 
         async def open_rq() -> RequestQueue:
-            config = Configuration.get_global_configuration()
-            if config.is_at_home:
-                storage_client = ApifyStorageClient.from_config(config)
-                return await RequestQueue.open(storage_client=storage_client)
+            configuration = Configuration.get_global_configuration()
+            if configuration.is_at_home:
+                storage_client = ApifyStorageClient()
+                return await RequestQueue.open(
+                    configuration=configuration,
+                    storage_client=storage_client,
+                )
             return await RequestQueue.open()
 
         try:

tests/integration/conftest.py

@@ -15,7 +15,7 @@
 from apify_client import ApifyClient, ApifyClientAsync
 from apify_shared.consts import ActorJobStatus, ActorSourceType, ApifyEnvVars
 from crawlee import service_locator
-from crawlee.storages import _creation_management
+from crawlee.storages import Dataset, KeyValueStore, RequestQueue
 
 import apify._actor
 from ._utils import generate_unique_resource_name
@@ -65,12 +65,15 @@ def _prepare_test_env() -> None:
         service_locator._storage_client = None
 
         # Clear creation-related caches to ensure no state is carried over between tests.
-        monkeypatch.setattr(_creation_management, '_cache_dataset_by_id', {})
-        monkeypatch.setattr(_creation_management, '_cache_dataset_by_name', {})
-        monkeypatch.setattr(_creation_management, '_cache_kvs_by_id', {})
-        monkeypatch.setattr(_creation_management, '_cache_kvs_by_name', {})
-        monkeypatch.setattr(_creation_management, '_cache_rq_by_id', {})
-        monkeypatch.setattr(_creation_management, '_cache_rq_by_name', {})
+        Dataset._cache_by_id.clear()
+        Dataset._cache_by_name.clear()
+        Dataset._default_instance = None
+        KeyValueStore._cache_by_id.clear()
+        KeyValueStore._cache_by_name.clear()
+        KeyValueStore._default_instance = None
+        RequestQueue._cache_by_id.clear()
+        RequestQueue._cache_by_name.clear()
+        RequestQueue._default_instance = None
 
         # Verify that the test environment was set up correctly.
         assert os.environ.get(ApifyEnvVars.LOCAL_STORAGE_DIR) == str(tmp_path)