Merge remote-tracking branch 'origin/master' into ppe-guide

Author: janbuchar

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file.
 
+<!-- git-cliff-unreleased-start -->
+## 2.4.0 - **not yet released**
+
+### 🚀 Features
+
+- Update to Crawlee v0.6 ([#420](https://github.com/apify/apify-sdk-python/pull/420)) ([9be4336](https://github.com/apify/apify-sdk-python/commit/9be433667231cc5739861fa693d7a726860d6aca)) by [@vdusek](https://github.com/vdusek)
+- Add Actor `exit_process` option ([#424](https://github.com/apify/apify-sdk-python/pull/424)) ([994c832](https://github.com/apify/apify-sdk-python/commit/994c8323b994e009db0ccdcb624891a2fef97070)) by [@vdusek](https://github.com/vdusek), closes [#396](https://github.com/apify/apify-sdk-python/issues/396), [#401](https://github.com/apify/apify-sdk-python/issues/401)
+
+
+<!-- git-cliff-unreleased-end -->
 ## [2.3.1](https://github.com/apify/apify-sdk-python/releases/tag/v2.3.1) (2025-02-25)
 
 ### 🐛 Bug Fixes

docs/03_concepts/03_storages.mdx

@@ -38,6 +38,12 @@ Each storage is then stored in its own folder, named after the storage, or calle
 
 Each dataset item, key-value store record, or request in a request queue is then stored in its own file in the storage folder. Dataset items and request queue requests are always JSON files, and key-value store records can be any file type, based on its content type. For example, the Actor input is typically stored in `storage/key_value_stores/default/INPUT.json`.
 
+## Local Actor run with remote storage
+
+When developing locally, opening any storage will by default use local storage. To change this behavior and to use remote storage you have to use `force_cloud=True` argument in [`Actor.open_dataset`](../../reference/class/Actor#open_dataset), [`Actor.open_request_queue`](../../reference/class/Actor#open_request_queue) or [`Actor.open_key_value_store`](../../reference/class/Actor#open_key_value_store). Proper use of this argument allows you to work with both local and remote storages.
+
+Calling another remote Actor and accessing its default storage is typical use-case for using `force-cloud=True` argument to open remote Actor's storages.
+
 ### Local storage persistence
 
 By default, the storage contents are persisted across multiple Actor runs. To clean up the Actor storages before the running the Actor, use the `--purge` flag of the [`apify run`](https://docs.apify.com/cli/docs/reference#apify-run) command of the Apify CLI.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "2.3.1"
+version = "2.4.0"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "[email protected]" }]
 license = { file = "LICENSE" }
@@ -35,8 +35,8 @@ keywords = [
 ]
 dependencies = [
     "apify-client>=1.9.2",
-    "apify-shared>=1.2.1",
-    "crawlee~=0.5.0",
+    "apify-shared>=1.3.0",
+    "crawlee~=0.6.0",
     "cryptography>=42.0.0",
     "httpx>=0.27.0",
     "lazy-object-proxy>=1.10.0",
@@ -62,7 +62,7 @@ scrapy = ["scrapy>=2.11.0"]
 dev = [
     "build~=1.2.0",
     "filelock~=3.17.0",
-    "griffe~=1.5.0",
+    "griffe~=1.6.0",
     "mypy~=1.15.0",
     "pre-commit~=4.1.0",
     "pydoc-markdown~=4.8.0",

src/apify/_actor.py

@@ -3,6 +3,7 @@
 import asyncio
 import os
 import sys
+from contextlib import suppress
 from datetime import timedelta
 from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast, overload
 
@@ -43,7 +44,7 @@
     from typing_extensions import Self
 
     from crawlee.proxy_configuration import _NewUrlFunction
-    from crawlee.storage_clients import BaseStorageClient
+    from crawlee.storage_clients import StorageClient
 
     from apify._models import Webhook
 
@@ -64,6 +65,7 @@ def __init__(
         configuration: Configuration | None = None,
         *,
         configure_logging: bool = True,
+        exit_process: bool | None = None,
     ) -> None:
         """Create an Actor instance.
 
@@ -74,7 +76,10 @@ def __init__(
             configuration: The Actor configuration to be used. If not passed, a new Configuration instance will
                 be created.
             configure_logging: Should the default logging configuration be configured?
+            exit_process: Whether the Actor should call `sys.exit` when the context manager exits. The default is
+                True except for the IPython, Pytest and Scrapy environments.
         """
+        self._exit_process = self._get_default_exit_process() if exit_process is None else exit_process
         self._is_exiting = False
 
         self._configuration = configuration or Configuration.get_global_configuration()
@@ -141,9 +146,19 @@ def __repr__(self) -> str:
 
         return super().__repr__()
 
-    def __call__(self, configuration: Configuration | None = None, *, configure_logging: bool = True) -> Self:
+    def __call__(
+        self,
+        configuration: Configuration | None = None,
+        *,
+        configure_logging: bool = True,
+        exit_process: bool | None = None,
+    ) -> Self:
         """Make a new Actor instance with a non-default configuration."""
-        return self.__class__(configuration=configuration, configure_logging=configure_logging)
+        return self.__class__(
+            configuration=configuration,
+            configure_logging=configure_logging,
+            exit_process=exit_process,
+        )
 
     @property
     def apify_client(self) -> ApifyClientAsync:
@@ -171,7 +186,7 @@ def log(self) -> logging.Logger:
         return logger
 
     @property
-    def _local_storage_client(self) -> BaseStorageClient:
+    def _local_storage_client(self) -> StorageClient:
         """The local storage client the Actor instance uses."""
         return service_locator.get_storage_client()
 
@@ -281,13 +296,7 @@ async def finalize() -> None:
         await asyncio.wait_for(finalize(), cleanup_timeout.total_seconds())
         self._is_initialized = False
 
-        if is_running_in_ipython():
-            self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in IPython')
-        elif os.getenv('PYTEST_CURRENT_TEST', default=False):  # noqa: PLW1508
-            self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in an unit test')
-        elif os.getenv('SCRAPY_SETTINGS_MODULE'):
-            self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running with Scrapy')
-        else:
+        if self._exit_process:
             sys.exit(exit_code)
 
     async def fail(
@@ -1128,6 +1137,26 @@ async def create_proxy_configuration(
 
         return proxy_configuration
 
+    def _get_default_exit_process(self) -> bool:
+        """Returns False for IPython, Pytest, and Scrapy environments, True otherwise."""
+        if is_running_in_ipython():
+            self.log.debug('Running in IPython, setting default `exit_process` to False.')
+            return False
+
+        # Check if running in Pytest by detecting the relevant environment variable.
+        if os.getenv('PYTEST_CURRENT_TEST'):
+            self.log.debug('Running in Pytest, setting default `exit_process` to False.')
+            return False
+
+        # Check if running in Scrapy by attempting to import it.
+        with suppress(ImportError):
+            import scrapy  # noqa: F401
+
+            self.log.debug('Running in Scrapy, setting default `exit_process` to False.')
+            return False
+
+        return True
+
 
 Actor = cast(_ActorType, Proxy(_ActorType))
 """The entry point of the SDK, through which all the Actor operations should be done."""

src/apify/apify_storage_client/_apify_storage_client.py

@@ -6,7 +6,7 @@
 
 from apify_client import ApifyClientAsync
 from crawlee._utils.crypto import crypto_random_object_id
-from crawlee.storage_clients import BaseStorageClient
+from crawlee.storage_clients import StorageClient
 
 from apify._utils import docs_group
 from apify.apify_storage_client._dataset_client import DatasetClient
@@ -21,7 +21,7 @@
 
 
 @docs_group('Classes')
-class ApifyStorageClient(BaseStorageClient):
+class ApifyStorageClient(StorageClient):
     """A storage client implementation based on the Apify platform storage."""
 
     def __init__(self, *, configuration: Configuration) -> None:
@@ -68,5 +68,5 @@ async def purge_on_start(self) -> None:
         pass
 
     @override
-    def get_rate_limit_errors(self) -> dict[int, int]:  # type: ignore[misc]
+    def get_rate_limit_errors(self) -> dict[int, int]:
         return self._apify_client.stats.rate_limit_errors

src/apify/apify_storage_client/_dataset_client.py

@@ -4,7 +4,7 @@
 
 from typing_extensions import override
 
-from crawlee.storage_clients._base import BaseDatasetClient
+from crawlee.storage_clients._base import DatasetClient as BaseDatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
 if TYPE_CHECKING:

src/apify/apify_storage_client/_dataset_collection_client.py

@@ -4,7 +4,7 @@
 
 from typing_extensions import override
 
-from crawlee.storage_clients._base import BaseDatasetCollectionClient
+from crawlee.storage_clients._base import DatasetCollectionClient as BaseDatasetCollectionClient
 from crawlee.storage_clients.models import DatasetListPage, DatasetMetadata
 
 if TYPE_CHECKING:

src/apify/apify_storage_client/_key_value_store_client.py

@@ -5,7 +5,7 @@
 
 from typing_extensions import override
 
-from crawlee.storage_clients._base import BaseKeyValueStoreClient
+from crawlee.storage_clients._base import KeyValueStoreClient as BaseKeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
 
 if TYPE_CHECKING:

src/apify/apify_storage_client/_key_value_store_collection_client.py

@@ -4,7 +4,7 @@
 
 from typing_extensions import override
 
-from crawlee.storage_clients._base import BaseKeyValueStoreCollectionClient
+from crawlee.storage_clients._base import KeyValueStoreCollectionClient as BaseKeyValueStoreCollectionClient
 from crawlee.storage_clients.models import KeyValueStoreListPage, KeyValueStoreMetadata
 
 if TYPE_CHECKING:

src/apify/apify_storage_client/_request_queue_client.py

@@ -5,7 +5,7 @@
 from typing_extensions import override
 
 from crawlee import Request
-from crawlee.storage_clients._base import BaseRequestQueueClient
+from crawlee.storage_clients._base import RequestQueueClient as BaseRequestQueueClient
 from crawlee.storage_clients.models import (
     BatchRequestsOperationResponse,
     ProcessedRequest,