refactor!: Make Actor initialization stricter and more predictable

docs/04_upgrading/upgrading_to_v3.md

@@ -9,6 +9,38 @@ This page summarizes the breaking changes between Apify Python SDK v2.x and v3.0
 
 Support for Python 3.9 has been dropped. The Apify Python SDK v3.x now requires Python 3.10 or later. Make sure your environment is running a compatible version before upgrading.
 
+## Actor initialization and ServiceLocator changes
+
+`Actor` initialization and global `service_locator` services setup is more strict and predictable.
+- Services in `Actor` can't be changed after calling `Actor.init`, entering the `async with Actor` context manager or after requesting them from the `Actor`
+- Services in `Actor` can be different from services in Crawler
+
+**Now (v3.0):**
+
+```python
+from crawlee.crawlers import BasicCrawler
+from crawlee.storage_clients import MemoryStorageClient, FileSystemStorageClient
+from crawlee.configuration import Configuration
+from crawlee.events import LocalEventManager
+from apify import Actor
+
+async def main():
+
+    async with Actor():
+        # This crawler will use same services as Actor and global service_locator
+        crawler_1 = BasicCrawler()
+
+        # This crawler will use custom services
+        custom_configuration = Configuration()
+        custom_event_manager = LocalEventManager.from_config(custom_configuration)
+        custom_storage_client = MemoryStorageClient()
+        crawler_2 = BasicCrawler(
+            configuration=custom_configuration,
+            event_manager=custom_event_manager,
+            storage_client=custom_storage_client,
+        )
+```
+
 ## Storages
 
 <!-- TODO -->

pyproject.toml

@@ -36,7 +36,7 @@ keywords = [
 dependencies = [
     "apify-client>=2.0.0,<3.0.0",
     "apify-shared>=2.0.0,<3.0.0",
-    "crawlee==0.6.13b37",
+    "crawlee @ git+https://github.com/apify/crawlee-python.git@master",
     "cachetools>=5.5.0",
     "cryptography>=42.0.0",
     "impit>=0.6.1",

src/apify/_configuration.py

@@ -8,6 +8,7 @@
 from pydantic import AliasChoices, BeforeValidator, Field, model_validator
 from typing_extensions import Self, deprecated
 
+from crawlee import service_locator
 from crawlee._utils.models import timedelta_ms
 from crawlee._utils.urls import validate_http_url
 from crawlee.configuration import Configuration as CrawleeConfiguration
@@ -424,11 +425,35 @@ def disable_browser_sandbox_on_platform(self) -> Self:
     def get_global_configuration(cls) -> Configuration:
         """Retrieve the global instance of the configuration.
 
-        Mostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`
-        instead.
+        This method ensures that ApifyConfigration is returned, even if CrawleeConfiguration was set in the
+        service locator.
         """
-        return cls()
+        global_configuration = service_locator.get_configuration()
 
+        if isinstance(global_configuration, Configuration):
+            # If Apify configuration was already stored in service locator, return it.
+            return global_configuration
 
-# Monkey-patch the base class so that it works with the extended configuration
-CrawleeConfiguration.get_global_configuration = Configuration.get_global_configuration  # type: ignore[method-assign]
+        return cls.from_configuration(global_configuration)
+
+    @classmethod
+    def from_configuration(cls, configuration: CrawleeConfiguration) -> Configuration:
+        """Create Apify Configuration from existing Crawlee Configuration.
+
+        Args:
+            configuration: The existing Crawlee Configuration.
+
+        Returns:
+            The created Apify Configuration.
+        """
+        apify_configuration = cls()
+
+        # Ensure the returned configuration is of type Apify Configuration.
+        # Most likely crawlee configuration was already set. Create Apify configuration from it.
+        # Due to known Pydantic issue https://github.com/pydantic/pydantic/issues/9516, creating new instance of
+        # Configuration from existing one in situation where environment can have some fields set by alias is very
+        # unpredictable. Use the stable workaround.
+        for name in configuration.model_fields:
+            setattr(apify_configuration, name, getattr(configuration, name))
+
+        return apify_configuration

src/apify/storage_clients/_apify/_dataset_client.py

@@ -11,8 +11,9 @@
 from crawlee._utils.file import json_dumps
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
+from crawlee.storages import Dataset
 
-from ._utils import resolve_alias_to_id, store_alias_mapping
+from apify.storage_clients._apify._utils import Alias
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -126,19 +127,19 @@ async def open(
         # Normalize 'default' alias to None
         alias = None if alias == 'default' else alias
 
-        # Handle alias resolution
         if alias:
-            # Try to resolve alias to existing storage ID
-            resolved_id = await resolve_alias_to_id(alias, 'dataset', configuration)
-            if resolved_id:
-                id = resolved_id
-            else:
-                # Create a new storage and store the alias mapping
-                new_storage_metadata = DatasetMetadata.model_validate(
-                    await apify_datasets_client.get_or_create(),
-                )
-                id = new_storage_metadata.id
-                await store_alias_mapping(alias, 'dataset', id, configuration)
+            # Check if there is pre-existing alias mapping in the default KVS.
+            async with Alias(storage_type=Dataset, alias=alias, configuration=configuration) as _alias:
+                id = await _alias.resolve_id()
+
+                # There was no pre-existing alias in the mapping.
+                # Create a new unnamed storage and store the mapping.
+                if id is None:
+                    new_storage_metadata = DatasetMetadata.model_validate(
+                        await apify_datasets_client.get_or_create(),
+                    )
+                    id = new_storage_metadata.id
+                    await _alias.store_mapping(storage_id=id)
 
         # If name is provided, get or create the storage by name.
         elif name:

src/apify/storage_clients/_apify/_key_value_store_client.py

@@ -10,9 +10,10 @@
 from apify_client import ApifyClientAsync
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreRecord, KeyValueStoreRecordMetadata
+from crawlee.storages import KeyValueStore
 
 from ._models import ApifyKeyValueStoreMetadata, KeyValueStoreListKeysPage
-from ._utils import resolve_alias_to_id, store_alias_mapping
+from ._utils import Alias
 from apify._crypto import create_hmac_signature
 
 if TYPE_CHECKING:
@@ -117,19 +118,20 @@ async def open(
         # Normalize 'default' alias to None
         alias = None if alias == 'default' else alias
 
-        # Handle alias resolution
         if alias:
-            # Try to resolve alias to existing storage ID
-            resolved_id = await resolve_alias_to_id(alias, 'kvs', configuration)
-            if resolved_id:
-                id = resolved_id
-            else:
-                # Create a new storage and store the alias mapping
-                new_storage_metadata = ApifyKeyValueStoreMetadata.model_validate(
-                    await apify_kvss_client.get_or_create(),
-                )
-                id = new_storage_metadata.id
-                await store_alias_mapping(alias, 'kvs', id, configuration)
+            # Check if there is pre-existing alias mapping in the default KVS.
+            async with Alias(storage_type=KeyValueStore, alias=alias, configuration=configuration) as _alias:
+                id = await _alias.resolve_id()
+
+                # There was no pre-existing alias in the mapping.
+                # Create a new unnamed storage and store the mapping.
+                if id is None:
+                    # Create a new storage and store the alias mapping
+                    new_storage_metadata = ApifyKeyValueStoreMetadata.model_validate(
+                        await apify_kvss_client.get_or_create(),
+                    )
+                    id = new_storage_metadata.id
+                    await _alias.store_mapping(storage_id=id)
 
         # If name is provided, get or create the storage by name.
         elif name:

src/apify/storage_clients/_apify/_request_queue_client.py

@@ -16,9 +16,10 @@
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
+from crawlee.storages import RequestQueue
 
 from ._models import CachedRequest, ProlongRequestLockResponse, RequestQueueHead
-from ._utils import resolve_alias_to_id, store_alias_mapping
+from ._utils import Alias
 from apify import Request
 
 if TYPE_CHECKING:
@@ -192,22 +193,19 @@ async def open(
         )
         apify_rqs_client = apify_client_async.request_queues()
 
-        # Normalize 'default' alias to None
-        alias = None if alias == 'default' else alias
-
-        # Handle alias resolution
         if alias:
-            # Try to resolve alias to existing storage ID
-            resolved_id = await resolve_alias_to_id(alias, 'rq', configuration)
-            if resolved_id:
-                id = resolved_id
-            else:
-                # Create a new storage and store the alias mapping
-                new_storage_metadata = RequestQueueMetadata.model_validate(
-                    await apify_rqs_client.get_or_create(),
-                )
-                id = new_storage_metadata.id
-                await store_alias_mapping(alias, 'rq', id, configuration)
+            # Check if there is pre-existing alias mapping in the default KVS.
+            async with Alias(storage_type=RequestQueue, alias=alias, configuration=configuration) as _alias:
+                id = await _alias.resolve_id()
+
+                # There was no pre-existing alias in the mapping.
+                # Create a new unnamed storage and store the mapping.
+                if id is None:
+                    new_storage_metadata = RequestQueueMetadata.model_validate(
+                        await apify_rqs_client.get_or_create(),
+                    )
+                    id = new_storage_metadata.id
+                    await _alias.store_mapping(storage_id=id)
 
         # If name is provided, get or create the storage by name.
         elif name:

src/apify/storage_clients/_apify/_storage_client.py

@@ -9,36 +9,47 @@
 from ._dataset_client import ApifyDatasetClient
 from ._key_value_store_client import ApifyKeyValueStoreClient
 from ._request_queue_client import ApifyRequestQueueClient
+from ._utils import Alias
+from apify._configuration import Configuration as ApifyConfiguration
 from apify._utils import docs_group
 
 if TYPE_CHECKING:
-    from crawlee.configuration import Configuration
+    from collections.abc import Hashable
+
+    from crawlee.configuration import Configuration as CrawleeConfiguration
 
 
 @docs_group('Storage clients')
 class ApifyStorageClient(StorageClient):
     """Apify storage client."""
 
+    # This class breaches Liskov Substitution Principle. It requires specialized Configuration compared to its parent.
+    _lsp_violation_error_message_template = (
+        'Expected "configuration" to be an instance of "apify.Configuration", but got {} instead.'
+    )
+
+    @override
+    def get_additional_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
+        if isinstance(configuration, ApifyConfiguration):
+            if configuration.api_base_url is None or configuration.token is None:
+                raise ValueError("'Configuration.api_base_url' and 'Configuration.token' must be set.")
+            return Alias.get_additional_cache_key(configuration)
+        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
+
     @override
     async def create_dataset_client(
         self,
         *,
         id: str | None = None,
         name: str | None = None,
         alias: str | None = None,
-        configuration: Configuration | None = None,
+        configuration: CrawleeConfiguration | None = None,
     ) -> ApifyDatasetClient:
-        # Import here to avoid circular imports.
-        from apify import Configuration as ApifyConfiguration  # noqa: PLC0415
-
         configuration = configuration or ApifyConfiguration.get_global_configuration()
         if isinstance(configuration, ApifyConfiguration):
             return await ApifyDatasetClient.open(id=id, name=name, alias=alias, configuration=configuration)
 
-        raise TypeError(
-            f'Expected "configuration" to be an instance of "apify.Configuration", '
-            f'but got {type(configuration).__name__} instead.'
-        )
+        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
 
     @override
     async def create_kvs_client(
@@ -47,19 +58,13 @@ async def create_kvs_client(
         id: str | None = None,
         name: str | None = None,
         alias: str | None = None,
-        configuration: Configuration | None = None,
+        configuration: CrawleeConfiguration | None = None,
     ) -> ApifyKeyValueStoreClient:
-        # Import here to avoid circular imports.
-        from apify import Configuration as ApifyConfiguration  # noqa: PLC0415
-
         configuration = configuration or ApifyConfiguration.get_global_configuration()
         if isinstance(configuration, ApifyConfiguration):
             return await ApifyKeyValueStoreClient.open(id=id, name=name, alias=alias, configuration=configuration)
 
-        raise TypeError(
-            f'Expected "configuration" to be an instance of "apify.Configuration", '
-            f'but got {type(configuration).__name__} instead.'
-        )
+        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
 
     @override
     async def create_rq_client(
@@ -68,16 +73,10 @@ async def create_rq_client(
         id: str | None = None,
         name: str | None = None,
         alias: str | None = None,
-        configuration: Configuration | None = None,
+        configuration: CrawleeConfiguration | None = None,
     ) -> ApifyRequestQueueClient:
-        # Import here to avoid circular imports.
-        from apify import Configuration as ApifyConfiguration  # noqa: PLC0415
-
         configuration = configuration or ApifyConfiguration.get_global_configuration()
         if isinstance(configuration, ApifyConfiguration):
             return await ApifyRequestQueueClient.open(id=id, name=name, alias=alias, configuration=configuration)
 
-        raise TypeError(
-            f'Expected "configuration" to be an instance of "apify.Configuration", '
-            f'but got {type(configuration).__name__} instead.'
-        )
+        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))