Skip to content

Commit 0d16af0

Browse files
committed
Review comments
1 parent 8861c5e commit 0d16af0

File tree

4 files changed

+18
-44
lines changed

4 files changed

+18
-44
lines changed

src/apify/_actor.py

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,6 @@
2525
EventPersistStateData,
2626
EventSystemInfoData,
2727
)
28-
from crawlee.storage_clients import FileSystemStorageClient
2928

3029
from apify._charging import ChargeResult, ChargingManager, ChargingManagerImplementation
3130
from apify._configuration import Configuration
@@ -258,24 +257,17 @@ def _storage_client(self) -> SmartApifyStorageClient:
258257
return implicit_storage_client
259258

260259
# User set something in the service locator.
261-
storage_client = service_locator.get_storage_client()
262-
if isinstance(storage_client, SmartApifyStorageClient):
260+
explicit_storage_client = service_locator.get_storage_client()
261+
if isinstance(explicit_storage_client, SmartApifyStorageClient):
263262
# The client was manually set to the right type in the service locator. This is the explicit way.
264-
return storage_client
265-
266-
if isinstance(storage_client, ApifyStorageClient):
267-
# The cloud storage client was manually set in the service locator.
268-
return SmartApifyStorageClient(cloud_storage_client=storage_client)
269-
270-
# The local storage client was manually set in the service locator
271-
if type(storage_client) is FileSystemStorageClient:
272-
self.log.warning(
273-
f'Using {FileSystemStorageClient.__module__}.{FileSystemStorageClient.__name__} in Actor context is not'
274-
f' recommended and can lead to problems with reading the input file. Use '
275-
f'`apify.storage_clients.FileSystemStorageClient` instead.'
276-
)
263+
return explicit_storage_client
277264

278-
return SmartApifyStorageClient(cloud_storage_client=ApifyStorageClient(), local_storage_client=storage_client)
265+
raise RuntimeError(
266+
'The storage client in the service locator has to be instance of SmartApifyStorageClient. If you want to '
267+
'set the storage client manually you have to call '
268+
'`service_locator.set_storage_client(SmartApifyStorageClient(...))` before entering Actor context or '
269+
'awaiting `Actor.init`.'
270+
)
279271

280272
async def init(self) -> None:
281273
"""Initialize the Actor instance.

src/apify/storage_clients/_apify/_utils.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ async def store_mapping(self, storage_id: str) -> None:
107107
# Update in-memory mapping
108108
(await self._get_alias_map())[self._storage_key] = storage_id
109109
if not Configuration.get_global_configuration().is_at_home:
110-
logging.getLogger(__name__).warning(
110+
logging.getLogger(__name__).debug(
111111
'AliasResolver storage limited retention is only supported on Apify platform. Storage is not exported.'
112112
)
113113
return

src/apify/storage_clients/_smart_apify/_storage_client.py

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
from __future__ import annotations
22

3-
from functools import cached_property
43
from typing import TYPE_CHECKING
54

65
from typing_extensions import override
@@ -56,7 +55,7 @@ def get_suitable_storage_client(self, *, force_cloud: bool = False) -> StorageCl
5655
Args:
5756
force_cloud: If True, return `cloud_storage_client`.
5857
"""
59-
if self._is_at_home:
58+
if ApifyConfiguration.get_global_configuration().is_at_home:
6059
return self._cloud_storage_client
6160

6261
configuration = ApifyConfiguration.get_global_configuration()
@@ -72,7 +71,7 @@ def get_suitable_storage_client(self, *, force_cloud: bool = False) -> StorageCl
7271

7372
@override
7473
def get_additional_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
75-
if self._is_at_home:
74+
if ApifyConfiguration.get_global_configuration().is_at_home:
7675
if isinstance(configuration, ApifyConfiguration):
7776
return self._cloud_storage_client.get_additional_cache_key(configuration)
7877
raise TypeError('Expecting ApifyConfiguration')
@@ -117,8 +116,3 @@ async def create_rq_client(
117116
return await self.get_suitable_storage_client().create_rq_client(
118117
id=id, name=id, alias=alias, configuration=configuration
119118
)
120-
121-
@cached_property
122-
def _is_at_home(self) -> bool:
123-
configuration = ApifyConfiguration.get_global_configuration()
124-
return configuration.is_at_home

tests/integration/test_apify_storages.py

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ async def test_aliases_not_stored_on_platform_when_local(
6767
) -> None:
6868
"""Test that default Apify storage used locally is not persisting aliases to Apify based default KVS."""
6969
service_locator.set_configuration(Configuration(token=apify_token))
70-
service_locator.set_storage_client(ApifyStorageClient())
7170
async with Actor(configure_logging=False):
7271
await storage_type.open(alias='test')
7372
default_kvs = await Actor.open_key_value_store(force_cloud=True)
@@ -111,23 +110,12 @@ async def test_actor_full_explicit_storage_init_same_client(apify_token: str) ->
111110
async def test_actor_partial_explicit_cloud_storage_init(apify_token: str) -> None:
112111
service_locator.set_configuration(Configuration(token=apify_token))
113112
service_locator.set_storage_client(ApifyStorageClient(request_queue_access='shared'))
114-
async with Actor:
115-
# If service locator was already set with ApifyStorageClient, the actor will use it as cloud_storage_client of
116-
# SmartApifyStorageClient
117-
assert await Actor.open_dataset() is not await Actor.open_dataset(force_cloud=True)
118-
assert await Actor.open_key_value_store() is not await Actor.open_key_value_store(force_cloud=True)
119-
assert await Actor.open_request_queue() is not await Actor.open_request_queue(force_cloud=True)
120-
121-
122-
async def test_actor_partial_explicit_local_storage_init(apify_token: str) -> None:
123-
service_locator.set_configuration(Configuration(token=apify_token))
124-
service_locator.set_storage_client(MemoryStorageClient())
125-
async with Actor:
126-
# If service locator was already set with non-ApifyStorageClient, the actor will use it as local_storage_client
127-
# of SmartApifyStorageClient
128-
assert await Actor.open_dataset() is not await Actor.open_dataset(force_cloud=True)
129-
assert await Actor.open_key_value_store() is not await Actor.open_key_value_store(force_cloud=True)
130-
assert await Actor.open_request_queue() is not await Actor.open_request_queue(force_cloud=True)
113+
with pytest.raises(
114+
RuntimeError, match=r'^The storage client in the service locator has to be instance of SmartApifyStorageClient'
115+
):
116+
async with Actor:
117+
# If service locator was explicitly set to something different than SmartApifyStorageClient, raise an error.
118+
...
131119

132120

133121
async def test_actor_implicit_storage_init(apify_token: str) -> None:

0 commit comments

Comments
 (0)