Skip to content

Commit 7d1fc84

Browse files
committed
Review comments
1 parent 47f84eb commit 7d1fc84

File tree

8 files changed

+49
-52
lines changed

8 files changed

+49
-52
lines changed

src/apify/_actor.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ def _finalize_implicit_configuration(self) -> Configuration:
221221
return self.configuration
222222

223223
def _finalize_implicit_local_storage_client(self) -> StorageClient:
224-
"""Set implicit local storage client in the actor and return it.
224+
"""Set implicit local storage client in the Actor and return it.
225225
226226
Changing Actor or service_locator storage client after this method was run is not possible.
227227
"""
@@ -863,7 +863,7 @@ async def start(
863863
return ActorRun.model_validate(api_result)
864864

865865
def _get_remaining_time(self) -> timedelta | None:
866-
"""Get time remaining from the actor timeout. Returns `None` if not on an Apify platform."""
866+
"""Get time remaining from the Actor timeout. Returns `None` if not on an Apify platform."""
867867
if self.is_at_home() and self.configuration.timeout_at:
868868
return self.configuration.timeout_at - datetime.now(tz=timezone.utc)
869869

@@ -1077,7 +1077,7 @@ async def metamorph(
10771077
if not custom_after_sleep:
10781078
custom_after_sleep = self.configuration.metamorph_after_sleep
10791079

1080-
# If is_at_home() is True, config.actor_run_id is always set
1080+
# If is_at_home() is True, configuration.actor_run_id is always set
10811081
if not self.configuration.actor_run_id:
10821082
raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
10831083

@@ -1229,7 +1229,7 @@ async def create_proxy_configuration(
12291229
self,
12301230
*,
12311231
actor_proxy_input: dict
1232-
| None = None, # this is the raw proxy input from the actor run input, it is not spread or snake_cased in here
1232+
| None = None, # this is the raw proxy input from the Actor run input, it is not spread or snake_cased in here
12331233
password: str | None = None,
12341234
groups: list[str] | None = None,
12351235
country_code: str | None = None,

src/apify/storage_clients/_apify/_dataset_client.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
1414
from crawlee.storages import Dataset
1515

16-
from apify.storage_clients._apify._utils import Alias
16+
from ._utils import AliasResolver
1717

1818
if TYPE_CHECKING:
1919
from collections.abc import AsyncIterator
@@ -129,7 +129,7 @@ async def open(
129129

130130
if alias:
131131
# Check if there is pre-existing alias mapping in the default KVS.
132-
async with Alias(storage_type=Dataset, alias=alias, configuration=configuration) as _alias:
132+
async with AliasResolver(storage_type=Dataset, alias=alias, configuration=configuration) as _alias:
133133
id = await _alias.resolve_id()
134134

135135
# There was no pre-existing alias in the mapping.

src/apify/storage_clients/_apify/_key_value_store_client.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
from crawlee.storages import KeyValueStore
1414

1515
from ._models import ApifyKeyValueStoreMetadata, KeyValueStoreListKeysPage
16-
from ._utils import Alias
16+
from ._utils import AliasResolver
1717
from apify._crypto import create_hmac_signature
1818

1919
if TYPE_CHECKING:
@@ -120,7 +120,7 @@ async def open(
120120

121121
if alias:
122122
# Check if there is pre-existing alias mapping in the default KVS.
123-
async with Alias(storage_type=KeyValueStore, alias=alias, configuration=configuration) as _alias:
123+
async with AliasResolver(storage_type=KeyValueStore, alias=alias, configuration=configuration) as _alias:
124124
id = await _alias.resolve_id()
125125

126126
# There was no pre-existing alias in the mapping.

src/apify/storage_clients/_apify/_request_queue_client.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
from crawlee.storages import RequestQueue
2020

2121
from ._models import CachedRequest, ProlongRequestLockResponse, RequestQueueHead
22-
from ._utils import Alias
22+
from ._utils import AliasResolver
2323
from apify import Request
2424

2525
if TYPE_CHECKING:
@@ -195,7 +195,7 @@ async def open(
195195

196196
if alias:
197197
# Check if there is pre-existing alias mapping in the default KVS.
198-
async with Alias(storage_type=RequestQueue, alias=alias, configuration=configuration) as _alias:
198+
async with AliasResolver(storage_type=RequestQueue, alias=alias, configuration=configuration) as _alias:
199199
id = await _alias.resolve_id()
200200

201201
# There was no pre-existing alias in the mapping.

src/apify/storage_clients/_apify/_storage_client.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from ._dataset_client import ApifyDatasetClient
1010
from ._key_value_store_client import ApifyKeyValueStoreClient
1111
from ._request_queue_client import ApifyRequestQueueClient
12-
from ._utils import Alias
12+
from ._utils import hash_api_base_url_and_token
1313
from apify._configuration import Configuration as ApifyConfiguration
1414
from apify._utils import docs_group
1515

@@ -31,7 +31,7 @@ class ApifyStorageClient(StorageClient):
3131
@override
3232
def get_additional_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
3333
if isinstance(configuration, ApifyConfiguration):
34-
return Alias.get_additional_cache_key(configuration)
34+
return hash_api_base_url_and_token(configuration)
3535

3636
config_class = type(configuration)
3737
raise TypeError(

src/apify/storage_clients/_apify/_utils.py

Lines changed: 29 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
logger = getLogger(__name__)
2121

2222

23-
class Alias:
23+
class AliasResolver:
2424
"""Class for handling aliases.
2525
2626
The purpose of this is class is to ensure that alias storages are created with correct id. This is achieved by using
@@ -42,9 +42,9 @@ def __init__(
4242
) -> None:
4343
self._storage_type = storage_type
4444
self._alias = alias
45-
self._additional_cache_key = self.get_additional_cache_key(configuration)
45+
self._additional_cache_key = hash_api_base_url_and_token(configuration)
4646

47-
async def __aenter__(self) -> Alias:
47+
async def __aenter__(self) -> AliasResolver:
4848
"""Context manager to prevent race condition in alias creation."""
4949
lock = await self._get_alias_init_lock()
5050
await lock.acquire()
@@ -60,26 +60,26 @@ async def __aexit__(
6060
async def _get_alias_init_lock(cls) -> Lock:
6161
"""Get lock for controlling the creation of the alias storages.
6262
63-
The lock is shared for all instances of the Alias class.
63+
The lock is shared for all instances of the AliasResolver class.
6464
It is created in async method to ensure that some event loop is already running.
6565
"""
6666
if cls._alias_init_lock is None:
6767
cls._alias_init_lock = Lock()
6868
return cls._alias_init_lock
6969

7070
@classmethod
71-
async def get_alias_map(cls) -> dict[str, str]:
71+
async def _get_alias_map(cls) -> dict[str, str]:
7272
"""Get the aliases and storage ids mapping from the default kvs.
7373
74-
Mapping is loaded from kvs only once and is shared for all instances of the Alias class.
74+
Mapping is loaded from kvs only once and is shared for all instances of the AliasResolver class.
7575
7676
Returns:
7777
Map of aliases and storage ids.
7878
"""
7979
if not cls._alias_map:
80-
default_kvs_client = await get_default_kvs_client()
80+
default_kvs_client = await _get_default_kvs_client()
8181

82-
record = await default_kvs_client.get_record(cls.ALIAS_MAPPING_KEY)
82+
record = await default_kvs_client.get_record(cls._ALIAS_MAPPING_KEY)
8383

8484
# get_record can return {key: ..., value: ..., content_type: ...}
8585
if isinstance(record, dict):
@@ -92,24 +92,14 @@ async def get_alias_map(cls) -> dict[str, str]:
9292

9393
return cls._alias_map
9494

95-
@classmethod
96-
def get_additional_cache_key(cls, configuration: Configuration) -> str:
97-
"""Get additional cache key based on configuration.
98-
99-
Use only api_public_base_url and token as the relevant for differentiating storages.
100-
"""
101-
if configuration.api_public_base_url is None or configuration.token is None:
102-
raise ValueError("'Configuration.api_public_base_url' and 'Configuration.token' must be set.")
103-
return compute_short_hash(f'{configuration.api_public_base_url}{configuration.token}'.encode())
104-
10595
@property
106-
def storage_key(self) -> str:
96+
def _storage_key(self) -> str:
10797
"""Get a unique storage key used for storing the alias in the mapping."""
108-
return self.ALIAS_STORAGE_KEY_SEPARATOR.join(
98+
return self._ALIAS_STORAGE_KEY_SEPARATOR.join(
10999
[
110-
self.storage_type.__name__,
111-
self.alias,
112-
self.additional_cache_key,
100+
self._storage_type.__name__,
101+
self._alias,
102+
self._additional_cache_key,
113103
]
114104
)
115105

@@ -121,38 +111,38 @@ async def resolve_id(self) -> str | None:
121111
Returns:
122112
Storage id if it exists, None otherwise.
123113
"""
124-
return (await self.get_alias_map()).get(self.storage_key, None)
114+
return (await self._get_alias_map()).get(self._storage_key, None)
125115

126116
async def store_mapping(self, storage_id: str) -> None:
127117
"""Add alias and related storage id to the mapping in default kvs and local in-memory mapping."""
128118
# Update in-memory mapping
129-
(await self.get_alias_map())[self.storage_key] = storage_id
119+
(await self._get_alias_map())[self._storage_key] = storage_id
130120
if not Configuration.get_global_configuration().is_at_home:
131121
logging.getLogger(__name__).warning(
132-
'Alias storage limited retention is only supported on Apify platform. Storage is not exported.'
122+
'AliasResolver storage limited retention is only supported on Apify platform. Storage is not exported.'
133123
)
134124
return
135125

136-
default_kvs_client = await get_default_kvs_client()
126+
default_kvs_client = await _get_default_kvs_client()
137127
await default_kvs_client.get()
138128

139129
try:
140-
record = await default_kvs_client.get_record(self.ALIAS_MAPPING_KEY)
130+
record = await default_kvs_client.get_record(self._ALIAS_MAPPING_KEY)
141131

142132
# get_record can return {key: ..., value: ..., content_type: ...}
143133
if isinstance(record, dict) and 'value' in record:
144134
record = record['value']
145135

146136
# Update or create the record with the new alias mapping
147137
if isinstance(record, dict):
148-
record[self.storage_key] = storage_id
138+
record[self._storage_key] = storage_id
149139
else:
150-
record = {self.storage_key: storage_id}
140+
record = {self._storage_key: storage_id}
151141

152142
# Store the mapping back in the KVS.
153-
await default_kvs_client.set_record(self.ALIAS_MAPPING_KEY, record)
143+
await default_kvs_client.set_record(self._ALIAS_MAPPING_KEY, record)
154144
except Exception as exc:
155-
logger.warning(f'Error storing alias mapping for {self.alias}: {exc}')
145+
logger.warning(f'Error storing alias mapping for {self._alias}: {exc}')
156146

157147

158148
async def _get_default_kvs_client() -> KeyValueStoreClientAsync:
@@ -168,3 +158,10 @@ async def _get_default_kvs_client() -> KeyValueStoreClientAsync:
168158
)
169159

170160
return apify_client_async.key_value_store(key_value_store_id=configuration.default_key_value_store_id)
161+
162+
163+
def hash_api_base_url_and_token(configuration: Configuration) -> str:
164+
"""Hash configuration.api_public_base_url and configuration.token in deterministic way."""
165+
if configuration.api_public_base_url is None or configuration.token is None:
166+
raise ValueError("'Configuration.api_public_base_url' and 'Configuration.token' must be set.")
167+
return compute_short_hash(f'{configuration.api_public_base_url}{configuration.token}'.encode())

tests/integration/conftest.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
from ._utils import generate_unique_resource_name
2121
from apify import Actor
2222
from apify._models import ActorRun
23-
from apify.storage_clients._apify._utils import Alias
23+
from apify.storage_clients._apify._utils import AliasResolver
2424

2525
if TYPE_CHECKING:
2626
from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Iterator, Mapping
@@ -61,9 +61,9 @@ def _prepare_test_env() -> None:
6161
service_locator._storage_client = None
6262
service_locator.storage_instance_manager.clear_cache()
6363

64-
# Reset the Alias class state.
65-
Alias._alias_map = {}
66-
Alias._alias_init_lock = None
64+
# Reset the AliasResolver class state.
65+
AliasResolver._alias_map = {}
66+
AliasResolver._alias_init_lock = None
6767

6868
# Verify that the test environment was set up correctly.
6969
assert os.environ.get(ApifyEnvVars.LOCAL_STORAGE_DIR) == str(tmp_path)

tests/unit/conftest.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
import apify._actor
1919
import apify.log
20-
from apify.storage_clients._apify._utils import Alias
20+
from apify.storage_clients._apify._utils import AliasResolver
2121

2222
if TYPE_CHECKING:
2323
from collections.abc import Callable, Iterator
@@ -73,9 +73,9 @@ def _prepare_test_env() -> None:
7373
service_locator._storage_client = None
7474
service_locator.storage_instance_manager.clear_cache()
7575

76-
# Reset the Alias class state.
77-
Alias._alias_map = {}
78-
Alias._alias_init_lock = None
76+
# Reset the AliasResolver class state.
77+
AliasResolver._alias_map = {}
78+
AliasResolver._alias_init_lock = None
7979

8080
# Verify that the test environment was set up correctly.
8181
assert os.environ.get(ApifyEnvVars.LOCAL_STORAGE_DIR) == str(tmp_path)

0 commit comments

Comments
 (0)