Skip to content

Commit 860b0ec

Browse files
committed
Rename access to request_queue_access
1 parent 1cc80bb commit 860b0ec

File tree

7 files changed

+26
-22
lines changed

7 files changed

+26
-22
lines changed

docs/04_upgrading/upgrading_to_v3.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ async def main():
111111
- It is now possible to have full control over which storage clients are used by the `Actor`. To make development of Actors convenient, the `Actor` has two storage clients. One that is used when running on Apify platform or when opening storages with `force_cloud=True` and the other client that is used when running outside the Apify platform. The `Actor` has reasonable defaults and for the majority of use-cases there is no need to change it. However, if you need to use a different storage client, you can set it up before entering `Actor` context through `service_locator`.
112112

113113
**Now (v3.0):**
114+
114115
```python
115116
from crawlee import service_locator
116117
from apify.storage_clients import ApifyStorageClient, ApifyHybridStorageClient, MemoryStorageClient
@@ -120,7 +121,7 @@ from apify import Actor
120121
async def main():
121122
service_locator.set_storage_client(
122123
ApifyHybridStorageClient(
123-
cloud_storage_client=ApifyStorageClient(access="single"),
124+
cloud_storage_client=ApifyStorageClient(request_queue_access="single"),
124125
local_storage_client=MemoryStorageClient()
125126
)
126127
)
@@ -132,7 +133,7 @@ async def main():
132133
## The default use of optimized ApifyRequestQueueClient
133134

134135
- The default client for working with Apify platform based `RequestQueue` is now optimized and simplified client which does significantly lower amount of API calls, but does not support multiple consumers working on the same queue. It is cheaper and faster and is suitable for the majority of the use cases.
135-
- The full client is still available, but it has to be explicitly requested via `access="shared"` argument when using the `ApifyStorageClient`.
136+
- The full client is still available, but it has to be explicitly requested via `request_queue_access="shared"` argument when using the `ApifyStorageClient`.
136137

137138
**Now (v3.0):**
138139

@@ -141,9 +142,10 @@ from crawlee import service_locator
141142
from apify.storage_clients import ApifyStorageClient
142143
from apify import Actor
143144

145+
144146
async def main():
145-
# Full client that supports multiple consumers of the Apify Request Queue
146-
service_locator.set_storage_client(ApifyStorageClient(access="shared"))
147+
# Full client that supports multiple consumers of the Apify Request Queue
148+
service_locator.set_storage_client(ApifyStorageClient(request_queue_access="shared"))
147149
async with Actor:
148150
rq = await Actor.open_request_queue()
149151
```

src/apify/storage_clients/_apify/_storage_client.py

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,16 @@
2626
class ApifyStorageClient(StorageClient):
2727
"""Apify storage client."""
2828

29-
def __init__(self, *, access: Literal['single', 'shared'] = 'single') -> None:
29+
def __init__(self, *, request_queue_access: Literal['single', 'shared'] = 'single') -> None:
3030
"""Initialize the Apify storage client.
3131
3232
Args:
33-
access: If 'single', the `create_rq_client` will return `ApifyRequestQueueSingleClient`, if 'shared' it
34-
will return `ApifyRequestQueueSharedClient`.
35-
- 'single' is suitable for single consumer scenarios. It makes less API calls, is cheaper and faster.
36-
- 'shared' is suitable for multiple consumers scenarios at the cost of higher API usage.
33+
request_queue_access: If 'single', the `create_rq_client` will return `ApifyRequestQueueSingleClient`, if
34+
'shared' it will return `ApifyRequestQueueSharedClient`.
35+
- 'single' is suitable for single consumer scenarios. It makes less API calls, is cheaper and faster.
36+
- 'shared' is suitable for multiple consumers scenarios at the cost of higher API usage.
3737
"""
38-
self._access = access
38+
self._request_queue_access = request_queue_access
3939

4040
# This class breaches Liskov Substitution Principle. It requires specialized Configuration compared to its parent.
4141
_lsp_violation_error_message_template = (
@@ -96,7 +96,9 @@ async def create_rq_client(
9696
configuration = configuration or ApifyConfiguration.get_global_configuration()
9797
if isinstance(configuration, ApifyConfiguration):
9898
client: type[ApifyRequestQueueClient] = (
99-
ApifyRequestQueueSingleClient if self._access == 'single' else ApifyRequestQueueSharedClient
99+
ApifyRequestQueueSingleClient
100+
if self._request_queue_access == 'single'
101+
else ApifyRequestQueueSharedClient
100102
)
101103
return await client.open(id=id, name=name, alias=alias, configuration=configuration)
102104

src/apify/storage_clients/_hybrid_apify/_storage_client.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ def __init__(
4343
local_storage_client: Client used to communicate with the storage when not running on the Apify
4444
platform and not using `force_cloud` argument when opening storages.
4545
"""
46-
self._cloud_storage_client = cloud_storage_client or ApifyStorageClient(access='single')
46+
self._cloud_storage_client = cloud_storage_client or ApifyStorageClient(request_queue_access='single')
4747
self._local_storage_client = local_storage_client or ApifyFileSystemStorageClient()
4848

4949
def _get_suitable_storage_client(self, *, force_cloud: bool = False) -> StorageClient:

tests/integration/conftest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ async def request_queue_apify(
115115
monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_token)
116116

117117
async with Actor:
118-
rq = await RequestQueue.open(storage_client=ApifyStorageClient(access=request.param))
118+
rq = await RequestQueue.open(storage_client=ApifyStorageClient(request_queue_access=request.param))
119119
yield rq
120120
await rq.drop()
121121

tests/integration/test_actor_request_queue.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ async def apify_named_rq(
3333

3434
async with Actor:
3535
request_queue = await RequestQueue.open(
36-
name=request_queue_name, storage_client=ApifyStorageClient(access=request.param)
36+
name=request_queue_name, storage_client=ApifyStorageClient(request_queue_access=request.param)
3737
)
3838
yield request_queue
3939
await request_queue.drop()

tests/integration/test_apify_storages.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ async def test_actor_full_explicit_storage_init(apify_token: str) -> None:
4040
service_locator.set_storage_client(
4141
ApifyHybridStorageClient(
4242
local_storage_client=MemoryStorageClient(),
43-
cloud_storage_client=ApifyStorageClient(access='shared'),
43+
cloud_storage_client=ApifyStorageClient(request_queue_access='shared'),
4444
)
4545
)
4646
async with Actor():
@@ -55,8 +55,8 @@ async def test_actor_full_explicit_storage_init_same_client(apify_token: str) ->
5555
service_locator.set_configuration(Configuration(token=apify_token))
5656
service_locator.set_storage_client(
5757
ApifyHybridStorageClient(
58-
local_storage_client=ApifyStorageClient(access='shared'),
59-
cloud_storage_client=ApifyStorageClient(access='shared'),
58+
local_storage_client=ApifyStorageClient(request_queue_access='shared'),
59+
cloud_storage_client=ApifyStorageClient(request_queue_access='shared'),
6060
)
6161
)
6262
async with Actor():
@@ -69,7 +69,7 @@ async def test_actor_full_explicit_storage_init_same_client(apify_token: str) ->
6969

7070
async def test_actor_partial_explicit_cloud_storage_init(apify_token: str) -> None:
7171
service_locator.set_configuration(Configuration(token=apify_token))
72-
service_locator.set_storage_client(ApifyStorageClient(access='shared'))
72+
service_locator.set_storage_client(ApifyStorageClient(request_queue_access='shared'))
7373
async with Actor():
7474
# If service locator was already set with ApifyStorageClient, the actor will use it as cloud_storage_client of
7575
# ApifyHybridStorageClient
@@ -108,7 +108,7 @@ async def main() -> None:
108108
service_locator.set_storage_client(
109109
ApifyHybridStorageClient(
110110
local_storage_client=MemoryStorageClient(),
111-
cloud_storage_client=ApifyStorageClient(access='shared'),
111+
cloud_storage_client=ApifyStorageClient(request_queue_access='shared'),
112112
)
113113
)
114114
async with Actor():

tests/integration/test_request_queue.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1086,8 +1086,8 @@ async def test_request_queue_simple_and_full_at_the_same_time(
10861086
monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_token)
10871087

10881088
async with Actor:
1089-
rq_simple = await RequestQueue.open(storage_client=ApifyStorageClient(access='single'))
1090-
rq_full = await RequestQueue.open(storage_client=ApifyStorageClient(access='shared'))
1089+
rq_simple = await RequestQueue.open(storage_client=ApifyStorageClient(request_queue_access='single'))
1090+
rq_full = await RequestQueue.open(storage_client=ApifyStorageClient(request_queue_access='shared'))
10911091
# Opening same queue again with different ApifyStorageClient will resolve to the first client used.
10921092
assert rq_simple is rq_full
10931093
await rq_simple.drop()
@@ -1112,7 +1112,7 @@ async def test_crawler_run_request_queue_variant_stats(
11121112
monkeypatch.setenv(ApifyEnvVars.TOKEN, apify_token)
11131113
async with Actor:
11141114
requests = 5
1115-
rq = await RequestQueue.open(storage_client=ApifyStorageClient(access=access))
1115+
rq = await RequestQueue.open(storage_client=ApifyStorageClient(request_queue_access=access))
11161116
crawler = BasicCrawler(request_manager=rq)
11171117

11181118
@crawler.router.default_handler

0 commit comments

Comments
 (0)