@@ -27,10 +27,17 @@ def __init__(self, *, request_queue_access: Literal['single', 'shared'] = 'singl
27
27
"""Initialize the Apify storage client.
28
28
29
29
Args:
30
- request_queue_access: If 'single', the `create_rq_client` will return `ApifyRequestQueueSingleClient`, if
31
- 'shared' it will return `ApifyRequestQueueSharedClient`.
30
+ request_queue_access: Controls the implementation of the request queue client based on expected scenario:
32
31
- 'single' is suitable for single consumer scenarios. It makes less API calls, is cheaper and faster.
33
32
- 'shared' is suitable for multiple consumers scenarios at the cost of higher API usage.
33
+ Detailed constraints for the 'single' access type:
34
+ - Only one client is consuming the request queue at the time.
35
+ - Multiple producers can put requests to the queue, but their forefront requests are not guaranteed to
36
+ be handled so quickly as this client does not aggressively fetch the forefront and relies on local
37
+ head estimation.
38
+ - Requests are only added to the queue, never deleted by other clients. (Marking as handled is ok.)
39
+ - Other producers can add new requests, but not modify existing ones.
40
+ (Modifications would not be included in local cache)
34
41
"""
35
42
self ._request_queue_access = request_queue_access
36
43
@@ -42,8 +49,8 @@ def __init__(self, *, request_queue_access: Literal['single', 'shared'] = 'singl
42
49
@override
43
50
def get_storage_client_cache_key (self , configuration : CrawleeConfiguration ) -> Hashable :
44
51
if isinstance (configuration , ApifyConfiguration ):
45
- # Current design does not support opening exactly same queue with full and simple client at the same time,
46
- # due to default and unnamed storages. Whichever client variation gets used first, wins.
52
+ # It is not supported to open exactly same queue with 'single' and 'shared' client at the same time.
53
+ # Whichever client variation gets used first, wins.
47
54
return super ().get_storage_client_cache_key (configuration ), hash_api_base_url_and_token (configuration )
48
55
49
56
config_class = type (configuration )
0 commit comments