Skip to content

Commit 3363478

Browse files
authored
fix: Fix RQ usage in Scrapy scheduler (#385)
Relates: apify/actor-templates#303
1 parent 18ec440 commit 3363478

File tree

3 files changed

+13
-4
lines changed

3 files changed

+13
-4
lines changed

src/apify/_actor.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -889,7 +889,7 @@ async def reboot(
889889
The system stops the current container and starts a new one, with the same run ID and default storages.
890890
891891
Args:
892-
event_listeners_timeout: How long should the Actor wait for Actor event listeners to finish before exiting
892+
event_listeners_timeout: How long should the Actor wait for Actor event listeners to finish before exiting.
893893
custom_after_sleep: How long to sleep for after the reboot, to wait for the container to be stopped.
894894
"""
895895
self._raise_if_not_initialized()

src/apify/scrapy/middlewares/apify_proxy.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> Apify
6262
if use_apify_proxy is not True:
6363
Actor.log.warning(
6464
'ApifyHttpProxyMiddleware is not going to be used. Actor input field '
65-
'"proxyConfiguration.useApifyProxy" is probably set to False.'
65+
'"proxyConfiguration.useApifyProxy" is set to False.'
6666
)
6767
raise NotConfigured
6868

src/apify/scrapy/scheduler.py

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
import traceback
44
from typing import TYPE_CHECKING
55

6+
from crawlee.storage_clients import MemoryStorageClient
7+
68
from apify._configuration import Configuration
79
from apify.apify_storage_client import ApifyStorageClient
810

@@ -52,8 +54,15 @@ def open(self, spider: Spider) -> None: # this has to be named "open"
5254
self.spider = spider
5355

5456
async def open_queue() -> RequestQueue:
55-
custom_loop_apify_client = ApifyStorageClient(configuration=Configuration.get_global_configuration())
56-
return await RequestQueue.open(storage_client=custom_loop_apify_client)
57+
config = Configuration.get_global_configuration()
58+
59+
# Use the ApifyStorageClient if the Actor is running on the Apify platform,
60+
# otherwise use the MemoryStorageClient.
61+
storage_client = (
62+
ApifyStorageClient.from_config(config) if config.is_at_home else MemoryStorageClient.from_config(config)
63+
)
64+
65+
return await RequestQueue.open(storage_client=storage_client)
5766

5867
try:
5968
self._rq = nested_event_loop.run_until_complete(open_queue())

0 commit comments

Comments
 (0)