fix: Fix RQ usage in Scrapy scheduler

src/apify/_actor.py

@@ -889,7 +889,7 @@ async def reboot(
         The system stops the current container and starts a new one, with the same run ID and default storages.
 
         Args:
-            event_listeners_timeout: How long should the Actor wait for Actor event listeners to finish before exiting
+            event_listeners_timeout: How long should the Actor wait for Actor event listeners to finish before exiting.
             custom_after_sleep: How long to sleep for after the reboot, to wait for the container to be stopped.
         """
         self._raise_if_not_initialized()

src/apify/scrapy/middlewares/apify_proxy.py

@@ -62,7 +62,7 @@ def from_crawler(cls: type[ApifyHttpProxyMiddleware], crawler: Crawler) -> Apify
         if use_apify_proxy is not True:
             Actor.log.warning(
                 'ApifyHttpProxyMiddleware is not going to be used. Actor input field '
-                '"proxyConfiguration.useApifyProxy" is probably set to False.'
+                '"proxyConfiguration.useApifyProxy" is set to False.'
             )
             raise NotConfigured
 
@@ -78,6 +78,11 @@ async def process_request(self, request: Request, spider: Spider) -> None:
         Raises:
             ValueError: If username and password are not provided in the proxy URL.
         """
+        # Do not use proxy for robots.txt, as it causes 403 Forbidden.
+        if request.url.endswith('/robots.txt'):
+            request.meta.pop('proxy', None)
+            return
+
         Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: request={request}, spider={spider}')
         url = await self._get_new_proxy_url()
 

src/apify/scrapy/scheduler.py

@@ -3,6 +3,8 @@
 import traceback
 from typing import TYPE_CHECKING
 
+from crawlee.storage_clients import MemoryStorageClient
+
 from apify._configuration import Configuration
 from apify.apify_storage_client import ApifyStorageClient
 
@@ -52,8 +54,15 @@ def open(self, spider: Spider) -> None:  # this has to be named "open"
         self.spider = spider
 
         async def open_queue() -> RequestQueue:
-            custom_loop_apify_client = ApifyStorageClient(configuration=Configuration.get_global_configuration())
-            return await RequestQueue.open(storage_client=custom_loop_apify_client)
+            config = Configuration.get_global_configuration()
+
+            # Use the ApifyStorageClient if the Actor is running on the Apify platform,
+            # otherwise use the MemoryStorageClient.
+            storage_client = (
+                ApifyStorageClient.from_config(config) if config.is_at_home else MemoryStorageClient.from_config(config)
+            )
+
+            return await RequestQueue.open(storage_client=storage_client)
 
         try:
             self._rq = nested_event_loop.run_until_complete(open_queue())