Merge pull request #150 from scrapinghub/scrapy-210

Bugfix, compatibility with scrapy 2.10: Do not pass spider to engine crawl

Author: pawelmhm

.github/workflows/main.yml

@@ -13,8 +13,8 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
-        scrapy-version: ["2.5.1", "2.1.0"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        scrapy-version: ["2.10", "2.9"]
 
     steps:
     - uses: actions/checkout@v2

scrapyrt/core.py

@@ -178,7 +178,7 @@ def spider_idle(self, spider):
                 self.crawler.spider, "modify_realtime_request", None)
             if callable(modify_request):
                 self.request = modify_request(self.request)
-            spider.crawler.engine.crawl(self.request, spider)
+            spider.crawler.engine.crawl(self.request)
             self._request_scheduled = True
             raise DontCloseSpider
 

tests/test_crawl_manager.py

@@ -104,7 +104,7 @@ def test_spider_opened(self):
         self.assertIsNone(self.crawl_manager.request.callback)
         self._call_spider_idle()
         self.crawler.engine.crawl.assert_called_once_with(
-            self.crawl_manager.request, self.spider)
+            self.crawl_manager.request)
         self.assertNotEqual(self.request, self.crawl_manager.request)
         self.assertEquals(
             self.crawl_manager.request.callback, self.spider.parse_something)
@@ -127,15 +127,15 @@ def modify_realtime_request(request):
         self.spider.modify_realtime_request = modify_realtime_request
         self._call_spider_idle()
         self.crawler.engine.crawl.assert_called_once_with(
-            self.crawl_manager.request, self.spider)
+            self.crawl_manager.request)
         self.assertEqual(self.crawl_manager.request.method, 'POST')
         self.assertEqual(self.crawl_manager.request.meta['foo'], 'bar')
 
     def test_modify_realtime_request_is_not_callable(self):
         self.spider.modify_realtime_request = None
         self._call_spider_idle()
         self.crawler.engine.crawl.assert_called_once_with(
-            self.crawl_manager.request, self.spider)
+            self.crawl_manager.request)
         self.assertNotEqual(self.request, self.crawl_manager.request)
 
     def test_pass_wrong_spider_errback(self):