Check Crawlera Error Header when checking bans (#64)

* Check Crawlera Error Header when checking bans

* fix tests for new feature

* Added trailing comma and remove empty line

Author: VMRuiz

scrapy_crawlera/middleware.py

@@ -134,12 +134,18 @@ def process_request(self, request, spider):
         else:
             self._clean_crawlera_headers(request)
 
+    def _is_banned(self, response):
+        return (
+            response.status == self.ban_code and
+            response.headers.get('X-Crawlera-Error') == b'banned'
+        )
+
     def process_response(self, request, response, spider):
         if not self._is_enabled_for_request(request):
             return response
         key = self._get_slot_key(request)
         self._restore_original_delay(request)
-        if response.status == self.ban_code:
+        if self._is_banned(response):
             self._bans[key] += 1
             if self._bans[key] > self.maxbans:
                 self.crawler.engine.close_spider(spider, 'banned')

tests/test_crawlera.py

@@ -102,7 +102,11 @@ def _assert_enabled(self, spider,
         # check for not banning before maxbans for bancode
         for x in range(maxbans + 1):
             self.assertEqual(crawler.engine.fake_spider_closed_result, None)
-            res = Response('http://ban.me/%d' % x, status=self.bancode)
+            res = Response(
+                'http://ban.me/%d' % x,
+                status=self.bancode,
+                headers={'X-Crawlera-Error': 'banned'},
+            )
             assert mw.process_response(req, res, spider) is res
 
         # max bans reached and close_spider called
@@ -219,15 +223,30 @@ def test_delay_adjustment(self):
         slot = MockedSlot(self.spider.download_delay)
         crawler.engine.downloader.slots[slot_key] = slot
 
-        # ban
+        # no ban
         req = Request(url, meta={'download_slot': slot_key})
-        res = Response(ban_url, status=self.bancode, request=req)
+        headers = {'X-Crawlera-Error': 'no_proxies'}
+        res = Response(
+            ban_url, status=self.bancode, headers=headers, request=req)
         mw.process_response(req, res, self.spider)
         self.assertEqual(slot.delay, delay)
         self.assertEqual(self.spider.download_delay, delay)
 
+        # ban without retry-after
+        req = Request(url, meta={'download_slot': slot_key})
+        headers = {'X-Crawlera-Error': 'banned'}
+        res = Response(
+            ban_url, status=self.bancode, headers=headers, request=req)
+        mw.process_response(req, res, self.spider)
+        self.assertEqual(slot.delay, delay)
+        self.assertEqual(self.spider.download_delay, delay)
+
+        # ban with retry-after
         retry_after = 1.5
-        headers = {'retry-after': str(retry_after)}
+        headers = {
+            'retry-after': str(retry_after),
+            'X-Crawlera-Error': 'banned'
+        }
         res = Response(
             ban_url, status=self.bancode, headers=headers, request=req)
         mw.process_response(req, res, self.spider)
@@ -335,8 +354,12 @@ def test_stats(self):
         assert mw.process_response(req, res, spider) is res
         self.assertEqual(crawler.stats.get_value('crawlera/response'), 2)
         self.assertEqual(crawler.stats.get_value('crawlera/response/status/{}'.format(mw.ban_code)), 1)
-        self.assertEqual(crawler.stats.get_value('crawlera/response/banned'), 1)
         self.assertEqual(crawler.stats.get_value('crawlera/response/error/somethingbad'), 1)
+        res = Response(req.url, status=mw.ban_code, headers={'X-Crawlera-Error': 'banned'})
+        assert mw.process_response(req, res, spider) is res
+        self.assertEqual(crawler.stats.get_value('crawlera/response'), 3)
+        self.assertEqual(crawler.stats.get_value('crawlera/response/status/{}'.format(mw.ban_code)), 2)
+        self.assertEqual(crawler.stats.get_value('crawlera/response/banned'), 1)
 
     def _make_fake_request(self, spider, crawlera_enabled):
         spider.crawlera_enabled = crawlera_enabled
@@ -441,3 +464,16 @@ def test_dont_proxy_false_does_nothing(self):
         req.meta['dont_proxy'] = False
         assert mw.process_request(req, spider) is None
         self.assertIsNotNone(req.meta.get('proxy'))
+
+    def test_is_banned(self):
+        self.spider.crawlera_enabled = True
+        crawler = self._mock_crawler(self.spider, self.settings)
+        mw = self.mwcls.from_crawler(crawler)
+        mw.open_spider(self.spider)
+        req = self._make_fake_request(self.spider, crawlera_enabled=True)
+        res = Response(req.url, status=200)
+        self.assertFalse(mw._is_banned(res))
+        res = Response(req.url, status=503, headers={'X-Crawlera-Error': 'no_proxies'})
+        self.assertFalse(mw._is_banned(res))
+        res = Response(req.url, status=503, headers={'X-Crawlera-Error': 'banned'})
+        self.assertTrue(mw._is_banned(res))