Merge pull request #74 from scrapy-plugins/retry_on_407

Retry 407 crawlera errors with exp backoff up to 10 times

Author: hcoura

scrapy_crawlera/middleware.py

@@ -27,6 +27,7 @@ class CrawleraMiddleware(object):
     backoff_step = 15
     backoff_max = 180
     exp_backoff = None
+    max_auth_retry_times = 10
 
     _settings = [
         ('apikey', str),
@@ -156,17 +157,33 @@ def _is_no_available_proxies(self, response):
             response.headers.get('X-Crawlera-Error') == b'noslaves'
         )
 
+    def _is_auth_error(self, response):
+        return (
+            response.status == 407 and
+            response.headers.get('X-Crawlera-Error') == b'bad_proxy_auth'
+        )
+
     def process_response(self, request, response, spider):
         if not self._is_enabled_for_request(request):
             return response
         key = self._get_slot_key(request)
         self._restore_original_delay(request)
 
-        if self._is_no_available_proxies(response):
+        if self._is_no_available_proxies(response) or self._is_auth_error(response):
             self._set_custom_delay(request, next(self.exp_backoff))
         else:
             self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
 
+        if self._is_auth_error(response):
+            # When crawlera has issues it might not be able to authenticate users
+            # we must retry
+            retries = response.meta.get('crawlera_auth_retry_times', 0)
+            if retries < self.max_auth_retry_times:
+                return self._retry_auth(response, request)
+            else:
+                logging.warning("Max retries for authentication issues reached, please check"
+                                "auth information settings")
+
         if self._is_banned(response):
             self._bans[key] += 1
             if self._bans[key] > self.maxbans:
@@ -196,6 +213,14 @@ def process_exception(self, request, exception, spider):
             self._clear_dns_cache()
             self._set_custom_delay(request, self.connection_refused_delay)
 
+    def _retry_auth(self, response, request):
+        logging.warning("Retrying crawlera request for authentication issue")
+        retries = response.meta.get('crawlera_auth_retry_times', 0) + 1
+        retryreq = request.copy()
+        retryreq.meta['crawlera_auth_retry_times'] = retries
+        retryreq.dont_filter = True
+        return retryreq
+
     def _clear_dns_cache(self):
         # Scrapy doesn't expire dns records by default, so we force it here,
         # so client can reconnect trough DNS failover.

tests/test_crawlera.py

@@ -27,6 +27,7 @@ class CrawleraMiddlewareTestCase(TestCase):
 
     mwcls = CrawleraMiddleware
     bancode = 503
+    auth_error_code = 407
 
     def setUp(self):
         self.spider = Spider('foo')
@@ -474,7 +475,6 @@ def test_noslaves_delays(self, random_uniform_patch):
         crawler = self._mock_crawler(self.spider, self.settings)
         mw = self.mwcls.from_crawler(crawler)
         mw.open_spider(self.spider)
-        mw.noslaves_max_delay = max_delay
 
         slot = MockedSlot()
         crawler.engine.downloader.slots[slot_key] = slot
@@ -514,6 +514,77 @@ def test_noslaves_delays(self, random_uniform_patch):
         mw.process_response(good_req, good_res, self.spider)
         self.assertEqual(slot.delay, default_delay)
 
+    @patch('random.uniform')
+    def test_auth_error_retries(self, random_uniform_patch):
+        # mock random.uniform to just return the max delay
+        random_uniform_patch.side_effect = lambda x, y: y
+
+        slot_key = 'www.scrapytest.org'
+        url = 'http://www.scrapytest.org'
+        ban_url = 'http://auth.error'
+        max_delay = 70
+        backoff_step = 15
+        default_delay = 0
+
+        self.settings['CRAWLERA_BACKOFF_STEP'] = backoff_step
+        self.settings['CRAWLERA_BACKOFF_MAX'] = max_delay
+
+        self.spider.crawlera_enabled = True
+        crawler = self._mock_crawler(self.spider, self.settings)
+        mw = self.mwcls.from_crawler(crawler)
+        mw.open_spider(self.spider)
+        mw.max_auth_retry_times = 4
+
+        slot = MockedSlot()
+        crawler.engine.downloader.slots[slot_key] = slot
+
+        auth_error_req = Request(url, meta={'download_slot': slot_key})
+        auth_error_headers = {'X-Crawlera-Error': 'bad_proxy_auth'}
+        auth_error_response = Response(
+            ban_url,
+            status=self.auth_error_code,
+            request=auth_error_req,
+            headers=auth_error_headers
+        )
+
+        # delays grow exponentially, retry times increase accordingly
+        req = mw.process_response(auth_error_req, auth_error_response, self.spider)
+        self.assertEqual(slot.delay, backoff_step)
+        retry_times = req.meta["crawlera_auth_retry_times"]
+        self.assertEqual(retry_times, 1)
+
+        auth_error_response.meta["crawlera_auth_retry_times"] = retry_times
+        req = mw.process_response(auth_error_req, auth_error_response, self.spider)
+        self.assertEqual(slot.delay, backoff_step * 2 ** 1)
+        retry_times = req.meta["crawlera_auth_retry_times"]
+        self.assertEqual(retry_times, 2)
+
+        auth_error_response.meta["crawlera_auth_retry_times"] = retry_times
+        req = mw.process_response(auth_error_req, auth_error_response, self.spider)
+        self.assertEqual(slot.delay, backoff_step * 2 ** 2)
+        retry_times = req.meta["crawlera_auth_retry_times"]
+        self.assertEqual(retry_times, 3)
+
+        auth_error_response.meta["crawlera_auth_retry_times"] = retry_times
+        req = mw.process_response(auth_error_req, auth_error_response, self.spider)
+        self.assertEqual(slot.delay, max_delay)
+        retry_times = req.meta["crawlera_auth_retry_times"]
+        self.assertEqual(retry_times, 4)
+
+        # Should return a response when after max number of retries
+        auth_error_response.meta["crawlera_auth_retry_times"] = retry_times
+        res = mw.process_response(auth_error_req, auth_error_response, self.spider)
+        self.assertIsInstance(res, Response)
+
+        # non crawlera 407 is not retried
+        non_crawlera_407_response = Response(
+            ban_url,
+            status=self.auth_error_code,
+            request=auth_error_req,
+        )
+        res = mw.process_response(auth_error_req, non_crawlera_407_response, self.spider)
+        self.assertIsInstance(res, Response)
+
     @patch('scrapy_crawlera.middleware.logging')
     def test_open_spider_logging(self, mock_logger):
         spider = self.spider