Merge pull request #70 from scrapy-plugins/exponential-backoff-noproxies

Add exponential backoff when No Available Proxies

Author: hcoura

docs/settings.rst

@@ -64,6 +64,20 @@ CRAWLERA_DEFAULT_HEADERS
 
 Default: ``{}``
 
-Default headers added only to crawlera requests. Headers defined on ``DEFAULT_REQUEST_HEADERS`` will take precedence as long as the ``CrawleraMiddleware`` is placed after the ``DefaultHeadersMiddleware``*. Headers set on the requests have precedence over the two settings.
+Default headers added only to crawlera requests. Headers defined on ``DEFAULT_REQUEST_HEADERS`` will take precedence as long as the ``CrawleraMiddleware`` is placed after the ``DefaultHeadersMiddleware``. Headers set on the requests have precedence over the two settings.
 
-*This is the default behavior, ``DefaultHeadersMiddleware`` default priority is ``400`` and we recommend ``CrawleraMiddleware`` priority to be ``610``
+* This is the default behavior, ``DefaultHeadersMiddleware`` default priority is ``400`` and we recommend ``CrawleraMiddleware`` priority to be ``610``
+
+CRAWLERA_BACKOFF_STEP
+-----------------------
+
+Default: ``15``
+
+Step size used for calculating exponential backoff according to the formula: ``random.uniform(0, min(max, step * 2 ** attempt))``.
+
+CRAWLERA_BACKOFF_MAX
+-----------------------
+
+Default: ``180``
+
+Max value for exponential backoff as showed in the formula above.

scrapy_crawlera/middleware.py

@@ -10,6 +10,8 @@
 from scrapy.exceptions import ScrapyDeprecationWarning
 from twisted.internet.error import ConnectionRefusedError, ConnectionDone
 
+from scrapy_crawlera.utils import exp_backoff
+
 
 class CrawleraMiddleware(object):
 
@@ -22,6 +24,9 @@ class CrawleraMiddleware(object):
     preserve_delay = False
     header_prefix = 'X-Crawlera-'
     conflicting_headers = ('X-Crawlera-Profile', 'X-Crawlera-UA')
+    backoff_step = 15
+    backoff_max = 180
+    exp_backoff = None
 
     _settings = [
         ('apikey', str),
@@ -31,6 +36,8 @@ class CrawleraMiddleware(object):
         ('maxbans', int),
         ('download_timeout', int),
         ('preserve_delay', bool),
+        ('backoff_step', int),
+        ('backoff_max', int),
     ]
 
     def __init__(self, crawler):
@@ -66,6 +73,7 @@ def open_spider(self, spider):
                 "To avoid this behaviour you can use the CRAWLERA_PRESERVE_DELAY setting but keep in mind that this may slow down the crawl significantly")
 
         self._headers = self.crawler.settings.get('CRAWLERA_DEFAULT_HEADERS', {}).items()
+        self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
 
     def _settings_get(self, type_, *a, **kw):
         if type_ is int:
@@ -140,11 +148,23 @@ def _is_banned(self, response):
             response.headers.get('X-Crawlera-Error') == b'banned'
         )
 
+    def _is_no_available_proxies(self, response):
+        return (
+            response.status == self.ban_code and
+            response.headers.get('X-Crawlera-Error') == b'noslaves'
+        )
+
     def process_response(self, request, response, spider):
         if not self._is_enabled_for_request(request):
             return response
         key = self._get_slot_key(request)
         self._restore_original_delay(request)
+
+        if self._is_no_available_proxies(response):
+            self._set_custom_delay(request, next(self.exp_backoff))
+        else:
+            self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
+
         if self._is_banned(response):
             self._bans[key] += 1
             if self._bans[key] > self.maxbans:

scrapy_crawlera/utils.py

@@ -0,0 +1,16 @@
+import math
+import random
+
+from itertools import count
+
+
+def exp_backoff(step, max):
+    """ Exponential backoff time with Full Jitter """
+    # this is a numerically stable version of
+    # random.uniform(0, min(max, step * 2 ** attempt))
+    max_attempts = math.log(max / step, 2)
+    for attempt in count(0, 1):
+        if attempt <= max_attempts:
+            yield random.uniform(0, step * 2 ** attempt)
+        else:
+            yield random.uniform(0, max)

tests/test_crawlera.py

@@ -14,6 +14,8 @@
 from scrapy_crawlera import CrawleraMiddleware
 import os
 
+from scrapy_crawlera.utils import exp_backoff
+
 
 class MockedSlot(object):
 
@@ -223,15 +225,6 @@ def test_delay_adjustment(self):
         slot = MockedSlot(self.spider.download_delay)
         crawler.engine.downloader.slots[slot_key] = slot
 
-        # no ban
-        req = Request(url, meta={'download_slot': slot_key})
-        headers = {'X-Crawlera-Error': 'no_proxies'}
-        res = Response(
-            ban_url, status=self.bancode, headers=headers, request=req)
-        mw.process_response(req, res, self.spider)
-        self.assertEqual(slot.delay, delay)
-        self.assertEqual(self.spider.download_delay, delay)
-
         # ban without retry-after
         req = Request(url, meta={'download_slot': slot_key})
         headers = {'X-Crawlera-Error': 'banned'}
@@ -473,7 +466,66 @@ def test_is_banned(self):
         req = self._make_fake_request(self.spider, crawlera_enabled=True)
         res = Response(req.url, status=200)
         self.assertFalse(mw._is_banned(res))
-        res = Response(req.url, status=503, headers={'X-Crawlera-Error': 'no_proxies'})
+        res = Response(req.url, status=503, headers={'X-Crawlera-Error': 'noslaves'})
         self.assertFalse(mw._is_banned(res))
         res = Response(req.url, status=503, headers={'X-Crawlera-Error': 'banned'})
         self.assertTrue(mw._is_banned(res))
+
+    @patch('random.uniform')
+    def test_noslaves_delays(self, random_uniform_patch):
+        # mock random.uniform to just return the max delay
+        random_uniform_patch.side_effect = lambda x, y: y
+
+        slot_key = 'www.scrapytest.org'
+        url = 'http://www.scrapytest.org'
+        ban_url = 'http://ban.me'
+        max_delay = 70
+        backoff_step = 15
+        default_delay = 0
+
+        self.settings['CRAWLERA_BACKOFF_STEP'] = backoff_step
+        self.settings['CRAWLERA_BACKOFF_MAX'] = max_delay
+
+        self.spider.crawlera_enabled = True
+        crawler = self._mock_crawler(self.spider, self.settings)
+        mw = self.mwcls.from_crawler(crawler)
+        mw.open_spider(self.spider)
+        mw.noslaves_max_delay = max_delay
+
+        slot = MockedSlot()
+        crawler.engine.downloader.slots[slot_key] = slot
+
+        noslaves_req = Request(url, meta={'download_slot': slot_key})
+        headers = {'X-Crawlera-Error': 'noslaves'}
+        noslaves_res = Response(
+            ban_url, status=self.bancode, headers=headers, request=noslaves_req)
+
+        # delays grow exponentially
+        mw.process_response(noslaves_req, noslaves_res, self.spider)
+        self.assertEqual(slot.delay, backoff_step)
+
+        mw.process_response(noslaves_req, noslaves_res, self.spider)
+        self.assertEqual(slot.delay, backoff_step * 2 ** 1)
+
+        mw.process_response(noslaves_req, noslaves_res, self.spider)
+        self.assertEqual(slot.delay, backoff_step * 2 ** 2)
+
+        mw.process_response(noslaves_req, noslaves_res, self.spider)
+        self.assertEqual(slot.delay, max_delay)
+
+        # other responses reset delay
+        ban_req = Request(url, meta={'download_slot': slot_key})
+        ban_headers = {'X-Crawlera-Error': 'banned'}
+        ban_res = Response(
+            ban_url, status=self.bancode, headers=ban_headers, request=ban_req)
+        mw.process_response(ban_req, ban_res, self.spider)
+        self.assertEqual(slot.delay, default_delay)
+
+        mw.process_response(noslaves_req, noslaves_res, self.spider)
+        self.assertEqual(slot.delay, backoff_step)
+
+        good_req = Request(url, meta={'download_slot': slot_key})
+        good_res = Response(
+            url, status=200, request=good_req)
+        mw.process_response(good_req, good_res, self.spider)
+        self.assertEqual(slot.delay, default_delay)