small refactor

Author: hcoura

scrapy_crawlera/middleware.py

@@ -163,7 +163,7 @@ def process_response(self, request, response, spider):
         if self._is_no_available_proxies(response):
             self._set_custom_delay(request, next(self.exp_backoff))
         else:
-            self._reset_noslaves_delay()
+            self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
 
         if self._is_banned(response):
             self._bans[key] += 1
@@ -209,10 +209,6 @@ def _get_slot(self, request):
         key = self._get_slot_key(request)
         return key, self.crawler.engine.downloader.slots.get(key)
 
-    def _reset_noslaves_delay(self):
-        """Reset the number of attempts due to no available proxies"""
-        self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
-
     def _set_custom_delay(self, request, delay):
         """Set custom delay for slot and save original one."""
         key, slot = self._get_slot(request)

scrapy_crawlera/utils.py

@@ -10,7 +10,7 @@ def exp_backoff(step, max):
     # random.uniform(0, min(max, step * 2 ** attempt))
     max_attempts = math.log(max / step, 2)
     for attempt in count(0, 1):
-        if attempt < max_attempts:
+        if attempt <= max_attempts:
             yield random.uniform(0, step * 2 ** attempt)
         else:
             yield random.uniform(0, max)

tests/test_crawlera.py

@@ -503,6 +503,7 @@ def test_noslaves_delays(self, random_uniform_patch):
         # delays grow exponentially
         mw.process_response(noslaves_req, noslaves_res, self.spider)
         self.assertEqual(slot.delay, backoff_step)
+
         mw.process_response(noslaves_req, noslaves_res, self.spider)
         self.assertEqual(slot.delay, backoff_step * 2 ** 1)