Changelog, add spider to some log messages

Author: hcoura

docs/news.rst

@@ -3,6 +3,14 @@
 Changes
 =======
 
+v1.5.1 (2019-05-21)
+-------------------
+
+- Remove username and password from settings since it's removed from crawlera.
+- Include affected spider in logs.
+- Handle situations when crawlera is restarted and reply with 407's for a few minutes
+  by retrying the requests with a exponential backoff system.
+
 v1.5.0 (2019-01-23)
 -------------------
 

scrapy_crawlera/middleware.py

@@ -179,10 +179,13 @@ def process_response(self, request, response, spider):
             # we must retry
             retries = response.meta.get('crawlera_auth_retry_times', 0)
             if retries < self.max_auth_retry_times:
-                return self._retry_auth(response, request)
+                return self._retry_auth(response, request, spider)
             else:
-                logging.warning("Max retries for authentication issues reached, please check"
-                                "auth information settings")
+                logging.warning(
+                    "Max retries for authentication issues reached, please check auth"
+                    " information settings",
+                    extra={'spider': self.spider},
+                )
 
         if self._is_banned(response):
             self._bans[key] += 1
@@ -213,8 +216,11 @@ def process_exception(self, request, exception, spider):
             self._clear_dns_cache()
             self._set_custom_delay(request, self.connection_refused_delay)
 
-    def _retry_auth(self, response, request):
-        logging.warning("Retrying crawlera request for authentication issue")
+    def _retry_auth(self, response, request, spider):
+        logging.warning(
+            "Retrying crawlera request for authentication issue",
+            extra={'spider': self.spider},
+        )
         retries = response.meta.get('crawlera_auth_retry_times', 0) + 1
         retryreq = request.copy()
         retryreq.meta['crawlera_auth_retry_times'] = retries