Merge branch 'adding-pre-commit' of github.com:emarondan/scrapy-zyte-smartproxy into adding-pre-commit

Author: Emmanuel Rondan

.bandit.yml

@@ -3,4 +3,4 @@ skips:
 - B311
 - B320
 - B410
-exclude_dirs: ['tests']
+exclude_dirs: ['tests']

.git-blame-ignore-revs

@@ -1,2 +1,2 @@
 # applying pre-commit hooks to the project
-05665a6fb1717ef513d7a8ac87b8eb499a64cdc9
+05665a6fb1717ef513d7a8ac87b8eb499a64cdc9

scrapy_zyte_smartproxy/middleware.py

@@ -239,7 +239,7 @@ def _translate_headers(self, request, targets_zyte_api):
                 continue
             request.headers[translation] = value = request.headers.pop(header)
             logger.warning(
-                "Translating (and dropping) header %r (%r) as %r on request %r",
+                "Translating header %r (%r) to %r on request %r",
                 header,
                 value,
                 translation,
@@ -371,8 +371,8 @@ def process_response(self, request, response, spider):
                     "retries/auth/max_reached", targets_zyte_api=targets_zyte_api
                 )
                 logger.warning(
-                    "Max retries for authentication issues reached, please check auth"
-                    " information settings",
+                    "Max retries for authentication issues reached,"
+                    "please check auth information settings",
                     extra={"spider": self.spider},
                 )
 
@@ -392,7 +392,8 @@ def process_response(self, request, response, spider):
             self._inc_stat("response/banned", targets_zyte_api=targets_zyte_api)
         else:
             self._bans[key] = 0
-        # If placed behind `RedirectMiddleware`, it would not count 3xx responses
+        # If placed behind `RedirectMiddleware`,
+        # it would not count 3xx responses
         self._inc_stat("response", targets_zyte_api=targets_zyte_api)
         self._inc_stat(
             "response/status/{}".format(response.status),
@@ -402,7 +403,8 @@ def process_response(self, request, response, spider):
             self._inc_stat("response/error", targets_zyte_api=targets_zyte_api)
             error_msg = zyte_smartproxy_error.decode("utf8")
             self._inc_stat(
-                "response/error/{}".format(error_msg), targets_zyte_api=targets_zyte_api
+                "response/error/{}".format(error_msg),
+                targets_zyte_api=targets_zyte_api,
             )
         return response
 
@@ -428,7 +430,8 @@ def _handle_not_enabled_response(self, request, response, targets_zyte_api):
             retryreq = request.copy()
             retryreq.dont_filter = True
             self._inc_stat(
-                "retries/should_have_been_enabled", targets_zyte_api=targets_zyte_api
+                "retries/should_have_been_enabled",
+                targets_zyte_api=targets_zyte_api,
             )
             return retryreq
         return response
@@ -531,7 +534,8 @@ def _clean_zyte_smartproxy_headers(self, request, targets_zyte_api=None):
                         "request is proxied with %s and not with %s, and "
                         "automatic translation is not supported for this "
                         "header. See "
-                        "https://docs.zyte.com/zyte-api/migration/zyte/smartproxy.html#parameter-mapping"  # noqa
+                        "https://docs.zyte.com/zyte-api/migration/zyte/"
+                        "smartproxy.html#parameter-mapping"
                         " to learn the right way to translate this header "
                         "manually."
                     ),
@@ -557,11 +561,13 @@ def _set_zyte_smartproxy_default_headers(self, request):
             header.decode("utf-8").lower() for header in request.headers
         ]
         if all(h.lower() in lower_case_headers for h in self.conflicting_headers):
-            # Send a general warning once, and specific urls if LOG_LEVEL = DEBUG
+            # Send a general warning once,
+            # and specific urls if LOG_LEVEL = DEBUG
             warnings.warn(
                 "The headers %s are conflicting on some of your requests. "
                 "Please check "
-                "https://docs.zyte.com/smart-proxy-manager.html#request-headers "
+                "https://docs.zyte.com/smart-proxy-manager.html"
+                "#request-headers "
                 "for more information. You can set LOG_LEVEL=DEBUG to see the "
                 "urls with problems." % str(self.conflicting_headers)
             )

tests/test_all.py

@@ -740,7 +740,9 @@ def test_is_banned(self):
         res = mw.process_response(req, res, self.spider)
         self.assertFalse(mw._is_banned(res))
         res = Response(
-            req.url, status=503, headers={"Zyte-Error": "/limits/over-global-limit"}
+            req.url,
+            status=503,
+            headers={"Zyte-Error": "/limits/over-global-limit"},
         )
         res = mw.process_response(req, res, self.spider)
         self.assertFalse(mw._is_banned(res))
@@ -754,7 +756,9 @@ def test_is_banned(self):
         res = mw.process_response(req, res, self.spider)
         self.assertTrue(mw._is_banned(res))
         res = Response(
-            req.url, status=521, headers={"Zyte-Error": "/download/internal-error"}
+            req.url,
+            status=521,
+            headers={"Zyte-Error": "/download/internal-error"},
         )
         res = mw.process_response(req, res, self.spider)
         self.assertTrue(mw._is_banned(res))
@@ -1311,7 +1315,8 @@ def test_header_drop_warnings(self, mock_logger):
                 "request is proxied with %s and not with %s, and "
                 "automatic translation is not supported for this "
                 "header. See "
-                "https://docs.zyte.com/zyte-api/migration/zyte/smartproxy.html#parameter-mapping"  # noqa
+                "https://docs.zyte.com/zyte-api/migration/zyte/smartproxy.html"
+                "#parameter-mapping"
                 " to learn the right way to translate this header "
                 "manually."
             ),
@@ -1335,7 +1340,8 @@ def test_header_drop_warnings(self, mock_logger):
                 "request is proxied with %s and not with %s, and "
                 "automatic translation is not supported for this "
                 "header. See "
-                "https://docs.zyte.com/zyte-api/migration/zyte/smartproxy.html#parameter-mapping"  # noqa
+                "https://docs.zyte.com/zyte-api/migration/zyte/smartproxy.html"
+                "#parameter-mapping"
                 " to learn the right way to translate this header "
                 "manually."
             ),
@@ -1355,7 +1361,8 @@ def test_header_drop_warnings(self, mock_logger):
             headers={"Zyte-Foo": "bar", "X-Crawlera-Foo": "bar"},
         )
         self.assertEqual(mw.process_request(request, self.spider), None)
-        mock_logger.warning.assert_not_called()  # No warnings for "drop all" scenarios
+        # No warnings for "drop all" scenarios
+        mock_logger.warning.assert_not_called()
 
     def test_header_based_handling(self):
         self.spider.zyte_smartproxy_enabled = True