Use a custom logger instead of the root one (#101)

Author: elacuesta

scrapy_zyte_smartproxy/middleware.py

@@ -13,6 +13,9 @@
 from scrapy_zyte_smartproxy.utils import exp_backoff
 
 
+logger = logging.getLogger(__name__)
+
+
 class ZyteSmartProxyMiddleware(object):
 
     url = 'http://proxy.zyte.com:8011'
@@ -71,15 +74,15 @@ def open_spider(self, spider):
             return
 
         if not self.apikey:
-            logging.warning(
+            logger.warning(
                 "Zyte Smart Proxy Manager cannot be used without an API key",
                 extra={'spider': spider},
             )
             return
 
         self._proxyauth = self.get_proxyauth(spider)
 
-        logging.info(
+        logger.info(
             "Using Zyte Smart Proxy Manager at %s (apikey: %s)" % (
                 self.url, self.apikey[:7]
             ),
@@ -89,7 +92,7 @@ def open_spider(self, spider):
         if not self.preserve_delay:
             # Setting spider download delay to 0 to get maximum crawl rate
             spider.download_delay = 0
-            logging.info(
+            logger.info(
                 "ZyteSmartProxyMiddleware: disabling download delays in "
                 "Scrapy to optimize delays introduced by Zyte Smart Proxy "
                 "Manager. To avoid this behaviour you can use the "
@@ -130,9 +133,9 @@ def _get_setting_value(self, spider, k, type_):
 
     def _fix_url_protocol(self):
         if self.url.startswith('https://'):
-            logging.warning('ZYTE_SMARTPROXY_URL "%s" set with "https://" protocol.' % self.url)
+            logger.warning('ZYTE_SMARTPROXY_URL "%s" set with "https://" protocol.' % self.url)
         elif not self.url.startswith('http://'):
-            logging.warning('Adding "http://" to ZYTE_SMARTPROXY_URL %s' % self.url)
+            logger.warning('Adding "http://" to ZYTE_SMARTPROXY_URL %s' % self.url)
             self.url = 'http://' + self.url
 
     def is_enabled(self, spider):
@@ -216,7 +219,7 @@ def process_response(self, request, response, spider):
                 return self._retry_auth(response, request, spider)
             else:
                 self.crawler.stats.inc_value('zyte_smartproxy/retries/auth/max_reached')
-                logging.warning(
+                logger.warning(
                     "Max retries for authentication issues reached, please check auth"
                     " information settings",
                     extra={'spider': self.spider},
@@ -263,7 +266,7 @@ def _handle_not_enabled_response(self, request, response):
         return response
 
     def _retry_auth(self, response, request, spider):
-        logging.warning(
+        logger.warning(
             "Retrying a Zyte Smart Proxy Manager request due to an "
             "authentication issue",
             extra={'spider': self.spider},
@@ -357,7 +360,7 @@ def _set_zyte_smartproxy_default_headers(self, request):
                 'urls with problems.'
                 % str(self.conflicting_headers)
             )
-            logging.debug(
+            logger.debug(
                 'The headers %s are conflicting on request %s. X-Crawlera-UA '
                 'will be ignored. Please check '
                 'https://docs.zyte.com/smart-proxy-manager.html#request-headers '

tests/test_all.py

@@ -462,7 +462,7 @@ def test_zyte_smartproxy_default_headers(self):
         self.assertNotIn('X-Crawlera-Profile', req.headers)
 
     @patch('scrapy_zyte_smartproxy.middleware.warnings')
-    @patch('scrapy_zyte_smartproxy.middleware.logging')
+    @patch('scrapy_zyte_smartproxy.middleware.logger')
     def test_zyte_smartproxy_default_headers_conflicting_headers(self, mock_logger, mock_warnings):
         spider = self.spider
         self.spider.zyte_smartproxy_enabled = True
@@ -671,7 +671,7 @@ def test_auth_error_retries(self, random_uniform_patch):
         res = mw.process_response(auth_error_req, non_zyte_smartproxy_407_response, self.spider)
         self.assertIsInstance(res, Response)
 
-    @patch('scrapy_zyte_smartproxy.middleware.logging')
+    @patch('scrapy_zyte_smartproxy.middleware.logger')
     def test_open_spider_logging(self, mock_logger):
         spider = self.spider
         self.spider.zyte_smartproxy_enabled = True
@@ -767,7 +767,7 @@ def test_process_response_from_file_scheme(self):
         self.assertEqual(mw.crawler.stats.get_stats(), {})
         self.assertEqual(out.status, 200)
 
-    @patch('scrapy_zyte_smartproxy.middleware.logging')
+    @patch('scrapy_zyte_smartproxy.middleware.logger')
     def test_apikey_warning_zyte_smartproxy_disabled(self, mock_logger):
         self.spider.zyte_smartproxy_enabled = False
         settings = {}
@@ -777,7 +777,7 @@ def test_apikey_warning_zyte_smartproxy_disabled(self, mock_logger):
         self.assertFalse(mw.enabled)
         mock_logger.warning.assert_not_called()
 
-    @patch('scrapy_zyte_smartproxy.middleware.logging')
+    @patch('scrapy_zyte_smartproxy.middleware.logger')
     def test_no_apikey_warning_zyte_smartproxy_enabled(self, mock_logger):
         self.spider.zyte_smartproxy_enabled = True
         settings = {}
@@ -790,7 +790,7 @@ def test_no_apikey_warning_zyte_smartproxy_enabled(self, mock_logger):
             extra={'spider': self.spider}
         )
 
-    @patch('scrapy_zyte_smartproxy.middleware.logging')
+    @patch('scrapy_zyte_smartproxy.middleware.logger')
     def test_no_apikey_warning_force_enable(self, mock_logger):
         self.spider.zyte_smartproxy_enabled = False
         settings = {'ZYTE_SMARTPROXY_FORCE_ENABLE_ON_HTTP_CODES': [403]}
@@ -803,7 +803,7 @@ def test_no_apikey_warning_force_enable(self, mock_logger):
             extra={'spider': self.spider}
         )
 
-    @patch('scrapy_zyte_smartproxy.middleware.logging')
+    @patch('scrapy_zyte_smartproxy.middleware.logger')
     def test_apikey_warning_force_enable(self, mock_logger):
         self.spider.zyte_smartproxy_enabled = False
         settings = {