Merge pull request #69 from scrapy-plugins/fix-default-crawlera-headers

[MRG + 1] Fix default crawlera headers

Author: hcoura

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.4.0
+current_version = 1.5.0
 commit = True
 tag = True
 tag_name = v{new_version}

scrapy_crawlera/__init__.py

@@ -1,4 +1,4 @@
 from .middleware import CrawleraMiddleware
 
 
-__version__ = '1.4.0'
+__version__ = '1.5.0'

scrapy_crawlera/middleware.py

@@ -231,7 +231,14 @@ def _set_crawlera_default_headers(self, request):
             header.decode('utf-8').lower() for header in request.headers
         ]
         if all(h.lower() in lower_case_headers for h in self.conflicting_headers):
-            logging.warn(
+            # Send a general warning once, and specific urls if LOG_LEVEL = DEBUG
+            warnings.warn(
+                'The headers %s are conflicting on some of your requests. '
+                'Please check https://doc.scrapinghub.com/crawlera.html '
+                'for more information. You can set LOG_LEVEL=DEBUG to see the urls with problems'
+                % str(self.conflicting_headers)
+            )
+            logging.debug(
                 'The headers %s are conflicting on request %s. X-Crawlera-UA '
                 'will be ignored. Please check https://doc.scrapinghub.com/cr'
                 'awlera.html for more information'

setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name='scrapy-crawlera',
-    version='1.4.0',
+    version='1.5.0',
     license='BSD',
     description='Crawlera middleware for Scrapy',
     long_description=readme,

tests/test_crawlera.py

@@ -434,7 +434,7 @@ def test_crawlera_default_headers_conflicting_headers(self, mock_logger):
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.headers['X-Crawlera-UA'], b'desktop')
         self.assertEqual(req.headers['X-Crawlera-Profile'], b'desktop')
-        mock_logger.warn.assert_called_with(
+        mock_logger.debug.assert_called_with(
             "The headers ('X-Crawlera-Profile', 'X-Crawlera-UA') are conflictin"
             "g on request http://www.scrapytest.org/other. X-Crawlera-UA will b"
             "e ignored. Please check https://doc.scrapinghub.com/crawlera.html "
@@ -447,7 +447,7 @@ def test_crawlera_default_headers_conflicting_headers(self, mock_logger):
         assert mw.process_request(req, spider) is None
         self.assertEqual(req.headers['X-Crawlera-UA'], b'desktop')
         self.assertEqual(req.headers['X-Crawlera-Profile'], b'desktop')
-        mock_logger.warn.assert_called_with(
+        mock_logger.debug.assert_called_with(
             "The headers ('X-Crawlera-Profile', 'X-Crawlera-UA') are conflictin"
             "g on request http://www.scrapytest.org/other. X-Crawlera-UA will b"
             "e ignored. Please check https://doc.scrapinghub.com/crawlera.html "