Add url to the warning logs

Author: hcoura

scrapy_crawlera/middleware.py

@@ -226,8 +226,8 @@ def _set_crawlera_default_headers(self, request):
         ]
         if all(h.lower() in lower_case_headers for h in self.conflicting_headers):
             logging.warn(
-                'The headers %s are conflicting, X-Crawlera-UA will be ignored. '
-                'Please check https://doc.scrapinghub.com/crawlera.html for '
-                'more information'
-                % str(self.conflicting_headers)
+                'The headers %s are conflicting on request %s. X-Crawlera-UA '
+                'will be ignored. Please check https://doc.scrapinghub.com/cr'
+                'awlera.html for more information'
+                % (str(self.conflicting_headers), request.url)
             )

tests/test_crawlera.py

@@ -413,8 +413,9 @@ def test_crawlera_default_headers_conflicting_headers(self, mock_logger):
         self.assertEqual(req.headers['X-Crawlera-Profile'], b'desktop')
         mock_logger.warn.assert_called_with(
             "The headers ('X-Crawlera-Profile', 'X-Crawlera-UA') are conflictin"
-            "g, X-Crawlera-UA will be ignored. Please check https://doc.scraping"
-            "hub.com/crawlera.html for more information"
+            "g on request http://www.scrapytest.org/other. X-Crawlera-UA will b"
+            "e ignored. Please check https://doc.scrapinghub.com/crawlera.html "
+            "for more information"
         )
 
         # test it ignores case
@@ -425,6 +426,7 @@ def test_crawlera_default_headers_conflicting_headers(self, mock_logger):
         self.assertEqual(req.headers['X-Crawlera-Profile'], b'desktop')
         mock_logger.warn.assert_called_with(
             "The headers ('X-Crawlera-Profile', 'X-Crawlera-UA') are conflictin"
-            "g, X-Crawlera-UA will be ignored. Please check https://doc.scraping"
-            "hub.com/crawlera.html for more information"
+            "g on request http://www.scrapytest.org/other. X-Crawlera-UA will b"
+            "e ignored. Please check https://doc.scrapinghub.com/crawlera.html "
+            "for more information"
         )