Merge pull request #58 from hcoura/default-crawlera-headers

[MRG + 1] Add DEFAULT_CRAWLERA_HEADERS settings

Author: hcoura

.gitignore

@@ -53,6 +53,7 @@ htmlcov/
 nosetests.xml
 coverage.xml
 *,cover
+.pytest_cache
 
 # Translations
 *.mo
@@ -67,4 +68,7 @@ docs/_build/
 target/
 
 # IDEA
-.idea/
+.idea/
+
+# Pipenv
+Pipfile*

docs/settings.rst

@@ -57,4 +57,13 @@ CRAWLERA_PRESERVE_DELAY
 Default: ``False``
 
 If ``False`` Sets Scrapy's ``DOWNLOAD_DELAY`` to ``0``, making the spider to crawl faster. If set to ``True``, it will
-respect the provided ``DOWNLOAD_DELAY`` from Scrapy.
+respect the provided ``DOWNLOAD_DELAY`` from Scrapy.
+
+CRAWLERA_DEFAULT_HEADERS
+-----------------------
+
+Default: ``{}``
+
+Default headers added only to crawlera requests. Headers defined on ``DEFAULT_REQUEST_HEADERS`` will take precedence as long as the ``CrawleraMiddleware`` is placed after the ``DefaultHeadersMiddleware``*. Headers set on the requests have precedence over the two settings.
+
+*This is the default behavior, ``DefaultHeadersMiddleware`` default priority is ``400`` and we recommend ``CrawleraMiddleware`` priority to be ``610``

scrapy_crawlera/middleware.py

@@ -21,6 +21,7 @@ class CrawleraMiddleware(object):
     connection_refused_delay = 90
     preserve_delay = False
     header_prefix = 'X-Crawlera-'
+    conflicting_headers = ('X-Crawlera-Profile', 'X-Crawlera-UA')
 
     _settings = [
         ('apikey', str),
@@ -64,6 +65,8 @@ def open_spider(self, spider):
                 "CrawleraMiddleware: disabling download delays on Scrapy side to optimize delays introduced by Crawlera. "
                 "To avoid this behaviour you can use the CRAWLERA_PRESERVE_DELAY setting but keep in mind that this may slow down the crawl significantly")
 
+        self._headers = self.crawler.settings.get('CRAWLERA_DEFAULT_HEADERS', {}).items()
+
     def _settings_get(self, type_, *a, **kw):
         if type_ is int:
             return self.crawler.settings.getint(*a, **kw)
@@ -120,6 +123,7 @@ def get_proxyauth(self, spider):
 
     def process_request(self, request, spider):
         if self._is_enabled_for_request(request):
+            self._set_crawlera_default_headers(request)
             request.meta['proxy'] = self.url
             request.meta['download_timeout'] = self.download_timeout
             request.headers['Proxy-Authorization'] = self._proxyauth
@@ -211,3 +215,19 @@ def _is_crawlera_header(self, header_name):
             return False
         header_name = header_name.decode('utf-8').lower()
         return header_name.startswith(self.header_prefix.lower())
+
+    def _set_crawlera_default_headers(self, request):
+        for header, value in self._headers:
+            if value is None:
+                continue
+            request.headers.setdefault(header, value)
+        lower_case_headers = [
+            header.decode('utf-8').lower() for header in request.headers
+        ]
+        if all(h.lower() in lower_case_headers for h in self.conflicting_headers):
+            logging.warn(
+                'The headers %s are conflicting on request %s. X-Crawlera-UA '
+                'will be ignored. Please check https://doc.scrapinghub.com/cr'
+                'awlera.html for more information'
+                % (str(self.conflicting_headers), request.url)
+            )

tests/requirements.txt

@@ -1,2 +1,3 @@
 pytest
 pytest-cov
+mock; python_version == '2.7'

tests/test_crawlera.py

@@ -1,4 +1,8 @@
 from unittest import TestCase
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
 
 from w3lib.http import basic_auth_header
 from scrapy.http import Request, Response
@@ -362,3 +366,67 @@ def test_clean_headers_when_enabled(self):
         self.assertIn(b'X-Crawlera-Debug', req.headers)
         self.assertIn(b'X-Crawlera-Profile', req.headers)
         self.assertIn(b'User-Agent', req.headers)
+
+    def test_crawlera_default_headers(self):
+        spider = self.spider
+        self.spider.crawlera_enabled = True
+
+        self.settings['CRAWLERA_DEFAULT_HEADERS'] = {
+            'X-Crawlera-Profile': 'desktop'
+        }
+        crawler = self._mock_crawler(spider, self.settings)
+        mw = self.mwcls.from_crawler(crawler)
+        mw.open_spider(spider)
+        req = Request('http://www.scrapytest.org/other')
+        assert mw.process_request(req, spider) is None
+        self.assertEqual(req.headers['X-Crawlera-Profile'], b'desktop')
+
+        # test ignore None headers
+        self.settings['CRAWLERA_DEFAULT_HEADERS'] = {
+            'X-Crawlera-Profile': None,
+            'X-Crawlera-Cookies': 'disable'
+        }
+        crawler = self._mock_crawler(spider, self.settings)
+        mw = self.mwcls.from_crawler(crawler)
+        mw.open_spider(spider)
+        req = Request('http://www.scrapytest.org/other')
+        assert mw.process_request(req, spider) is None
+        self.assertEqual(req.headers['X-Crawlera-Cookies'], b'disable')
+        self.assertNotIn('X-Crawlera-Profile', req.headers)
+
+    @patch('scrapy_crawlera.middleware.logging')
+    def test_crawlera_default_headers_conflicting_headers(self, mock_logger):
+        spider = self.spider
+        self.spider.crawlera_enabled = True
+
+        self.settings['CRAWLERA_DEFAULT_HEADERS'] = {
+            'X-Crawlera-Profile': 'desktop'
+        }
+        crawler = self._mock_crawler(spider, self.settings)
+        mw = self.mwcls.from_crawler(crawler)
+        mw.open_spider(spider)
+
+        req = Request('http://www.scrapytest.org/other',
+                      headers={'X-Crawlera-UA': 'desktop'})
+        assert mw.process_request(req, spider) is None
+        self.assertEqual(req.headers['X-Crawlera-UA'], b'desktop')
+        self.assertEqual(req.headers['X-Crawlera-Profile'], b'desktop')
+        mock_logger.warn.assert_called_with(
+            "The headers ('X-Crawlera-Profile', 'X-Crawlera-UA') are conflictin"
+            "g on request http://www.scrapytest.org/other. X-Crawlera-UA will b"
+            "e ignored. Please check https://doc.scrapinghub.com/crawlera.html "
+            "for more information"
+        )
+
+        # test it ignores case
+        req = Request('http://www.scrapytest.org/other',
+                      headers={'x-crawlera-ua': 'desktop'})
+        assert mw.process_request(req, spider) is None
+        self.assertEqual(req.headers['X-Crawlera-UA'], b'desktop')
+        self.assertEqual(req.headers['X-Crawlera-Profile'], b'desktop')
+        mock_logger.warn.assert_called_with(
+            "The headers ('X-Crawlera-Profile', 'X-Crawlera-UA') are conflictin"
+            "g on request http://www.scrapytest.org/other. X-Crawlera-UA will b"
+            "e ignored. Please check https://doc.scrapinghub.com/crawlera.html "
+            "for more information"
+        )