Merge pull request #55 from stummjr/clean-headers

[MRG+2] clean X-Crawlera-* headers when crawlera is disabled

Author: eLRuLL

docs/index.rst

@@ -73,6 +73,12 @@ Here we have an example of specifying a Crawlera header into a Scrapy request::
 Remember that you could also set which headers to use by default by all
 requests with `DEFAULT_REQUEST_HEADERS <http://doc.scrapy.org/en/1.0/topics/settings.html#default-request-headers>`_
 
+.. note:: Crawlera headers are removed from requests when the middleware is activated but Crawlera
+    is disabled. For example, if you accidentally disable Crawlera via ``crawlera_enabled = False``
+    but keep sending ``X-Crawlera-*`` headers in your requests, those will be removed from the
+    request headers.
+
+
 This Middleware also adds some configurable Scrapy Settings, check :ref:`the complete list here <settings>`.
 
 All the rest

scrapy_crawlera/middleware.py

@@ -20,6 +20,7 @@ class CrawleraMiddleware(object):
     # Handle crawlera server failures
     connection_refused_delay = 90
     preserve_delay = False
+    header_prefix = 'X-Crawlera-'
 
     _settings = [
         ('apikey', str),
@@ -126,6 +127,8 @@ def process_request(self, request, spider):
                 request.headers['X-Crawlera-Jobid'] = self.job_id
             self.crawler.stats.inc_value('crawlera/request')
             self.crawler.stats.inc_value('crawlera/request/method/%s' % request.method)
+        else:
+            self._clean_crawlera_headers(request)
 
     def process_response(self, request, response, spider):
         if not self._is_enabled_for_request(request):
@@ -192,3 +195,19 @@ def _restore_original_delay(self, request):
             return
         if self._saved_delays[key] is not None:
             slot.delay, self._saved_delays[key] = self._saved_delays[key], None
+
+    def _clean_crawlera_headers(self, request):
+        """Remove X-Crawlera-* headers from the request."""
+        targets = [
+            header
+            for header in request.headers
+            if self._is_crawlera_header(header)
+        ]
+        for header in targets:
+            request.headers.pop(header, None)
+
+    def _is_crawlera_header(self, header_name):
+        if not header_name:
+            return False
+        header_name = header_name.decode('utf-8').lower()
+        return header_name.startswith(self.header_prefix.lower())

tests/test_crawlera.py

@@ -333,3 +333,32 @@ def test_stats(self):
         self.assertEqual(crawler.stats.get_value('crawlera/response/status/{}'.format(mw.ban_code)), 1)
         self.assertEqual(crawler.stats.get_value('crawlera/response/banned'), 1)
         self.assertEqual(crawler.stats.get_value('crawlera/response/error/somethingbad'), 1)
+
+    def _make_fake_request(self, spider, crawlera_enabled):
+        spider.crawlera_enabled = crawlera_enabled
+        crawler = self._mock_crawler(spider, self.settings)
+        mw = self.mwcls.from_crawler(crawler)
+        mw.open_spider(spider)
+        headers = {
+            'X-Crawlera-Debug': True,
+            'X-Crawlera-Profile': 'desktop',
+            'User-Agent': 'Scrapy',
+            '': None,
+        }
+        req = Request('http://www.scrapytest.org', headers=headers)
+        out = mw.process_request(req, spider)
+        return req
+
+    def test_clean_headers_when_disabled(self):
+        req = self._make_fake_request(self.spider, crawlera_enabled=False)
+
+        self.assertNotIn(b'X-Crawlera-Debug', req.headers)
+        self.assertNotIn(b'X-Crawlera-Profile', req.headers)
+        self.assertIn(b'User-Agent', req.headers)
+
+    def test_clean_headers_when_enabled(self):
+        req = self._make_fake_request(self.spider, crawlera_enabled=True)
+
+        self.assertIn(b'X-Crawlera-Debug', req.headers)
+        self.assertIn(b'X-Crawlera-Profile', req.headers)
+        self.assertIn(b'User-Agent', req.headers)