Merge pull request #23 from scrapy-plugins/clear-dnscache-on-errors

Clear Scrapy dns cache on connection errors to proxy service

Author: redapple

scrapy_crawlera/middleware.py

@@ -1,12 +1,14 @@
-from collections import defaultdict
-import warnings
 import os
 import logging
+import warnings
+from collections import defaultdict
 
+from six.moves.urllib.parse import urlparse
 from w3lib.http import basic_auth_header
 from scrapy import signals
+from scrapy.resolver import dnscache
 from scrapy.exceptions import ScrapyDeprecationWarning
-from twisted.internet.error import ConnectionRefusedError
+from twisted.internet.error import ConnectionRefusedError, ConnectionDone
 
 
 class CrawleraMiddleware(object):
@@ -48,6 +50,7 @@ def open_spider(self, spider):
 
         for k, type_ in self._settings:
             setattr(self, k, self._get_setting_value(spider, k, type_))
+
         if '?noconnect' not in self.url:
             self.url += '?noconnect'
 
@@ -155,10 +158,16 @@ def process_response(self, request, response, spider):
     def process_exception(self, request, exception, spider):
         if not self._is_enabled_for_request(request):
             return
-        if isinstance(exception, ConnectionRefusedError):
+        if isinstance(exception, (ConnectionRefusedError, ConnectionDone)):
             # Handle crawlera downtime
+            self._clear_dns_cache()
             self._set_custom_delay(request, self.connection_refused_delay)
 
+    def _clear_dns_cache(self):
+        # Scrapy doesn't expire dns records by default, so we force it here,
+        # so client can reconnect trough DNS failover.
+        dnscache.pop(urlparse(self.url).hostname, None)
+
     def _is_enabled_for_request(self, request):
         return self.enabled and 'dont_proxy' not in request.meta
 

tests/test_crawlera.py

@@ -4,7 +4,8 @@
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
-from twisted.internet.error import ConnectionRefusedError
+from scrapy.resolver import dnscache
+from twisted.internet.error import ConnectionRefusedError, ConnectionDone
 
 from scrapy_crawlera import CrawleraMiddleware
 import os
@@ -229,29 +230,44 @@ def test_delay_adjustment(self):
         self.assertEqual(slot.delay, retry_after)
         self.assertEqual(self.spider.download_delay, delay)
 
+        # DNS cache should be cleared in case of errors
+        dnscache['proxy.crawlera.com'] = '1.1.1.1'
+
         res = Response(url, request=req)
         mw.process_response(req, res, self.spider)
         self.assertEqual(slot.delay, delay)
         self.assertEqual(self.spider.download_delay, delay)
+        self.assertIn('proxy.crawlera.com', dnscache)
 
         # server failures
         mw.process_exception(req, ConnectionRefusedError(), self.spider)
         self.assertEqual(slot.delay, mw.connection_refused_delay)
         self.assertEqual(self.spider.download_delay, delay)
+        self.assertNotIn('proxy.crawlera.com', dnscache)
 
+        dnscache['proxy.crawlera.com'] = '1.1.1.1'
         res = Response(ban_url, request=req)
         mw.process_response(req, res, self.spider)
         self.assertEqual(slot.delay, delay)
         self.assertEqual(self.spider.download_delay, delay)
+        self.assertIn('proxy.crawlera.com', dnscache)
 
         mw.process_exception(req, ConnectionRefusedError(), self.spider)
         self.assertEqual(slot.delay, mw.connection_refused_delay)
         self.assertEqual(self.spider.download_delay, delay)
+        self.assertNotIn('proxy.crawlera.com', dnscache)
 
+        dnscache['proxy.crawlera.com'] = '1.1.1.1'
         res = Response(ban_url, status=self.bancode, request=req)
         mw.process_response(req, res, self.spider)
         self.assertEqual(slot.delay, delay)
         self.assertEqual(self.spider.download_delay, delay)
+        self.assertIn('proxy.crawlera.com', dnscache)
+
+        mw.process_exception(req, ConnectionDone(), self.spider)
+        self.assertEqual(slot.delay, mw.connection_refused_delay)
+        self.assertEqual(self.spider.download_delay, delay)
+        self.assertNotIn('proxy.crawlera.com', dnscache)
 
     def test_jobid_header(self):
         # test without the environment variable 'SCRAPY_JOB'