don't log response content when it is website who returns 400, not Splash

it only works with a proper Lua script

Author: kmike

scrapy_splash/middleware.py

@@ -23,6 +23,7 @@
     json_based_hash,
     parse_x_splash_saved_arguments_header,
 )
+from scrapy_splash.response import get_splash_status, get_splash_headers
 
 
 logger = logging.getLogger(__name__)
@@ -390,7 +391,7 @@ def process_response(self, request, response, spider):
 
         response = self._change_response_class(request, response)
 
-        if self.log_400 and response.status == 400:
+        if self.log_400 and get_splash_status(response) == 400:
             self._log_400(request, response, spider)
 
         return response

scrapy_splash/response.py

@@ -11,6 +11,14 @@
 from scrapy_splash.utils import headers_to_scrapy
 
 
+def get_splash_status(resp):
+    return getattr(resp, 'splash_response_status', resp.status)
+
+
+def get_splash_headers(resp):
+    return getattr(resp, 'splash_response_headers', resp.headers)
+
+
 class _SplashResponseMixin(object):
     """
     This mixin fixes response.url and adds response.real_url

tests/test_integration.py

@@ -41,6 +41,11 @@ class HelloWorld(HtmlResource):
     extra_headers = {'X-MyHeader': 'my value', 'Set-Cookie': 'sessionid=ABCD'}
 
 
+class Http400Resource(HtmlResource):
+    status_code = 400
+    html = "Website returns HTTP 400 error"
+
+
 
 class ManyCookies(Resource, object):
     class SetMyCookie(HtmlResource):
@@ -131,6 +136,37 @@ def start_requests(self):
     assert resp.data['args']['foo'] == 'bar'
 
 
+@requires_splash
+@inlineCallbacks
+def test_bad_request(settings):
+    class BadRequestSpider(ResponseSpider):
+        custom_settings = {'HTTPERROR_ALLOW_ALL': True}
+
+        def start_requests(self):
+            yield SplashRequest(self.url, endpoint='execute',
+                                args={'lua_source': DEFAULT_SCRIPT, 'wait': 'bar'})
+
+    class GoodRequestSpider(ResponseSpider):
+        custom_settings = {'HTTPERROR_ALLOW_ALL': True}
+
+        def start_requests(self):
+            yield SplashRequest(self.url, endpoint='execute',
+                                args={'lua_source': DEFAULT_SCRIPT})
+
+
+    items, url, crawler = yield crawl_items(BadRequestSpider, HelloWorld,
+                                            settings)
+    resp = items[0]['response']
+    assert resp.status == 400
+    assert resp.splash_response_status == 400
+
+    items, url, crawler = yield crawl_items(GoodRequestSpider, Http400Resource,
+                                            settings)
+    resp = items[0]['response']
+    assert resp.status == 400
+    assert resp.splash_response_status == 200
+
+
 @requires_splash
 @inlineCallbacks
 def test_cookies(settings):

tests/test_middleware.py

@@ -401,12 +401,13 @@ def test_magic_response_http_error():
         "error": 400,
         "type": "ScriptError"
     }
-    resp = TextResponse("http://mysplash.example.com/execute",
+    resp = TextResponse("http://mysplash.example.com/execute", status=400,
                         headers={b'Content-Type': b'application/json'},
                         body=json.dumps(resp_data).encode('utf8'))
     resp = mw.process_response(req, resp, None)
     assert resp.data == resp_data
     assert resp.status == 404
+    assert resp.splash_response_status == 400
     assert resp.url == "http://example.com/foo"
 
 

tests/utils.py

@@ -20,11 +20,13 @@ class HtmlResource(Resource):
     content_type = 'text/html'
     html = ''
     extra_headers = {}
+    status_code = 200
 
     def render_GET(self, request):
         request.setHeader(b'content-type', to_bytes(self.content_type))
         for name, value in self.extra_headers.items():
             request.setHeader(to_bytes(name), to_bytes(value))
+        request.setResponseCode(self.status_code)
         return to_bytes(self.html)