don't handle 498 HTTP responses from remote websites; more tests for cache_args

it requires a proper Lua script to work

Author: kmike

scrapy_splash/middleware.py

@@ -380,7 +380,7 @@ def process_response(self, request, response, spider):
 
         # handle save_args/load_args
         self._process_x_splash_saved_arguments(request, response)
-        if response.status == 498:
+        if get_splash_status(response) == 498:
             logger.debug("Got HTTP 498 response for {}; "
                          "sending arguments again.".format(request),
                          extra={'spider': spider})
@@ -424,7 +424,7 @@ def _log_400(self, request, response, spider):
 
     def _process_x_splash_saved_arguments(self, request, response):
         """ Keep track of arguments saved by Splash. """
-        saved_args = response.headers.get(b'X-Splash-Saved-Arguments')
+        saved_args = get_splash_headers(response).get(b'X-Splash-Saved-Arguments')
         if not saved_args:
             return
         saved_args = parse_x_splash_saved_arguments_header(saved_args)

tests/test_integration.py

@@ -167,6 +167,38 @@ def start_requests(self):
     assert resp.splash_response_status == 200
 
 
+@requires_splash
+@inlineCallbacks
+def test_cache_args(settings):
+
+    class CacheArgsSpider(ResponseSpider):
+        def _request(self, url):
+            return SplashRequest(url, endpoint='execute',
+                                 args={'lua_source': DEFAULT_SCRIPT, 'x': 'yy'},
+                                 cache_args=['lua_source'])
+
+        def start_requests(self):
+            yield self._request(self.url)
+
+        def parse(self, response):
+            yield {'response': response}
+            yield self._request(self.url + "#foo")
+
+
+    items, url, crawler = yield crawl_items(CacheArgsSpider, HelloWorld,
+                                            settings)
+    assert len(items) == 2
+    resp = items[0]['response']
+    assert b"function main(splash)" in resp.request.body
+    assert b"yy" in resp.request.body
+    print(resp.body, resp.request.body)
+
+    resp = items[1]['response']
+    assert b"function main(splash)" not in resp.request.body
+    assert b"yy" in resp.request.body
+    print(resp.body, resp.request.body)
+
+
 @requires_splash
 @inlineCallbacks
 def test_cookies(settings):