[resources] add support for passing crawl arguments

Author: pawelmhm

scrapyrt/resources.py

@@ -1,4 +1,7 @@
 # -*- coding: utf-8 -*-
+import json
+from urllib.parse import unquote
+
 import demjson
 from scrapy.utils.misc import load_object
 from scrapy.utils.serialize import ScrapyJSONEncoder
@@ -134,6 +137,7 @@ def render_GET(self, request, **kwargs):
         scrapy_request_args = extract_scrapy_request_args(api_params,
                                                           raise_error=False)
         self.validate_options(scrapy_request_args, api_params)
+
         return self.prepare_crawl(api_params, scrapy_request_args, **kwargs)
 
     def render_POST(self, request, **kwargs):
@@ -171,7 +175,7 @@ def render_POST(self, request, **kwargs):
             _request = self.get_required_argument(api_params, "request")
         try:
             scrapy_request_args = extract_scrapy_request_args(
-                _request, raise_error=False
+                _request, raise_error=True
             )
         except ValueError as e:
             raise Error('400', str(e))
@@ -224,17 +228,33 @@ def prepare_crawl(self, api_params, scrapy_request_args, *args, **kwargs):
             max_requests = api_params['max_requests']
         except (KeyError, IndexError):
             max_requests = None
+
+        crawl_args = api_params.get("crawl_args")
+        if isinstance(crawl_args, str):
+            try:
+                crawl_args = json.loads(unquote(crawl_args))
+            except Exception as e:
+                msg = "crawl_args must be valid url encoded JSON"
+                msg += " this string cannot be decoded with JSON"
+                msg += f' {str(e)}'
+                raise Error('400', message=msg)
+
         dfd = self.run_crawl(
             spider_name, scrapy_request_args, max_requests,
-            start_requests=start_requests, *args, **kwargs)
+            start_requests=start_requests,
+            crawl_args=crawl_args,
+            *args,
+            **kwargs)
         dfd.addCallback(
             self.prepare_response, request_data=api_params, *args, **kwargs)
         return dfd
 
     def run_crawl(self, spider_name, scrapy_request_args,
-                  max_requests=None, start_requests=False, *args, **kwargs):
+                  max_requests=None, crawl_args=None, start_requests=False, *args, **kwargs):
         crawl_manager_cls = load_object(settings.CRAWL_MANAGER)
         manager = crawl_manager_cls(spider_name, scrapy_request_args, max_requests, start_requests=start_requests)
+        if crawl_args:
+            kwargs.update(crawl_args)
         dfd = manager.crawl(*args, **kwargs)
         return dfd
 

tests/test_resource_crawl.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import json
 import os
+from urllib.parse import quote
 
 import pytest
 import re
@@ -413,15 +414,14 @@ def test_bytes_in_item(self, server, method):
         assert res.status_code == 200
         assert res.json()["items"] == [{'name': 'Some bytes here'}]
 
-    @pytest.mark.parametrize("method", [
-        perform_get, perform_post
-    ])
-    def test_crawl_with_argument(self, server, method):
+    def test_crawl_with_argument_get(self, server):
         url = server.url("crawl.json")
         postcode = "43-300"
-        res = method(url, {"spider_name": "test"}, {
+        argument = json.dumps({"postcode": postcode})
+        argument = quote(argument)
+        res = perform_get(url, {"spider_name": "test"}, {
             "url": server.target_site.url("page1.html"),
-            "postcode": postcode,
+            "crawl_args": argument,
             "callback": 'return_argument'
         })
         expected_items = [{
@@ -430,7 +430,28 @@ def test_crawl_with_argument(self, server, method):
         res_json = res.json()
         assert res_json["status"] == "ok"
         assert res_json["items_dropped"] == []
+        assert res_json['items']
+        assert len(res_json['items']) == len(expected_items)
+        assert res_json["items"] == expected_items
 
+    def test_crawl_with_argument_post(self, server):
+        url = server.url("crawl.json")
+        postcode = "43-300"
+        argument = {"postcode": postcode}
+        res = perform_post(url, {
+            "spider_name": "test",
+            "crawl_args": argument
+        }, {
+            "url": server.target_site.url("page1.html"),
+            "callback": 'return_argument'
+        })
+        expected_items = [{
+            u'name': postcode,
+        }]
+        res_json = res.json()
+        assert res_json["status"] == "ok"
+        assert not res_json.get("errors")
+        assert res_json["items_dropped"] == []
         assert res_json['items']
         assert len(res_json['items']) == len(expected_items)
         assert res_json["items"] == expected_items