Merge pull request #120 from scrapinghub/params-for-spider

Params for spider

Author: pawelmhm

scrapyrt/core.py

@@ -60,6 +60,13 @@ def __init__(self, settings, scrapyrt_manager):
     def crawl(self, spidercls, *args, **kwargs):
         if isinstance(spidercls, six.string_types):
             spidercls = self.spider_loader.load(spidercls)
+
+        for kw in kwargs:
+            attr_or_m = getattr(spidercls, kw, None)
+            if attr_or_m and callable(attr_or_m):
+                msg = 'Crawl argument cannot override spider method.'
+                msg += ' Got argument {} that overrides spider method {}'
+                raise Error('400', message=msg.format(kw, getattr(spidercls, kw)))
         # creating our own crawler that will allow us to disable start requests easily
         crawler = ScrapyrtCrawler(
             spidercls, self.settings, self.scrapyrt_manager.start_requests)

scrapyrt/resources.py

@@ -1,4 +1,7 @@
 # -*- coding: utf-8 -*-
+import json
+from urllib.parse import unquote
+
 import demjson
 from scrapy.utils.misc import load_object
 from scrapy.utils.serialize import ScrapyJSONEncoder
@@ -134,6 +137,7 @@ def render_GET(self, request, **kwargs):
         scrapy_request_args = extract_scrapy_request_args(api_params,
                                                           raise_error=False)
         self.validate_options(scrapy_request_args, api_params)
+
         return self.prepare_crawl(api_params, scrapy_request_args, **kwargs)
 
     def render_POST(self, request, **kwargs):
@@ -154,10 +158,12 @@ def render_POST(self, request, **kwargs):
         """
         request_body = request.content.getvalue()
         try:
+            # TODO replace demjson with json.loads
             api_params = demjson.decode(request_body)
         except demjson.JSONDecodeError as e:
             message = "Invalid JSON in POST body. {}"
             message = message.format(e.pretty_description())
+            # TODO should be integer not string
             raise Error('400', message=message)
 
         log.msg("{}".format(api_params))
@@ -222,17 +228,33 @@ def prepare_crawl(self, api_params, scrapy_request_args, *args, **kwargs):
             max_requests = api_params['max_requests']
         except (KeyError, IndexError):
             max_requests = None
+
+        crawl_args = api_params.get("crawl_args")
+        if isinstance(crawl_args, str):
+            try:
+                crawl_args = json.loads(unquote(crawl_args))
+            except Exception as e:
+                msg = "crawl_args must be valid url encoded JSON"
+                msg += " this string cannot be decoded with JSON"
+                msg += f' {str(e)}'
+                raise Error('400', message=msg)
+
         dfd = self.run_crawl(
             spider_name, scrapy_request_args, max_requests,
-            start_requests=start_requests, *args, **kwargs)
+            start_requests=start_requests,
+            crawl_args=crawl_args,
+            *args,
+            **kwargs)
         dfd.addCallback(
             self.prepare_response, request_data=api_params, *args, **kwargs)
         return dfd
 
     def run_crawl(self, spider_name, scrapy_request_args,
-                  max_requests=None, start_requests=False, *args, **kwargs):
+                  max_requests=None, crawl_args=None, start_requests=False, *args, **kwargs):
         crawl_manager_cls = load_object(settings.CRAWL_MANAGER)
         manager = crawl_manager_cls(spider_name, scrapy_request_args, max_requests, start_requests=start_requests)
+        if crawl_args:
+            kwargs.update(crawl_args)
         dfd = manager.crawl(*args, **kwargs)
         return dfd
 

scrapyrt/utils.py

@@ -15,7 +15,7 @@ def extract_scrapy_request_args(dictionary, raise_error=False):
             arguments.
     """
     result = dictionary.copy()
-    args = inspect.getargspec(Request.__init__).args
+    args = inspect.getfullargspec(Request.__init__).args
     for key in dictionary.keys():
         if key not in args:
             result.pop(key)

tests/sample_data/testproject/testproject/spiders/testspider.py

@@ -7,6 +7,7 @@
 class TestSpider(scrapy.Spider):
 
     name = 'test'
+    some_attribute = "Yes|No"
 
     def parse(self, response):
         name = response.xpath('//h1/text()').extract()
@@ -18,3 +19,6 @@ def return_bytes(self, response):
     def some_errback(self, err):
         self.logger.error("Logging some error {}".format(err))
         return
+
+    def return_argument(self, response):
+        return TestprojectItem(name=self.postcode)

tests/test_resource_crawl.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import json
 import os
+from urllib.parse import quote
 
 import pytest
 import re
@@ -412,3 +413,113 @@ def test_bytes_in_item(self, server, method):
                       'callback': 'return_bytes'})
         assert res.status_code == 200
         assert res.json()["items"] == [{'name': 'Some bytes here'}]
+
+    def test_crawl_with_argument_get(self, server):
+        url = server.url("crawl.json")
+        postcode = "43-300"
+        argument = json.dumps({"postcode": postcode})
+        argument = quote(argument)
+        res = perform_get(url, {"spider_name": "test"}, {
+            "url": server.target_site.url("page1.html"),
+            "crawl_args": argument,
+            "callback": 'return_argument'
+        })
+        expected_items = [{
+            u'name': postcode,
+        }]
+        res_json = res.json()
+        assert res_json["status"] == "ok"
+        assert res_json["items_dropped"] == []
+        assert res_json['items']
+        assert len(res_json['items']) == len(expected_items)
+        assert res_json["items"] == expected_items
+
+    def test_crawl_with_argument_post(self, server):
+        url = server.url("crawl.json")
+        postcode = "43-300"
+        argument = {"postcode": postcode}
+        res = perform_post(url, {
+            "spider_name": "test",
+            "crawl_args": argument
+        }, {
+            "url": server.target_site.url("page1.html"),
+            "callback": 'return_argument'
+        })
+        expected_items = [{
+            u'name': postcode,
+        }]
+        res_json = res.json()
+        assert res.status_code == 200
+        assert res_json["status"] == "ok"
+        assert not res_json.get("errors")
+        assert res_json["items_dropped"] == []
+        assert res_json['items']
+        assert len(res_json['items']) == len(expected_items)
+        assert res_json["items"] == expected_items
+
+    def test_crawl_with_argument_invalid_json(self, server):
+        url = server.url("crawl.json")
+        argument = '"this is not valid json'
+        argument = quote(argument)
+        res = perform_get(url, {"spider_name": "test"}, {
+            "url": server.target_site.url("page1.html"),
+            "crawl_args": argument,
+            "callback": 'return_argument'
+        })
+        assert res.status_code == 400
+        res_json = res.json()
+        assert res_json["status"] == "error"
+        assert res_json.get('items') is None
+        assert res_json['code'] == 400
+        assert re.search(' must be valid url encoded JSON', res_json['message'])
+
+    def test_crawl_with_argument_invalid_name(self, server):
+        url = server.url("crawl.json")
+        argument = quote(json.dumps({"parse": "string"}))
+        res = perform_get(url, {"spider_name": "test"}, {
+            "url": server.target_site.url("page1.html"),
+            "crawl_args": argument,
+        })
+
+        def check_res(res):
+            res_json = res.json()
+            assert res.status_code == 400
+            assert res_json["status"] == "error"
+            assert res_json.get('items') is None
+            assert res_json['code'] == 400
+
+            msg = 'Crawl argument cannot override spider method'
+            assert re.search(msg, res_json['message'])
+
+        check_res(res)
+
+        res = perform_post(url, {
+            "spider_name": "test",
+            "crawl_args": argument
+        }, {
+            "url": server.target_site.url("page1.html"),
+            "callback": 'return_argument'
+        })
+
+        check_res(res)
+
+    def test_crawl_with_argument_attribute_collision(self, server):
+        """If there is attribute collision and some argument to spider
+         passed via API, and this argument collides with spider attribute,
+         argument from request overrides spider attribute.
+        """
+        url = server.url("crawl.json")
+        argument = quote(json.dumps({"some_attribute": "string"}))
+        res = perform_get(url, {"spider_name": "test"}, {
+            "url": server.target_site.url("page1.html"),
+            "crawl_args": argument,
+        })
+
+        def check_res(res):
+            res_json = res.json()
+            assert res_json["status"] == "ok"
+            assert res.status_code == 200
+            assert res_json['items']
+            assert len(res_json['items']) == 1
+
+        check_res(res)