[spider manager] handling crawl params edge cases

Author: pawelmhm

scrapyrt/core.py

@@ -60,6 +60,13 @@ def __init__(self, settings, scrapyrt_manager):
     def crawl(self, spidercls, *args, **kwargs):
         if isinstance(spidercls, six.string_types):
             spidercls = self.spider_loader.load(spidercls)
+
+        for kw in kwargs:
+            attr_or_m = getattr(spidercls, kw, None)
+            if attr_or_m and callable(attr_or_m):
+                msg = 'Crawl argument cannot override spider method.'
+                msg += ' Got argument {} that overrides spider method {}'
+                raise Error('400', message=msg.format(kw, getattr(spidercls, kw)))
         # creating our own crawler that will allow us to disable start requests easily
         crawler = ScrapyrtCrawler(
             spidercls, self.settings, self.scrapyrt_manager.start_requests)

tests/sample_data/testproject/testproject/spiders/testspider.py

@@ -7,6 +7,7 @@
 class TestSpider(scrapy.Spider):
 
     name = 'test'
+    some_attribute = "Yes|No"
 
     def parse(self, response):
         name = response.xpath('//h1/text()').extract()

tests/test_resource_crawl.py

@@ -449,6 +449,7 @@ def test_crawl_with_argument_post(self, server):
             u'name': postcode,
         }]
         res_json = res.json()
+        assert res.status_code == 200
         assert res_json["status"] == "ok"
         assert not res_json.get("errors")
         assert res_json["items_dropped"] == []
@@ -458,19 +459,67 @@ def test_crawl_with_argument_post(self, server):
 
     def test_crawl_with_argument_invalid_json(self, server):
         url = server.url("crawl.json")
-        postcode = "43-300"
         argument = '"this is not valid json'
         argument = quote(argument)
         res = perform_get(url, {"spider_name": "test"}, {
             "url": server.target_site.url("page1.html"),
             "crawl_args": argument,
             "callback": 'return_argument'
         })
-        expected_items = [{
-            u'name': postcode,
-        }]
+        assert res.status_code == 400
         res_json = res.json()
         assert res_json["status"] == "error"
         assert res_json.get('items') is None
         assert res_json['code'] == 400
         assert re.search(' must be valid url encoded JSON', res_json['message'])
+
+    def test_crawl_with_argument_invalid_name(self, server):
+        url = server.url("crawl.json")
+        argument = quote(json.dumps({"parse": "string"}))
+        res = perform_get(url, {"spider_name": "test"}, {
+            "url": server.target_site.url("page1.html"),
+            "crawl_args": argument,
+        })
+
+        def check_res(res):
+            res_json = res.json()
+            assert res.status_code == 400
+            assert res_json["status"] == "error"
+            assert res_json.get('items') is None
+            assert res_json['code'] == 400
+
+            msg = 'Crawl argument cannot override spider method'
+            assert re.search(msg, res_json['message'])
+
+        check_res(res)
+
+        res = perform_post(url, {
+            "spider_name": "test",
+            "crawl_args": argument
+        }, {
+            "url": server.target_site.url("page1.html"),
+            "callback": 'return_argument'
+        })
+
+        check_res(res)
+
+    def test_crawl_with_argument_attribute_collision(self, server):
+        """If there is attribute collision and some argument to spider
+         passed via API, and this argument collides with spider attribute,
+         argument from request overrides spider attribute.
+        """
+        url = server.url("crawl.json")
+        argument = quote(json.dumps({"some_attribute": "string"}))
+        res = perform_get(url, {"spider_name": "test"}, {
+            "url": server.target_site.url("page1.html"),
+            "crawl_args": argument,
+        })
+
+        def check_res(res):
+            res_json = res.json()
+            assert res_json["status"] == "ok"
+            assert res.status_code == 200
+            assert res_json['items']
+            assert len(res_json['items']) == 1
+
+        check_res(res)