[resources, tests] unit tests for passing arguments

to spider.

Author: pawelmhm

scrapyrt/resources.py

@@ -154,10 +154,12 @@ def render_POST(self, request, **kwargs):
         """
         request_body = request.content.getvalue()
         try:
+            # TODO replace demjson with json.loads
             api_params = demjson.decode(request_body)
         except demjson.JSONDecodeError as e:
             message = "Invalid JSON in POST body. {}"
             message = message.format(e.pretty_description())
+            # TODO should be integer not string
             raise Error('400', message=message)
 
         log.msg("{}".format(api_params))
@@ -169,7 +171,7 @@ def render_POST(self, request, **kwargs):
             _request = self.get_required_argument(api_params, "request")
         try:
             scrapy_request_args = extract_scrapy_request_args(
-                _request, raise_error=True
+                _request, raise_error=False
             )
         except ValueError as e:
             raise Error('400', str(e))

scrapyrt/utils.py

@@ -15,7 +15,7 @@ def extract_scrapy_request_args(dictionary, raise_error=False):
             arguments.
     """
     result = dictionary.copy()
-    args = inspect.getargspec(Request.__init__).args
+    args = inspect.getfullargspec(Request.__init__).args
     for key in dictionary.keys():
         if key not in args:
             result.pop(key)

tests/sample_data/testproject/testproject/spiders/testspider.py

@@ -18,3 +18,6 @@ def return_bytes(self, response):
     def some_errback(self, err):
         self.logger.error("Logging some error {}".format(err))
         return
+
+    def return_argument(self, response):
+        return TestprojectItem(name=self.postcode)

tests/test_resource_crawl.py

@@ -412,3 +412,25 @@ def test_bytes_in_item(self, server, method):
                       'callback': 'return_bytes'})
         assert res.status_code == 200
         assert res.json()["items"] == [{'name': 'Some bytes here'}]
+
+    @pytest.mark.parametrize("method", [
+        perform_get, perform_post
+    ])
+    def test_crawl_with_argument(self, server, method):
+        url = server.url("crawl.json")
+        postcode = "43-300"
+        res = method(url, {"spider_name": "test"}, {
+            "url": server.target_site.url("page1.html"),
+            "postcode": postcode,
+            "callback": 'return_argument'
+        })
+        expected_items = [{
+            u'name': postcode,
+        }]
+        res_json = res.json()
+        assert res_json["status"] == "ok"
+        assert res_json["items_dropped"] == []
+
+        assert res_json['items']
+        assert len(res_json['items']) == len(expected_items)
+        assert res_json["items"] == expected_items