Add user_error support

Currently the application is not reporting to the user when the user provides an invalid errback or callback method.  The scheduling of the request and validation of the spider callback and errback happens in a different thread than the one which is handling the api request. So, we need a different mechanism to communicate with the api request thread than simply raising the exception.  We already do this for other errors and responses by adding properties to the CrawlManager object.  So it seems best to also communicate this exception to the api request by using a user_error property on the CrawlManager.  Then the exception can be raised in the context of the api request.

Author: radostyle

scrapyrt/core.py

@@ -3,6 +3,7 @@
 from copy import deepcopy
 import datetime
 import os
+import traceback
 
 from scrapy import signals
 from scrapy.crawler import CrawlerRunner, Crawler
@@ -109,6 +110,7 @@ def __init__(self, spider_name, request_kwargs,
         self.items = []
         self.items_dropped = []
         self.errors = []
+        self.user_error = None
         self.max_requests = int(max_requests) if max_requests else None
         self.timeout_limit = int(app_settings.TIMEOUT_LIMIT)
         self.request_count = 0
@@ -171,22 +173,26 @@ def spider_idle(self, spider):
 
         """
         if spider is self.crawler.spider and self.request and not self._request_scheduled:
-            callback = getattr(self.crawler.spider, self.callback_name)
-            assert callable(callback), 'Invalid callback'
-            self.request = self.request.replace(callback=callback)
-
-
-            if self.errback_name:
-                errback = getattr(self.crawler.spider, self.errback_name)
-                assert callable(errback), 'Invalid errback'
-                self.request = self.request.replace(errback=errback)
-            modify_request = getattr(
-                self.crawler.spider, "modify_realtime_request", None)
-            if callable(modify_request):
-                self.request = modify_request(self.request)
-            spider.crawler.engine.crawl(self.request)
-            self._request_scheduled = True
-            raise DontCloseSpider
+            try:
+                callback = getattr(self.crawler.spider, self.callback_name)
+                assert callable(callback), 'Invalid callback'
+                self.request = self.request.replace(callback=callback)
+
+
+                if self.errback_name:
+                    errback = getattr(self.crawler.spider, self.errback_name)
+                    assert callable(errback), 'Invalid errback'
+                    self.request = self.request.replace(errback=errback)
+                modify_request = getattr(
+                    self.crawler.spider, "modify_realtime_request", None)
+                if callable(modify_request):
+                    self.request = modify_request(self.request)
+                spider.crawler.engine.crawl(self.request)
+                self._request_scheduled = True
+            except Exception as e:
+                self.user_error = Error(400, message=traceback.format_exc())
+            else:
+                raise DontCloseSpider
 
     def handle_scheduling(self, request, spider):
         """Handler of request_scheduled signal.
@@ -240,6 +246,9 @@ def return_items(self, result):
             "stats": stats,
             "spider_name": self.spider_name,
         }
+
+        results["user_error"] = self.user_error
+
         if self.debug:
             results["errors"] = self.errors
         return results

scrapyrt/resources.py

@@ -261,6 +261,9 @@ def run_crawl(self, spider_name, scrapy_request_args,
 
     def prepare_response(self, result, *args, **kwargs):
         items = result.get("items")
+        user_error = result.get("user_error", None)
+        if user_error:
+            raise user_error
         response = {
             "status": "ok",
             "items": items,

tests/test_crawl_manager.py

@@ -111,8 +111,10 @@ def test_spider_opened(self):
 
     def test_raise_error_if_not_callable(self):
         self.spider.parse_something = None
-        self.assertRaises(
-            AssertionError, self.crawl_manager.spider_idle, self.spider)
+        self._call_spider_idle()
+        self.assertIsNotNone(self.crawl_manager.user_error)
+        msg = "Invalid callback"
+        assert re.search(msg, self.crawl_manager.user_error.message)
         self.assertFalse(self.crawler.engine.crawl.called)
 
     def test_modify_realtime_request(self):
@@ -142,15 +144,17 @@ def test_pass_wrong_spider_errback(self):
         mng = self.create_crawl_manager(
             {'url': 'http://localhost', 'errback': 'handle_error'}
         )
+
         try:
-            with pytest.raises(AttributeError) as err:
-                mng.spider_idle(self.spider)
+            mng.spider_idle(self.spider)
         except DontCloseSpider:
             pass
 
         assert mng.request.errback is None
+
+        self.assertIsNotNone(mng.user_error)
         msg = "has no attribute 'handle_error'"
-        assert re.search(msg, str(err))
+        assert re.search(msg, mng.user_error.message)
 
     def test_pass_good_spider_errback(self):
         mng = self.create_crawl_manager(
@@ -330,6 +334,7 @@ def setUp(self):
             'items_dropped': self.crawl_manager.items_dropped,
             'stats': self.stats.copy(),
             'spider_name': self.spider.name,
+            'user_error': None,
         }
 
     def test_return_items(self):

tests/test_resource_crawl.py

@@ -142,6 +142,17 @@ def test_prepare_response(self, resource):
         for key, value in expected:
             assert prepared_res[key] == value
 
+    def test_prepare_response_user_error_raised(self, resource):
+        result = {
+            'items': [1, 2],
+            'stats': [99],
+            'spider_name': 'test'
+        }
+        result['user_error'] = Exception("my exception")
+        with pytest.raises(Exception) as e_info:
+            resource.prepare_response(result)
+            assert e_info.message == "my exception"
+            
 
 class TestCrawlResourceGetRequiredArgument(unittest.TestCase):