[resources] unit failing test for returning bytes

Author: pawelmhm

scrapyrt/resources.py

@@ -49,12 +49,10 @@ def handle_error(self, exception_or_failure, request):
         :return: dict which will be converted to JSON error response
 
         """
-        failure = None
         if isinstance(exception_or_failure, Exception):
             exception = exception_or_failure
         elif isinstance(exception_or_failure, Failure):
             exception = exception_or_failure.value
-            failure = exception_or_failure
         else:
             raise TypeError(
                 'Expected Exception or {} instances, got {}'.format(
@@ -71,22 +69,27 @@ def handle_error(self, exception_or_failure, request):
             else:
                 request.setResponseCode(500)
             if request.code == 500:
-                log.err(failure)
+                log.err(str(exception_or_failure))
         return self.format_error_response(exception, request)
 
     def format_error_response(self, exception, request):
         # Python exceptions don't have message attribute in Python 3+ anymore.
         # Twisted HTTP Error objects still have 'message' attribute even in 3+
         # and they fail on str(exception) call.
         msg = exception.message if hasattr(exception, 'message') else str(exception)
+
         return {
             "status": "error",
             "message": msg,
             "code": request.code
         }
 
     def render_object(self, obj, request):
-        r = self.json_encoder.encode(obj) + "\n"
+        try:
+            r = self.json_encoder.encode(obj) + "\n"
+        except Exception as e:
+            return self.json_encoder.encode(self.handle_error(e, request)).encode('utf8')
+
         request.setHeader('Content-Type', 'application/json')
         request.setHeader('Access-Control-Allow-Origin', '*')
         request.setHeader('Access-Control-Allow-Methods',

tests/sample_data/testproject/testproject/spiders/testspider.py

@@ -12,6 +12,9 @@ def parse(self, response):
         name = response.xpath('//h1/text()').extract()
         return TestprojectItem(name=name)
 
+    def return_bytes(self, response):
+        return TestprojectItem(name=b'Some bytes here')
+
     def some_errback(self, err):
         self.logger.error("Logging some error {}".format(err))
         return

tests/test_resource_crawl.py

@@ -400,3 +400,15 @@ def test_passing_errback(self, server, method):
 
         msg = 'ERROR: Logging some error'
         assert re.search(msg, log_file)
+
+
+    @pytest.mark.parametrize("method", [
+        perform_get, perform_post
+    ])
+    def test_bytes_in_item(self, server, method):
+        url = server.url("crawl.json")
+        res = method(url,
+                     {"spider_name": "test"},
+                     {"url": server.target_site.url("page1.html"),
+                      'callback': 'return_bytes'})
+        assert res.status_code == 200