/resources/ handle bytes in item from spider

ScrapyJSONEncoder should not crash if there are bytes in item generated
by spider. Also added error handling, we should not hang when there is
something unexpected in API, just return to 500 and go on.

Author: pawelmhm

.gitignore

@@ -33,3 +33,4 @@ testing_scripts*
 
 # sphinx
 docs/build/*
+.python-version

requirements.txt

@@ -2,3 +2,5 @@ Scrapy>=1.0.0
 service-identity>=1.0.0
 demjson==2.2.4
 six==1.12.0
+jmespath==0.10.0
+pyasn1==0.4.8

scrapyrt/resources.py

@@ -12,10 +12,18 @@
 from .utils import extract_scrapy_request_args, to_bytes
 
 
+class AdaptedScrapyJSONEncoder(ScrapyJSONEncoder):
+    def default(self, o):
+        if isinstance(o, bytes):
+            return o.decode('utf8')
+        else:
+            return super().default(o)
+
+
 # XXX super() calls won't work wihout object mixin in Python 2
 # maybe this can be removed at some point?
 class ServiceResource(resource.Resource, object):
-    json_encoder = ScrapyJSONEncoder()
+    json_encoder = AdaptedScrapyJSONEncoder()
 
     def __init__(self, root=None):
         resource.Resource.__init__(self)
@@ -69,7 +77,7 @@ def handle_error(self, exception_or_failure, request):
             else:
                 request.setResponseCode(500)
             if request.code == 500:
-                log.err(str(exception_or_failure))
+                log.err(f'Error when rendering response: "{exception}"')
         return self.format_error_response(exception, request)
 
     def format_error_response(self, exception, request):
@@ -78,17 +86,18 @@ def format_error_response(self, exception, request):
         # and they fail on str(exception) call.
         msg = exception.message if hasattr(exception, 'message') else str(exception)
 
-        return {
+        encoded_error_response = self.json_encoder.encode({
             "status": "error",
             "message": msg,
             "code": request.code
-        }
+        }) + "\n"
+        return encoded_error_response.encode('utf8')
 
     def render_object(self, obj, request):
         try:
             r = self.json_encoder.encode(obj) + "\n"
-        except Exception as e:
-            return self.json_encoder.encode(self.handle_error(e, request)).encode('utf8')
+        except Exception as exc:
+            return self.handle_error(exc, request)
 
         request.setHeader('Content-Type', 'application/json')
         request.setHeader('Access-Control-Allow-Origin', '*')