Merge pull request #122 from scrapinghub/remove-demjson

Remove demjson

Author: pawelmhm

requirements.txt

@@ -1,6 +1 @@
 Scrapy>=1.0.0
-service-identity>=1.0.0
-demjson>=2.2.4
-six>=1.12.0
-jmespath==0.10.0
-pyasn1>=0.4.8

scrapyrt/conf/__init__.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-import six
 from copy import deepcopy
 from importlib import import_module
 
@@ -12,7 +11,7 @@ def __init__(self):
         self.setmodule(default_settings)
 
     def setmodule(self, module):
-        if isinstance(module, six.string_types):
+        if isinstance(module, str):
             module = import_module(module)
         for setting in dir(module):
             self.set(setting, getattr(module, setting))

scrapyrt/core.py

@@ -3,7 +3,6 @@
 from copy import deepcopy
 import datetime
 import os
-import six
 
 from scrapy import signals
 from scrapy.crawler import CrawlerRunner, Crawler
@@ -58,7 +57,7 @@ def __init__(self, settings, scrapyrt_manager):
         self.scrapyrt_manager = scrapyrt_manager
 
     def crawl(self, spidercls, *args, **kwargs):
-        if isinstance(spidercls, six.string_types):
+        if isinstance(spidercls, str):
             spidercls = self.spider_loader.load(spidercls)
 
         for kw in kwargs:

scrapyrt/resources.py

@@ -2,7 +2,6 @@
 import json
 from urllib.parse import unquote
 
-import demjson
 from scrapy.utils.misc import load_object
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from twisted.internet.defer import Deferred
@@ -158,12 +157,11 @@ def render_POST(self, request, **kwargs):
         """
         request_body = request.content.getvalue()
         try:
-            # TODO replace demjson with json.loads
-            api_params = demjson.decode(request_body)
-        except demjson.JSONDecodeError as e:
+            api_params = json.loads(request_body)
+        except Exception as e:
             message = "Invalid JSON in POST body. {}"
-            message = message.format(e.pretty_description())
-            # TODO should be integer not string
+            message = message.format(e)
+            # TODO should be integer not string?
             raise Error('400', message=message)
 
         log.msg("{}".format(api_params))

scrapyrt/utils.py

@@ -1,8 +1,5 @@
 import inspect
 
-import sys
-
-import six
 from scrapy import Request
 
 
@@ -33,7 +30,7 @@ def to_bytes(text, encoding=None, errors='strict'):
         is already a bytes object, return it as-is."""
         if isinstance(text, bytes):
             return text
-        if not isinstance(text, six.string_types):
+        if not isinstance(text, str):
             raise TypeError('to_bytes must receive a unicode, str or bytes '
                             'object, got %s' % type(text).__name__)
         if encoding is None:

setup.py

@@ -37,10 +37,7 @@
         'License :: OSI Approved :: BSD License',
     ],
     install_requires=[
-        'Twisted>=14.0.0',
-        'Scrapy>=1.0.0',
-        'demjson',
-        'six>=1.5.2'
+        'Scrapy>=1.0.0'
     ],
     package_data={
         'scrapyrt': [

tests/servers.py

@@ -7,9 +7,9 @@
 import tempfile
 import time
 from subprocess import Popen, PIPE
+from urllib.parse import urljoin
 
 import port_for
-from six.moves.urllib.parse import urljoin
 
 from . import TESTS_PATH
 from .utils import get_testenv, generate_project

tests/test_crawl_manager.py

@@ -334,7 +334,7 @@ def setUp(self):
 
     def test_return_items(self):
         result = self.crawl_manager.return_items(None)
-        self.assertDictContainsSubset(self.expected_result, result)
+        self.assertEqual(dict(result, **self.expected_result), result)
         self.assertEqual(list(sorted(self.stats.keys())), list(result['stats'].keys()))
         # debug = True by default
         self.assertIn('errors', result)

tests/test_log_observer.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from six import StringIO
+from io import StringIO
 
 from mock import patch
 from twisted.python.log import startLoggingWithObserver, removeObserver

tests/test_resource_realtimeapi.py

@@ -9,7 +9,7 @@
 from scrapyrt.resources import RealtimeApi, ServiceResource, CrawlResource
 
 
-class TestResource(ServiceResource):
+class SampleResource(ServiceResource):
     isLeaf = True
     allowedMethods = ['GET', 'POST']
 
@@ -35,10 +35,10 @@ def test_realtimeapi_with_default_settings(self):
     @patch('scrapyrt.resources.settings', deepcopy(settings))
     def test_realtimeapi_with_custom_settings(self):
         from scrapyrt.resources import settings
-        settings.RESOURCES[b'test.json'] = self._get_class_path('TestResource')
+        settings.RESOURCES[b'test.json'] = self._get_class_path('SampleResource')
         expected_entities = {
             b'crawl.json': CrawlResource,
-            b'test.json': TestResource
+            b'test.json': SampleResource
         }
         service_root = RealtimeApi()
         self._check_entities(service_root, expected_entities)