Merge pull request #49 from pawelmhm/python3

Python 3 support

Author: chekunkov

.travis.yml

@@ -1,7 +1,13 @@
 language: python
-python: 2.7
-env:
-- TOXENV=py27
 install:
   - pip install -U tox
-script: tox
+matrix:
+  include:
+     - python: 2.7
+       env: TOXENV=py27
+     - python: 3.5
+       env: TOXENV=py35
+     - python: 2.7
+       env: TOXENV=scrapy10
+
+script: tox

README.rst

@@ -5,6 +5,15 @@ Scrapyrt (Scrapy realtime)
 .. image:: https://travis-ci.org/scrapinghub/scrapyrt.svg?branch=master
     :target: https://travis-ci.org/scrapinghub/scrapyrt
 
+.. image:: https://img.shields.io/pypi/pyversions/scrapyrt.svg
+    :target: https://pypi.python.org/pypi/scrapyrt
+
+.. image:: https://img.shields.io/pypi/v/scrapyrt.svg
+    :target: https://pypi.python.org/pypi/scrapyrt
+
+.. image:: https://img.shields.io/pypi/l/scrapyrt.svg
+    :target: https://pypi.python.org/pypi/scrapyrt
+
 HTTP server which provides API for scheduling Scrapy spiders and
 making requests with spiders.
 

requirements-dev.txt

@@ -1,4 +1,3 @@
--r requirements.txt
 
 fabric
 requests==2.9.1

requirements.txt

@@ -1,3 +1,4 @@
 Scrapy>=1.0.0
 service-identity>=1.0.0
 demjson
+six

scrapyrt/cmdline.py

@@ -1,5 +1,7 @@
 # -*- coding: utf-8 -*-
-from ConfigParser import SafeConfigParser, NoOptionError, NoSectionError
+from six.moves.configparser import (
+    SafeConfigParser, NoOptionError, NoSectionError
+)
 import argparse
 import os
 import sys

scrapyrt/conf/__init__.py

@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+import six
 from copy import deepcopy
 from importlib import import_module
 
@@ -11,7 +12,7 @@ def __init__(self):
         self.setmodule(default_settings)
 
     def setmodule(self, module):
-        if isinstance(module, basestring):
+        if isinstance(module, six.string_types):
             module = import_module(module)
         for setting in dir(module):
             self.set(setting, getattr(module, setting))

scrapyrt/conf/spider_settings.py

@@ -11,10 +11,10 @@ def get_scrapyrt_settings(log_file=None):
         "LOG_FILE": log_file,
         "LOG_STDOUT": False,
         "EXTENSIONS": {
-            'scrapy.contrib.logstats.LogStats': None,
+            'scrapy.extensions.logstats.LogStats': None,
             'scrapy.webservice.WebService': None,
-            'scrapy.telnet.TelnetConsole': None,
-            'scrapy.contrib.throttle.AutoThrottle': None
+            'scrapy.extensions.telnet.TelnetConsole': None,
+            'scrapy.extensions.throttle.AutoThrottle': None
         }
     }
     return spider_settings

scrapyrt/core.py

@@ -60,7 +60,7 @@ def __init__(self, settings, scrapyrt_manager):
 
     def crawl(self, spidercls, *args, **kwargs):
         if isinstance(spidercls, six.string_types):
-            spidercls = self.spiders.load(spidercls)
+            spidercls = self.spider_loader.load(spidercls)
         # creating our own crawler that will allow us to disable start requests easily
         crawler = ScrapyrtCrawler(
             spidercls, self.settings, self.scrapyrt_manager.start_requests)
@@ -159,7 +159,7 @@ def crawl(self, *args, **kwargs):
             dfd = self.crawler_process.crawl(self.spider_name, *args, **kwargs)
         except KeyError as e:
             # Spider not found.
-            raise Error('404', message=e.message)
+            raise Error('404', message=str(e))
         dfd.addCallback(self.return_items)
         return dfd
 
@@ -245,7 +245,7 @@ def collect_dropped(self, item, response, exception, spider):
         if spider is self.crawler.spider:
             self.items_dropped.append({
                 "item": item,
-                "exception": exception.message,
+                "exception": str(exception),
                 "response": response
             })
 
@@ -267,7 +267,8 @@ def create_spider_request(self, kwargs):
         try:
             req = Request(url, **kwargs)
         except (TypeError, ValueError) as e:
-            message = "Error while creating Scrapy Request, {}".format(e.message)
+            msg = "Error while creating Scrapy Request, {}"
+            message = msg.format(str(e))
             raise Error('400', message=message)
 
         req.dont_filter = True

scrapyrt/log.py

@@ -1,17 +1,17 @@
 # -*- coding: utf-8 -*-
-from logging.config import dictConfig
 import logging
 import os
 import sys
+from logging.config import dictConfig
 
-from scrapy.utils.python import unicode_to_str
+from scrapy.settings import Settings
+from scrapy.utils.log import DEFAULT_LOGGING, TopLevelFormatter
 from twisted.python import log
 from twisted.python.log import startLoggingWithObserver
 from twisted.python.logfile import DailyLogFile
-from scrapy.settings import Settings
-from scrapy.utils.log import DEFAULT_LOGGING, TopLevelFormatter
 
 from .conf import settings as scrapyrt_settings
+from .utils import to_bytes
 
 DEBUG = logging.DEBUG
 INFO = logging.INFO
@@ -64,7 +64,7 @@ def _unicode_to_str(self, eventDict):
         message = eventDict.get('message')
         if message:
             eventDict['message'] = tuple(
-                unicode_to_str(x, self.encoding) for x in message)
+                to_bytes(x, self.encoding) for x in message)
         return eventDict
 
     def emit(self, eventDict):