Merge branch 'master' into pedro-errback

Author: pawelmhm

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.10.0
+current_version = 0.11.0
 commit = True
 tag = True
 tag_name = {new_version}

.travis.yml

@@ -6,19 +6,17 @@ matrix:
      - python: 2.7
        env: TOXENV=py27
      - python: 2.7
-       env: TOXENV=py27-scrapy1.0
+       env: TOXENV=py27-scrapy1.5
      - python: 2.7
-       env: TOXENV=py27-scrapy1.1
-     - python: 2.7
-       env: TOXENV=py27-scrapy1.2
+       env: TOXENV=py27-scrapy1.6
      - python: 3.5
        env: TOXENV=py35
      - python: 3.6
        env: TOXENV=py36
      - python: 3.6
-       env: TOXENV=py36-scrapy1.1
+       env: TOXENV=py36-scrapy1.5
      - python: 3.6
-       env: TOXENV=py36-scrapy1.2
+       env: TOXENV=py36-scrapy1.6
 script: tox
 
 deploy:

requirements-dev.txt

@@ -1,6 +1,6 @@
 bumpversion==0.5.3
 fabric
-requests==2.9.1
+requests==2.22.0
 mock==1.3.0
 pytest==2.9.1
 pytest-cov==2.2.1

requirements.txt

@@ -1,4 +1,4 @@
 Scrapy>=1.0.0
 service-identity>=1.0.0
-demjson
-six
+demjson==2.2.4
+six==1.12.0

scrapyrt/VERSION

@@ -1 +1 @@
-0.10.0
+0.11.0

scrapyrt/core.py

@@ -4,9 +4,8 @@
 import datetime
 import os
 import six
-import types
 
-from scrapy import signals, log as scrapy_log
+from scrapy import signals
 from scrapy.crawler import CrawlerRunner, Crawler
 from scrapy.exceptions import DontCloseSpider
 from scrapy.http import Request
@@ -85,45 +84,6 @@ def cleanup_logging(result):
 
         return dfd.addBoth(cleanup_logging)
 
-    def _setup_crawler_logging(self, crawler):
-        log_observer = scrapy_log.start_from_crawler(crawler)
-        if log_observer:
-            monkey_patch_and_connect_log_observer(crawler, log_observer)
-        if self.log_observer:
-            monkey_patch_and_connect_log_observer(crawler, self.log_observer)
-
-    def _stop_logging(self):
-        if self.log_observer:
-            try:
-                self.log_observer.stop()
-            except ValueError:
-                # exception on kill
-                # exceptions.ValueError: list.remove(x): x not in list
-                # looks like it's safe to ignore it
-                pass
-
-
-def monkey_patch_and_connect_log_observer(crawler, log_observer):
-    """Ugly hack to close log file.
-
-    Monkey patch log_observer.stop method to close file each time
-    log observer is closed.
-    I prefer this to be fixed in Scrapy itself, but as
-    Scrapy is going to switch to standart python logging soon
-    https://github.com/scrapy/scrapy/pull/1060
-    this change wouldn't be accepted in preference of merging
-    new logging sooner.
-
-    """
-    def stop_and_close_log_file(self):
-        self.__stop()
-        self.write.__self__.close()
-
-    log_observer.__stop = log_observer.stop
-    log_observer.stop = types.MethodType(
-        stop_and_close_log_file, log_observer)
-    crawler.signals.connect(log_observer.stop, signals.engine_stopped)
-
 
 class CrawlManager(object):
     """

scrapyrt/resources.py

@@ -92,7 +92,7 @@ def render_object(self, obj, request):
         request.setHeader('Access-Control-Allow-Methods',
                           ', '.join(getattr(self, 'allowedMethods', [])))
         request.setHeader('Access-Control-Allow-Headers', 'X-Requested-With')
-        request.setHeader('Content-Length', len(r))
+        request.setHeader('Content-Length', str(len(r)))
         return r.encode("utf8")
 
 

tests/test_resource_serviceresource.py

@@ -172,7 +172,7 @@ def test_render_object(self):
         set_header_mock.assert_any_call('Access-Control-Allow-Origin', '*')
         set_header_mock.assert_any_call('Access-Control-Allow-Headers',
                                         'X-Requested-With')
-        set_header_mock.assert_any_call('Content-Length', len(result))
+        set_header_mock.assert_any_call('Content-Length', str(len(result)))
         # request.setHeader('Access-Control-Allow-Methods',
         #                   ', '.join(getattr(self, 'allowedMethods', [])))
         headers = dict(self.headers)

tox.ini

@@ -1,10 +1,9 @@
 [tox]
-envlist = py27, py27-scrapy{1.0,1.1,1.2}, py35, py36, {py35,py36}-scrapy{1.1,1.2}
+envlist = py27, py27-scrapy{1.5,1.6}, py35, py36, {py35,py36}-scrapy{1.5,1.6}
 
 [testenv]
 deps =
-    scrapy1.0: Scrapy>=1.0,<1.1
-    scrapy1.1: Scrapy>=1.1,<1.2
-    scrapy1.2: Scrapy>=1.2,<1.3
+    scrapy1.5: Scrapy>=1.5,<1.6
+    scrapy1.6: Scrapy>=1.6,<1.7
     -r{toxinidir}/requirements-dev.txt
 commands = py.test {posargs}