Merge pull request #90 from scrapinghub/logging-update

remove dead logging code + upgrade scrapy versions in travis cli

Author: pawelmhm

.travis.yml

@@ -6,19 +6,17 @@ matrix:
      - python: 2.7
        env: TOXENV=py27
      - python: 2.7
-       env: TOXENV=py27-scrapy1.0
+       env: TOXENV=py27-scrapy1.5
      - python: 2.7
-       env: TOXENV=py27-scrapy1.1
-     - python: 2.7
-       env: TOXENV=py27-scrapy1.2
+       env: TOXENV=py27-scrapy1.6
      - python: 3.5
        env: TOXENV=py35
      - python: 3.6
        env: TOXENV=py36
      - python: 3.6
-       env: TOXENV=py36-scrapy1.1
+       env: TOXENV=py36-scrapy1.5
      - python: 3.6
-       env: TOXENV=py36-scrapy1.2
+       env: TOXENV=py36-scrapy1.6
 script: tox
 
 deploy:

requirements-dev.txt

@@ -1,6 +1,6 @@
 bumpversion==0.5.3
 fabric
-requests==2.9.1
+requests==2.22.0
 mock==1.3.0
 pytest==2.9.1
 pytest-cov==2.2.1

requirements.txt

@@ -1,4 +1,4 @@
 Scrapy>=1.0.0
 service-identity>=1.0.0
-demjson
-six
+demjson==2.2.4
+six==1.12.0

scrapyrt/core.py

@@ -4,9 +4,8 @@
 import datetime
 import os
 import six
-import types
 
-from scrapy import signals, log as scrapy_log
+from scrapy import signals
 from scrapy.crawler import CrawlerRunner, Crawler
 from scrapy.exceptions import DontCloseSpider
 from scrapy.http import Request
@@ -85,45 +84,6 @@ def cleanup_logging(result):
 
         return dfd.addBoth(cleanup_logging)
 
-    def _setup_crawler_logging(self, crawler):
-        log_observer = scrapy_log.start_from_crawler(crawler)
-        if log_observer:
-            monkey_patch_and_connect_log_observer(crawler, log_observer)
-        if self.log_observer:
-            monkey_patch_and_connect_log_observer(crawler, self.log_observer)
-
-    def _stop_logging(self):
-        if self.log_observer:
-            try:
-                self.log_observer.stop()
-            except ValueError:
-                # exception on kill
-                # exceptions.ValueError: list.remove(x): x not in list
-                # looks like it's safe to ignore it
-                pass
-
-
-def monkey_patch_and_connect_log_observer(crawler, log_observer):
-    """Ugly hack to close log file.
-
-    Monkey patch log_observer.stop method to close file each time
-    log observer is closed.
-    I prefer this to be fixed in Scrapy itself, but as
-    Scrapy is going to switch to standart python logging soon
-    https://github.com/scrapy/scrapy/pull/1060
-    this change wouldn't be accepted in preference of merging
-    new logging sooner.
-
-    """
-    def stop_and_close_log_file(self):
-        self.__stop()
-        self.write.__self__.close()
-
-    log_observer.__stop = log_observer.stop
-    log_observer.stop = types.MethodType(
-        stop_and_close_log_file, log_observer)
-    crawler.signals.connect(log_observer.stop, signals.engine_stopped)
-
 
 class CrawlManager(object):
     """

tox.ini

@@ -1,10 +1,9 @@
 [tox]
-envlist = py27, py27-scrapy{1.0,1.1,1.2}, py35, py36, {py35,py36}-scrapy{1.1,1.2}
+envlist = py27, py27-scrapy{1.5,1.6}, py35, py36, {py35,py36}-scrapy{1.5,1.6}
 
 [testenv]
 deps =
-    scrapy1.0: Scrapy>=1.0,<1.1
-    scrapy1.1: Scrapy>=1.1,<1.2
-    scrapy1.2: Scrapy>=1.2,<1.3
+    scrapy1.5: Scrapy>=1.5,<1.6
+    scrapy1.6: Scrapy>=1.6,<1.7
     -r{toxinidir}/requirements-dev.txt
 commands = py.test {posargs}