TST run tests with Scrapy 1.0 again

Author: kmike

.travis.yml

@@ -20,7 +20,7 @@ matrix:
         - python: 3.6
           env: TOXENV=py36
         - python: 2.7
-          env: TOXENV=py27-scrapy11
+          env: TOXENV=py27-scrapy10
 
 before_install:
     - docker pull scrapinghub/splash

tests/test_integration.py

@@ -78,7 +78,7 @@ def test_basic(settings):
     assert len(items) == 1
     resp = items[0]['response']
     assert resp.url == url
-    assert resp.css('body::text').get().strip() == "hello world!"
+    assert resp.css('body::text').extract_first().strip() == "hello world!"
 
 
 @requires_splash
@@ -99,7 +99,7 @@ def parse(self, response):
     assert crawler.stats.get_value('dupefilter/filtered') == 1
     resp = items[0]['response']
     assert resp.url == url
-    assert resp.css('body::text').get().strip() == "hello world!"
+    assert resp.css('body::text').extract_first().strip() == "hello world!"
     assert resp.status == resp.splash_response_status == 200
     assert resp.headers == resp.splash_response_headers
     assert resp.splash_response_headers['Content-Type'] == b"text/html; charset=utf-8"
@@ -128,7 +128,7 @@ def start_requests(self):
     resp = items[0]['response']
     assert resp.url == url + "/#foo"
     assert resp.status == resp.splash_response_status == 200
-    assert resp.css('body::text').get().strip() == "hello world!"
+    assert resp.css('body::text').extract_first().strip() == "hello world!"
     assert resp.data['jsvalue'] == 3
     assert resp.headers['X-MyHeader'] == b'my value'
     assert resp.headers['Content-Type'] == b'text/html'

tests/utils.py

@@ -4,8 +4,9 @@
 from pytest_twisted import inlineCallbacks
 from twisted.internet.defer import returnValue
 from twisted.web.resource import Resource
-from scrapy.crawler import CrawlerRunner
-from scrapy.utils.python import to_bytes
+from scrapy.crawler import Crawler
+
+from scrapy_splash.utils import to_bytes
 from tests.mockserver import MockServer
 
 
@@ -52,7 +53,7 @@ class Spider(spider_cls):
         Spider.__name__ = spider_cls.__name__
         Spider.__module__ = spider_cls.__module__
         spider_cls = Spider
-    return CrawlerRunner(settings).create_crawler(spider_cls)
+    return Crawler(spider_cls, settings)
 
 
 class CollectorPipeline:

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27,py34,py35,py36,py27-scrapy11
+envlist = py27,py34,py35,py36,py27-scrapy10
 
 [testenv]
 passenv = SPLASH_URL
@@ -30,8 +30,8 @@ deps = {[testenv:py34]deps}
 basepython = python3.6
 deps = {[testenv:py34]deps}
 
-[testenv:py27-scrapy11]
+[testenv:py27-scrapy10]
 deps =
     -rrequirements-test.txt
-    scrapy == 1.1.4
+    scrapy < 1.1
     service_identity