Merge pull request #125 from scrapy/py37

Python 3.7 support

Author: lopuhin

.travis.yml

@@ -18,17 +18,21 @@ matrix:
       env: TOXENV=py35
     - python: 3.6
       env: TOXENV=py36
+    - python: 3.7
+      env: TOXENV=py37
+      dist: xenial
+      sudo: true
 install:
   - |
       if [ "$TOXENV" = "pypy" ]; then
-        export PYPY_VERSION="pypy-5.9-linux_x86_64-portable"
+        export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
         wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
         source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
       fi
       if [ "$TOXENV" = "pypy3" ]; then
-        export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
+        export PYPY_VERSION="pypy3.5-6.0.0-linux_x86_64-portable"
         wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
         tar -jxf ${PYPY_VERSION}.tar.bz2
         virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"

setup.py

@@ -72,6 +72,7 @@ def has_environment_marker_platform_impl_support():
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
     ],

tests/test_selector.py

@@ -114,16 +114,16 @@ def test_accessing_attributes(self):
 </html>
         """
         sel = self.sscls(text=body)
-        self.assertEquals({'lang': 'en', 'version': '1.0'}, sel.attrib)
-        self.assertEquals({'id': 'some-list', 'class': 'list-cls'}, sel.css('ul')[0].attrib)
+        self.assertEqual({'lang': 'en', 'version': '1.0'}, sel.attrib)
+        self.assertEqual({'id': 'some-list', 'class': 'list-cls'}, sel.css('ul')[0].attrib)
 
         # for a SelectorList, bring the attributes of first-element only
-        self.assertEquals({'id': 'some-list', 'class': 'list-cls'}, sel.css('ul').attrib)
-        self.assertEquals({'class': 'item-cls', 'id': 'list-item-1'}, sel.css('li').attrib)
-        self.assertEquals({}, sel.css('body').attrib)
-        self.assertEquals({}, sel.css('non-existing-element').attrib)
+        self.assertEqual({'id': 'some-list', 'class': 'list-cls'}, sel.css('ul').attrib)
+        self.assertEqual({'class': 'item-cls', 'id': 'list-item-1'}, sel.css('li').attrib)
+        self.assertEqual({}, sel.css('body').attrib)
+        self.assertEqual({}, sel.css('non-existing-element').attrib)
 
-        self.assertEquals(
+        self.assertEqual(
             [{'class': 'item-cls', 'id': 'list-item-1'},
              {'class': 'item-cls active', 'id': 'list-item-2'},
              {'class': 'item-cls', 'id': 'list-item-3'}],
@@ -211,17 +211,17 @@ def test_re_first(self):
         body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
         sel = self.sscls(text=body)
 
-        self.assertEqual(sel.xpath('//ul/li/text()').re_first('\d'),
-                         sel.xpath('//ul/li/text()').re('\d')[0])
+        self.assertEqual(sel.xpath('//ul/li/text()').re_first(r'\d'),
+                         sel.xpath('//ul/li/text()').re(r'\d')[0])
 
-        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first('\d'),
-                         sel.xpath('//ul/li[@id="1"]/text()').re('\d')[0])
+        self.assertEqual(sel.xpath('//ul/li[@id="1"]/text()').re_first(r'\d'),
+                         sel.xpath('//ul/li[@id="1"]/text()').re(r'\d')[0])
 
-        self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first('\d'),
-                         sel.xpath('//ul/li/text()').re('\d')[1])
+        self.assertEqual(sel.xpath('//ul/li[2]/text()').re_first(r'\d'),
+                         sel.xpath('//ul/li/text()').re(r'\d')[1])
 
-        self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+'), None)
-        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first('\d'), None)
+        self.assertEqual(sel.xpath('/ul/li/text()').re_first(r'\w+'), None)
+        self.assertEqual(sel.xpath('/ul/li[@id="doesnt-exist"]/text()').re_first(r'\d'), None)
 
         self.assertEqual(sel.re_first(r'id="(\d+)'), '1')
         self.assertEqual(sel.re_first(r'foo'), None)
@@ -232,8 +232,8 @@ def test_extract_first_default(self):
         body = u'<ul><li id="1">1</li><li id="2">2</li></ul>'
         sel = self.sscls(text=body)
 
-        self.assertEqual(sel.xpath('//div/text()').re_first('\w+', default='missing'), 'missing')
-        self.assertEqual(sel.xpath('/ul/li/text()').re_first('\w+', default='missing'), 'missing')
+        self.assertEqual(sel.xpath('//div/text()').re_first(r'\w+', default='missing'), 'missing')
+        self.assertEqual(sel.xpath('/ul/li/text()').re_first(r'\w+', default='missing'), 'missing')
 
     def test_select_unicode_query(self):
         body = u"<p><input name='\xa9' value='1'/></p>"
@@ -249,8 +249,8 @@ def test_list_elements_type(self):
     def test_boolean_result(self):
         body = u"<p><input name='a'value='1'/><input name='b'value='2'/></p>"
         xs = self.sscls(text=body)
-        self.assertEquals(xs.xpath("//input[@name='a']/@name='a'").extract(), [u'1'])
-        self.assertEquals(xs.xpath("//input[@name='a']/@name='n'").extract(), [u'0'])
+        self.assertEqual(xs.xpath("//input[@name='a']/@name='a'").extract(), [u'1'])
+        self.assertEqual(xs.xpath("//input[@name='a']/@name='n'").extract(), [u'0'])
 
     def test_differences_parsing_xml_vs_html(self):
         """Test that XML and HTML Selector's behave differently"""
@@ -493,10 +493,10 @@ def test_re(self):
                   </div>"""
         x = self.sscls(text=body)
 
-        name_re = re.compile("Name: (\w+)")
+        name_re = re.compile(r"Name: (\w+)")
         self.assertEqual(x.xpath("//ul/li").re(name_re),
                          ["John", "Paul"])
-        self.assertEqual(x.xpath("//ul/li").re("Age: (\d+)"),
+        self.assertEqual(x.xpath("//ul/li").re(r"Age: (\d+)"),
                          ["10", "20"])
 
         # Test named group, hit and miss
@@ -537,7 +537,7 @@ def test_re_replace_entities(self):
     def test_re_intl(self):
         body = u'<div>Evento: cumplea\xf1os</div>'
         x = self.sscls(text=body)
-        self.assertEqual(x.xpath("//div").re("Evento: (\w+)"), [u'cumplea\xf1os'])
+        self.assertEqual(x.xpath("//div").re(r"Evento: (\w+)"), [u'cumplea\xf1os'])
 
     def test_selector_over_text(self):
         hs = self.sscls(text=u'<root>lala</root>')
@@ -568,15 +568,15 @@ def test_http_header_encoding_precedence(self):
         <head><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1"></head>
         <body><span id="blank">\xa3</span></body></html>'''
         x = self.sscls(text=text)
-        self.assertEquals(x.xpath("//span[@id='blank']/text()").extract(),
+        self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(),
                           [u'\xa3'])
 
     def test_empty_bodies_shouldnt_raise_errors(self):
         self.sscls(text=u'').xpath('//text()').extract()
 
     def test_bodies_with_comments_only(self):
         sel = self.sscls(text=u'<!-- hello world -->', base_url='http://example.com')
-        self.assertEquals(u'http://example.com', sel.root.base)
+        self.assertEqual(u'http://example.com', sel.root.base)
 
     def test_null_bytes_shouldnt_raise_errors(self):
         text = u'<root>pre\x00post</root>'
@@ -585,35 +585,35 @@ def test_null_bytes_shouldnt_raise_errors(self):
     def test_replacement_char_from_badly_encoded_body(self):
         # \xe9 alone isn't valid utf8 sequence
         text = u'<html><p>an Jos\ufffd de</p><html>'
-        self.assertEquals([u'an Jos\ufffd de'],
-                          self.sscls(text).xpath('//text()').extract())
+        self.assertEqual([u'an Jos\ufffd de'],
+                         self.sscls(text).xpath('//text()').extract())
 
     def test_select_on_unevaluable_nodes(self):
         r = self.sscls(text=u'<span class="big">some text</span>')
         # Text node
         x1 = r.xpath('//text()')
-        self.assertEquals(x1.extract(), [u'some text'])
-        self.assertEquals(x1.xpath('.//b').extract(), [])
+        self.assertEqual(x1.extract(), [u'some text'])
+        self.assertEqual(x1.xpath('.//b').extract(), [])
         # Tag attribute
         x1 = r.xpath('//span/@class')
-        self.assertEquals(x1.extract(), [u'big'])
-        self.assertEquals(x1.xpath('.//text()').extract(), [])
+        self.assertEqual(x1.extract(), [u'big'])
+        self.assertEqual(x1.xpath('.//text()').extract(), [])
 
     def test_select_on_text_nodes(self):
         r = self.sscls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
         x1 = r.xpath("//div/descendant::text()[preceding-sibling::b[contains(text(), 'Options')]]")
-        self.assertEquals(x1.extract(), [u'opt1'])
+        self.assertEqual(x1.extract(), [u'opt1'])
 
         x1 = r.xpath("//div/descendant::text()/preceding-sibling::b[contains(text(), 'Options')]")
-        self.assertEquals(x1.extract(), [u'<b>Options:</b>'])
+        self.assertEqual(x1.extract(), [u'<b>Options:</b>'])
 
     @unittest.skip("Text nodes lost parent node reference in lxml")
     def test_nested_select_on_text_nodes(self):
         # FIXME: does not work with lxml backend [upstream]
         r = self.sscls(text=u'<div><b>Options:</b>opt1</div><div><b>Other</b>opt2</div>')
         x1 = r.xpath("//div/descendant::text()")
         x2 = x1.xpath("./preceding-sibling::b[contains(text(), 'Options')]")
-        self.assertEquals(x2.extract(), [u'<b>Options:</b>'])
+        self.assertEqual(x2.extract(), [u'<b>Options:</b>'])
 
     def test_weakref_slots(self):
         """Check that classes are using slots and are weak-referenceable"""
@@ -692,7 +692,7 @@ def test_xml_entity_expansion(self):
 
     def test_configure_base_url(self):
         sel = self.sscls(text=u'nothing', base_url='http://example.com')
-        self.assertEquals(u'http://example.com', sel.root.base)
+        self.assertEqual(u'http://example.com', sel.root.base)
 
 
     def test_extending_selector(self):
@@ -732,7 +732,7 @@ def test_regexp(self):
         self.assertEqual(
             [x.extract()
              for x in sel.xpath(
-                 '//a[re:test(@href, "\.html$")]/text()')],
+                 r'//a[re:test(@href, "\.html$")]/text()')],
             [u'first link', u'second link'])
         self.assertEqual(
             [x.extract()
@@ -753,20 +753,18 @@ def test_regexp(self):
         #u'<match></match>',
         #u'<match>/xml/index.xml?/xml/utils/rechecker.xml</match>']
         self.assertEqual(
-            sel.xpath('re:match(//a[re:test(@href, "\.xml$")]/@href,'
-                      '"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
+            sel.xpath(r're:match(//a[re:test(@href, "\.xml$")]/@href,'
+                      r'"(\w+):\/\/([^/:]+)(:\d*)?([^# ]*)")/text()').extract(),
             [u'http://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.xml',
              u'http',
              u'www.bayes.co.uk',
              u'',
              u'/xml/index.xml?/xml/utils/rechecker.xml'])
 
-
-
         # re:replace()
         self.assertEqual(
-            sel.xpath('re:replace(//a[re:test(@href, "\.xml$")]/@href,'
-                      '"(\w+)://(.+)(\.xml)", "","https://\\2.html")').extract(),
+            sel.xpath(r're:replace(//a[re:test(@href, "\.xml$")]/@href,'
+                      r'"(\w+)://(.+)(\.xml)", "","https://\2.html")').extract(),
             [u'https://www.bayes.co.uk/xml/index.xml?/xml/utils/rechecker.html'])
 
     def test_set(self):

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27, py34, py35, py36, pypy, pypy3
+envlist = py27, py34, py35, py36, py37, pypy, pypy3
 
 [testenv]
 deps =