Support Scrapy 2.6.2 (#103)

Author: Gallaecio

.github/workflows/main.yml

@@ -8,7 +8,7 @@ jobs:
         include:
         - python-version: 2.7
           env:
-            TOXENV: py27,stack-scrapy-1.0,stack-scrapy-1.1,stack-scrapy-1.2,stack-scrapy-1.3,stack-scrapy-1.4,stack-scrapy-1.5
+            TOXENV: py27,stack-scrapy-1.4,stack-scrapy-1.5
         - python-version: 3.5
           env:
             TOXENV: py35,stack-scrapy-1.8-py3,stack-scrapy-2.0-py3,stack-scrapy-2.1-py3,stack-scrapy-2.2-py3,stack-scrapy-2.3-py3

README.rst

@@ -21,7 +21,7 @@ Requirements
 ============
 
 * Python 2.7 or Python 3.4+
-* Scrapy
+* Scrapy 1.4+
 
 Installation
 ============

docs/conf.py

@@ -71,13 +71,6 @@
     version = ''
     release = ''
 
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path

requirements.txt

@@ -1,3 +1,3 @@
-scrapy>=1.0
+scrapy>=1.4
 six
 w3lib

scrapy_zyte_smartproxy/middleware.py

@@ -1,9 +1,10 @@
 import os
 import logging
 import warnings
+from base64 import urlsafe_b64decode
 from collections import defaultdict
 
-from six.moves.urllib.parse import urlparse
+from six.moves.urllib.parse import urlparse, urlunparse
 from w3lib.http import basic_auth_header
 from scrapy import signals
 from scrapy.resolver import dnscache
@@ -52,13 +53,28 @@ def __init__(self, crawler):
         self.spider = None
         self._bans = defaultdict(int)
         self._saved_delays = defaultdict(lambda: None)
+        self._auth_url = None
 
     @classmethod
     def from_crawler(cls, crawler):
         o = cls(crawler)
         crawler.signals.connect(o.open_spider, signals.spider_opened)
         return o
 
+    def _make_auth_url(self, spider):
+        parsed_url = urlparse(self.url)
+        auth = self.get_proxyauth(spider)
+        if not auth.startswith(b'Basic '):
+            raise ValueError(
+                'Zyte Smart Proxy Manager only supports HTTP basic access '
+                'authentication, but %s.%s.get_proxyauth() returned %r'
+                % (self.__module__, self.__class__.__name__, auth)
+            )
+        user_and_colon = urlsafe_b64decode(auth[6:].strip()).decode('utf-8')
+        netloc = user_and_colon + '@' + parsed_url.netloc.split('@')[-1]
+        parsed_url = parsed_url._replace(netloc=netloc)
+        return urlunparse(parsed_url)
+
     def open_spider(self, spider):
         self.enabled = self.is_enabled(spider)
         self.spider = spider
@@ -80,7 +96,7 @@ def open_spider(self, spider):
             )
             return
 
-        self._proxyauth = self.get_proxyauth(spider)
+        self._auth_url = self._make_auth_url(spider)
 
         logger.info(
             "Using Zyte Smart Proxy Manager at %s (apikey: %s)" % (
@@ -162,9 +178,8 @@ def process_request(self, request, spider):
         from scrapy_zyte_smartproxy import __version__
         if self._is_enabled_for_request(request):
             self._set_zyte_smartproxy_default_headers(request)
-            request.meta['proxy'] = self.url
+            request.meta['proxy'] = self._auth_url
             request.meta['download_timeout'] = self.download_timeout
-            request.headers['Proxy-Authorization'] = self._proxyauth
             if self.job_id:
                 request.headers['X-Crawlera-Jobid'] = self.job_id
             request.headers['X-Crawlera-Client'] = 'scrapy-zyte-smartproxy/%s' % __version__

setup.py

@@ -35,5 +35,5 @@
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    install_requires=['scrapy>=1.0.0', 'six', 'w3lib'],
+    install_requires=['scrapy>=1.4.0', 'six', 'w3lib'],
 )

tests/test_all.py

@@ -1,3 +1,4 @@
+import binascii
 import os
 import pytest
 from random import choice
@@ -8,6 +9,7 @@
     from mock import call, patch
 
 from w3lib.http import basic_auth_header
+from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
@@ -84,17 +86,19 @@ def _assert_disabled(self, spider, settings=None):
     def _assert_enabled(self, spider,
                         settings=None,
                         proxyurl='http://proxy.zyte.com:8011',
+                        proxyurlcreds='http://apikey:@proxy.zyte.com:8011',
                         proxyauth=basic_auth_header('apikey', ''),
                         maxbans=400,
                         download_timeout=190):
         crawler = self._mock_crawler(spider, settings)
         mw = self.mwcls.from_crawler(crawler)
         mw.open_spider(spider)
+        assert mw.url == proxyurl
         req = Request('http://www.scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta.get('proxy'), proxyurl)
+        self.assertEqual(req.meta.get('proxy'), proxyurlcreds)
         self.assertEqual(req.meta.get('download_timeout'), download_timeout)
-        self.assertEqual(req.headers.get('Proxy-Authorization'), proxyauth)
+        self.assertNotIn(b'Proxy-Authorization', req.headers)
         res = self._mock_zyte_smartproxy_response(req.url)
         assert mw.process_response(req, res, spider) is res
 
@@ -169,31 +173,31 @@ def test_apikey(self):
         self.spider.zyte_smartproxy_enabled = True
         self.settings['ZYTE_SMARTPROXY_APIKEY'] = apikey = 'apikey'
         proxyauth = basic_auth_header(apikey, '')
-        self._assert_enabled(self.spider, self.settings, proxyauth=proxyauth)
+        self._assert_enabled(self.spider, self.settings, proxyauth=proxyauth, proxyurlcreds='http://apikey:@proxy.zyte.com:8011')
 
         self.spider.zyte_smartproxy_apikey = apikey = 'notfromsettings'
         proxyauth = basic_auth_header(apikey, '')
-        self._assert_enabled(self.spider, self.settings, proxyauth=proxyauth)
+        self._assert_enabled(self.spider, self.settings, proxyauth=proxyauth, proxyurlcreds='http://notfromsettings:@proxy.zyte.com:8011')
 
     def test_proxyurl(self):
         self.spider.zyte_smartproxy_enabled = True
         self.settings['ZYTE_SMARTPROXY_URL'] = 'http://localhost:8011'
-        self._assert_enabled(self.spider, self.settings, proxyurl='http://localhost:8011')
+        self._assert_enabled(self.spider, self.settings, proxyurl='http://localhost:8011', proxyurlcreds='http://apikey:@localhost:8011')
 
     def test_proxyurl_no_protocol(self):
         self.spider.zyte_smartproxy_enabled = True
         self.settings['ZYTE_SMARTPROXY_URL'] = 'localhost:8011'
-        self._assert_enabled(self.spider, self.settings, proxyurl='http://localhost:8011')
+        self._assert_enabled(self.spider, self.settings, proxyurl='http://localhost:8011', proxyurlcreds='http://apikey:@localhost:8011')
 
     def test_proxyurl_https(self):
         self.spider.zyte_smartproxy_enabled = True
         self.settings['ZYTE_SMARTPROXY_URL'] = 'https://localhost:8011'
-        self._assert_enabled(self.spider, self.settings, proxyurl='https://localhost:8011')
+        self._assert_enabled(self.spider, self.settings, proxyurl='https://localhost:8011', proxyurlcreds='https://apikey:@localhost:8011')
 
     def test_proxyurl_including_noconnect(self):
         self.spider.zyte_smartproxy_enabled = True
         self.settings['ZYTE_SMARTPROXY_URL'] = 'http://localhost:8011?noconnect'
-        self._assert_enabled(self.spider, self.settings, proxyurl='http://localhost:8011?noconnect')
+        self._assert_enabled(self.spider, self.settings, proxyurl='http://localhost:8011?noconnect', proxyurlcreds='http://apikey:@localhost:8011?noconnect')
 
     def test_maxbans(self):
         self.spider.zyte_smartproxy_enabled = True
@@ -218,7 +222,7 @@ def test_download_timeout(self):
         self._assert_enabled(self.spider, self.settings, download_timeout=120)
 
     def test_hooks(self):
-        proxyauth = b'Basic Foo'
+        proxyauth = basic_auth_header('foo', '')
 
         class _ECLS(self.mwcls):
             def is_enabled(self, spider):
@@ -241,7 +245,7 @@ def get_proxyauth(self, spider):
         wascalled[:] = []  # reset
         enabled = True
         self.spider.zyte_smartproxy_enabled = False
-        self._assert_enabled(self.spider, self.settings, proxyauth=proxyauth)
+        self._assert_enabled(self.spider, self.settings, proxyauth=proxyauth, proxyurlcreds='http://foo:@proxy.zyte.com:8011')
         self.assertEqual(wascalled, ['is_enabled', 'get_proxyauth'])
 
     def test_delay_adjustment(self):
@@ -909,3 +913,72 @@ def test_client_header(self):
             req.headers.get('X-Crawlera-Client').decode('utf-8'),
             'scrapy-zyte-smartproxy/%s' % __version__
         )
+
+    def test_scrapy_httpproxy_integration(self):
+        self.spider.zyte_smartproxy_enabled = True
+        crawler = self._mock_crawler(self.spider, self.settings)
+        smartproxy = self.mwcls.from_crawler(crawler)
+        smartproxy.open_spider(self.spider)
+        httpproxy = HttpProxyMiddleware.from_crawler(crawler)
+        request = Request('https://example.com')
+        auth_header = basic_auth_header('apikey', '')
+
+        # 1st pass
+        self.assertEqual(smartproxy.process_request(request, self.spider), None)
+        self.assertEqual(httpproxy.process_request(request, self.spider), None)
+        self.assertEqual(request.meta['proxy'], 'http://proxy.zyte.com:8011')
+        self.assertEqual(request.headers[b'Proxy-Authorization'], auth_header)
+
+        # 2nd pass (e.g. retry or redirect)
+        self.assertEqual(smartproxy.process_request(request, self.spider), None)
+        self.assertEqual(httpproxy.process_request(request, self.spider), None)
+        self.assertEqual(request.meta['proxy'], 'http://proxy.zyte.com:8011')
+        self.assertEqual(request.headers[b'Proxy-Authorization'], auth_header)
+
+    def test_subclass_non_basic_header(self):
+
+        class Subclass(self.mwcls):
+            def get_proxyauth(self, spider):
+                return b'Non-Basic foo'
+
+        self.spider.zyte_smartproxy_enabled = True
+        crawler = self._mock_crawler(self.spider, self.settings)
+        smartproxy = Subclass.from_crawler(crawler)
+        with pytest.raises(ValueError):
+            smartproxy.open_spider(self.spider)
+
+    def test_subclass_basic_header_non_base64(self):
+
+        class Subclass(self.mwcls):
+            def get_proxyauth(self, spider):
+                return b'Basic foo'
+
+        self.spider.zyte_smartproxy_enabled = True
+        crawler = self._mock_crawler(self.spider, self.settings)
+        smartproxy = Subclass.from_crawler(crawler)
+        with pytest.raises((TypeError, binascii.Error)):
+            smartproxy.open_spider(self.spider)
+
+    def test_subclass_basic_header_nonurlsafe_base64(self):
+
+        class Subclass(self.mwcls):
+            def get_proxyauth(self, spider):
+                return b'Basic YWF+Og=='
+
+        self.spider.zyte_smartproxy_enabled = True
+        crawler = self._mock_crawler(self.spider, self.settings)
+        smartproxy = Subclass.from_crawler(crawler)
+        smartproxy.open_spider(self.spider)
+        self.assertEqual(smartproxy._auth_url, "http://aa~:@proxy.zyte.com:8011")
+
+    def test_subclass_basic_header_urlsafe_base64(self):
+
+        class Subclass(self.mwcls):
+            def get_proxyauth(self, spider):
+                return b'Basic YWF-Og=='
+
+        self.spider.zyte_smartproxy_enabled = True
+        crawler = self._mock_crawler(self.spider, self.settings)
+        smartproxy = Subclass.from_crawler(crawler)
+        smartproxy.open_spider(self.spider)
+        self.assertEqual(smartproxy._auth_url, "http://aa~:@proxy.zyte.com:8011")

tox.ini

@@ -15,75 +15,6 @@ deps =
 commands =
     bandit -r {posargs:scrapy_zyte_smartproxy setup.py}
 
-[testenv:stack-scrapy-1.0]
-basepython = python2.7
-deps =
-    Scrapy==1.0.6
-    six==1.10.0
-    Twisted==16.1.1
-    w3lib==1.14.2
-    -rtests/requirements.txt
-
-[testenv:stack-scrapy-1.1]
-basepython = python2.7
-deps =
-    Scrapy==1.1.3
-    Parsel==1.0.3
-    six==1.10.0
-    Twisted==16.1.1
-    w3lib==1.14.2
-    -rtests/requirements.txt
-
-[testenv:stack-scrapy-1.1-py3]
-basepython = python3.5
-deps =
-    Scrapy==1.1.3
-    Parsel==1.0.3
-    six==1.10.0
-    Twisted==16.3.0
-    w3lib==1.14.3
-    -rtests/requirements.txt
-
-[testenv:stack-scrapy-1.2]
-basepython = python2.7
-deps =
-    Scrapy==1.2.2
-    Parsel==1.1.0
-    six==1.10.0
-    Twisted==16.1.1
-    w3lib==1.15.0
-    -rtests/requirements.txt
-
-[testenv:stack-scrapy-1.2-py3]
-basepython = python3.5
-deps =
-    Scrapy==1.2.2
-    Parsel==1.1.0
-    six==1.10.0
-    Twisted==16.3.0
-    w3lib==1.15.0
-    -rtests/requirements.txt
-
-[testenv:stack-scrapy-1.3]
-basepython = python2.7
-deps =
-    Scrapy==1.3.3
-    Parsel==1.3.1
-    six==1.10.0
-    Twisted==16.6.0
-    w3lib==1.16.0
-    -rtests/requirements.txt
-
-[testenv:stack-scrapy-1.3-py3]
-basepython = python3.6
-deps =
-    Scrapy==1.3.3
-    Parsel==1.3.1
-    six==1.10.0
-    Twisted==16.6.0
-    w3lib==1.16.0
-    -rtests/requirements.txt
-
 [testenv:stack-scrapy-1.4]
 basepython = python2.7
 deps =