Merge branch 'master' into flake8

Author: Gallaecio

.bandit.yml

@@ -0,0 +1,3 @@
+skips:
+- B101
+- B107

.travis.yml

@@ -2,20 +2,22 @@ language: python
 sudo: false
 matrix:
   include:
+    - python: 3.8
+      env: TOXENV=security
     - python: 3.8
       env: TOXENV=flake8
     - python: 2.7
       env: TOXENV=py27
     - python: pypy
       env: TOXENV=pypy
-    - python: 3.4
-      env: TOXENV=py34
     - python: 3.5
       env: TOXENV=py35
     - python: 3.6
       env: TOXENV=py36
     - python: 3.7
       env: TOXENV=py37
+    - python: 3.8
+      env: TOXENV=py38
     - python: pypy3
       env: TOXENV=pypy3
     - python: 3.7
@@ -49,7 +51,7 @@ deploy:
     tags: true
     all_branches: true
     repo: scrapy/w3lib
-    condition: "$TOXENV == py27"
+    condition: "$TOXENV == py37"
 
 cache:
   directories:

README.rst

@@ -27,7 +27,7 @@ This is a Python library of web-related functions, such as:
 Requirements
 ============
 
-Python 2.7 or Python 3.4+
+Python 2.7 or Python 3.5+
 
 Install
 =======

docs/conf.py

@@ -26,6 +26,8 @@
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
 extensions = [
+    'hoverxref.extension',
+    'notfound.extension',
     'sphinx.ext.autodoc',
     'sphinx.ext.doctest',
     'sphinx.ext.intersphinx',
@@ -248,9 +250,19 @@
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'http://docs.python.org/': None}
+intersphinx_mapping = {
+    'pytest': ('https://docs.pytest.org/en/latest', None),
+    'python': ('https://docs.python.org/3', None),
+    'scrapy': ('https://scrapy.readthedocs.io/en/latest', None),
+    'tox': ('https://tox.readthedocs.io/en/latest', None),
+}
 
 
-# --- Nitpicking options ------------------------------------------------------
+# -- Nitpicking options -------------------------------------------------------
 
 nitpicky = True
+
+
+# -- sphinx-hoverxref options -------------------------------------------------
+
+hoverxref_auto_ref = True

docs/index.rst

@@ -39,17 +39,15 @@ Install
 Tests
 =====
 
-`pytest`_ is the preferred way to run tests. Just run: ``pytest`` from the
-root directory to execute tests using the default Python interpreter.
+:doc:`pytest <pytest:index>` is the preferred way to run tests. Just run:
+``pytest`` from the root directory to execute tests using the default Python
+interpreter.
 
-`tox`_ could be used to run tests for all supported Python versions.
-Install it (using 'pip install tox') and then run ``tox`` from
+:doc:`tox <tox:index>` could be used to run tests for all supported Python
+versions. Install it (using 'pip install tox') and then run ``tox`` from
 the root directory - tests will be executed for all available
 Python interpreters.
 
-.. _tox: http://tox.testrun.org
-.. _pytest: https://docs.pytest.org/en/latest/
-
 
 Changelog
 =========
@@ -60,12 +58,10 @@ Changelog
 History
 -------
 
-The code of w3lib was originally part of the `Scrapy framework`_ but was later
-stripped out of Scrapy, with the aim of make it more reusable and to provide a
-useful library of web functions without depending on Scrapy.
-
-.. _Scrapy framework: http://scrapy.org
-.. _NEWS file: https://github.com/scrapy/w3lib/blob/master/NEWS
+The code of w3lib was originally part of the :doc:`Scrapy framework
+<scrapy:index>` but was later stripped out of Scrapy, with the aim of make it
+more reusable and to provide a useful library of web functions without
+depending on Scrapy.
 
 
 Indices and tables

docs/requirements.txt

@@ -0,0 +1,4 @@
+sphinx
+sphinx-hoverxref
+sphinx-notfound-page
+sphinx_rtd_theme

setup.py

@@ -21,10 +21,10 @@
         'Programming Language :: Python :: 2',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',

tests/test_html.py

@@ -211,7 +211,7 @@ def test_with_escape_chars(self):
         # text with escape chars
         self.assertEqual(replace_escape_chars(u'escape\n\n'), u'escape')
         self.assertEqual(replace_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n')
-        self.assertEqual(replace_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n')
+        self.assertEqual(replace_escape_chars(u'escape\tchars\n', which_ones=('\t',)), 'escapechars\n')
         self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by=' '), 'escape chars ')
         self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by=u'\xa3'), u'escape\xa3chars\xa3')
         self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by=b'\xc2\xa3'), u'escape\xa3chars\xa3')

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py27, pypy, py34, py35, py36, py37, pypy3, docs
+envlist = py27, pypy, py35, py36, py37, py38, pypy3, docs
 
 [testenv]
 deps =
@@ -16,6 +16,12 @@ commands =
         --cov=w3lib --cov-report=term \
         {posargs:w3lib tests}
 
+[testenv:security]
+deps =
+    bandit
+commands =
+    bandit -r -c .bandit.yml {posargs:w3lib}
+
 [testenv:flake8]
 basepython = python3
 deps =
@@ -26,9 +32,7 @@ commands =
 
 [docs]
 changedir = docs
-deps =
-    sphinx
-    sphinx_rtd_theme
+deps = -rdocs/requirements.txt
 
 [testenv:docs]
 changedir = {[docs]changedir}