Merge pull request #38 from r1chardj0n3s/nightly-ci

Run tests nightly with the latest pip version

Author: adamtheturtle

.github/workflows/ci.yml

@@ -7,6 +7,10 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    # Run at 1:00 every day
+    - cron:  '0 1 * * *'
 
 jobs:
   build:
@@ -16,8 +20,9 @@ jobs:
     strategy:
       matrix:
         # These versions match the minimum and maximum versions of pip in
-        # requirements.txt
-        pip-version: ['==10.0.1', '<= 19']
+        # requirements.txt.
+        # An empty string here represents the latest version.
+        pip-version: ['==10.0.1', '']
         python-version: [3.6, 3.7, 3.8]
 
     steps:

CHANGELOG.rst

@@ -7,7 +7,8 @@ Release History
 - Remove support for Python 2.
   Please use an older version of this tool if you require that support.
 - Remove requirement for setuptools.
-- Add restriction on the pip version to install to match the pip versions which work.
+- Support newer versions of pip, including the current version, for more features (20.1.1).
+  Thanks to @Czaki for important parts of this change.
 
 2.0.1
 

pip_check_reqs/common.py

@@ -6,7 +6,11 @@
 import re
 
 from packaging.utils import canonicalize_name
-from pip._internal.download import PipSession
+# Between different versions of pip the location of PipSession has changed.
+try:
+    from pip._internal.network.session import PipSession
+except ImportError:  # pragma: no cover
+    from pip._internal.download import PipSession
 from pip._internal.req.req_file import parse_requirements
 
 log = logging.getLogger(__name__)
@@ -121,15 +125,26 @@ def find_imported_modules(options):
     return vis.finalise()
 
 
-def find_required_modules(options):
+def find_required_modules(options, requirements_filename: str):
     explicit = set()
-    for requirement in parse_requirements('requirements.txt',
+    for requirement in parse_requirements(requirements_filename,
                                           session=PipSession()):
+        try:
+            requirement_name = requirement.name
+        # The type of "requirement" changed between pip versions.
+        # We exclude the "except" from coverage so that on any pip version we
+        # can report 100% coverage.
+        except AttributeError:  # pragma: no cover
+            from pip._internal.req.constructors import install_req_from_line
+            requirement_name = install_req_from_line(
+                requirement.requirement,
+            ).name
+
         if options.ignore_reqs(requirement):
-            log.debug('ignoring requirement: %s', requirement.name)
+            log.debug('ignoring requirement: %s', requirement_name)
         else:
-            log.debug('found requirement: %s', requirement.name)
-            explicit.add(canonicalize_name(requirement.name))
+            log.debug('found requirement: %s', requirement_name)
+            explicit.add(canonicalize_name(requirement_name))
     return explicit
 
 
@@ -149,9 +164,22 @@ def ignorer(ignore_cfg):
 
     def f(candidate, ignore_cfg=ignore_cfg):
         for ignore in ignore_cfg:
-            if fnmatch.fnmatch(candidate, ignore):
+            try:
+                from pip._internal.req.constructors import (
+                    install_req_from_line,
+                )
+                candidate_path = install_req_from_line(  # pragma: no cover
+                    candidate.requirement,
+                ).name
+            except (ImportError, AttributeError):
+                try:
+                    candidate_path = candidate.name
+                except AttributeError:
+                    candidate_path = candidate
+
+            if fnmatch.fnmatch(candidate_path, ignore):
                 return True
-            elif fnmatch.fnmatch(os.path.relpath(candidate), ignore):
+            elif fnmatch.fnmatch(os.path.relpath(candidate_path), ignore):
                 return True
         return False
 

pip_check_reqs/find_extra_reqs.py

@@ -12,7 +12,7 @@
 log = logging.getLogger(__name__)
 
 
-def find_extra_reqs(options):
+def find_extra_reqs(options, requirements_filename):
     # 1. find files used by imports in the code (as best we can without
     #    executing)
     used_modules = common.find_imported_modules(options)
@@ -50,7 +50,10 @@ def find_extra_reqs(options):
                 modname, info.filename)
 
     # 4. compare with requirements.txt
-    explicit = common.find_required_modules(options)
+    explicit = common.find_required_modules(
+        options=options,
+        requirements_filename=requirements_filename,
+    )
 
     return [name for name in explicit if name not in used]
 
@@ -121,12 +124,20 @@ def main():
 
     log.info('using pip_check_reqs-%s from %s', __version__, __file__)
 
-    extras = find_extra_reqs(options)
+    requirements_filename = 'requirements.txt'
+    extras = find_extra_reqs(
+        options=options,
+        requirements_filename=requirements_filename,
+    )
 
     if extras:
         log.warning('Extra requirements:')
     for name in extras:
-        log.warning('%s in requirements.txt' % name)
+        message = '{name} in {requirements_filename}'.format(
+            name=name,
+            requirements_filename=requirements_filename,
+        )
+        log.warning(message)
 
     if extras:
         sys.exit(1)

pip_check_reqs/find_missing_reqs.py

@@ -6,7 +6,11 @@
 
 from packaging.utils import canonicalize_name
 from pip._internal.commands.show import search_packages_info
-from pip._internal.download import PipSession
+# Between different versions of pip the location of PipSession has changed.
+try:
+    from pip._internal.network.session import PipSession
+except ImportError:  # pragma: no cover
+    from pip._internal.download import PipSession
 from pip._internal.req.req_file import parse_requirements
 from pip._internal.utils.misc import get_installed_distributions
 
@@ -15,7 +19,7 @@
 log = logging.getLogger(__name__)
 
 
-def find_missing_reqs(options):
+def find_missing_reqs(options, requirements_filename):
     # 1. find files used by imports in the code (as best we can without
     #    executing)
     used_modules = common.find_imported_modules(options)
@@ -54,10 +58,23 @@ def find_missing_reqs(options):
 
     # 4. compare with requirements.txt
     explicit = set()
-    for requirement in parse_requirements('requirements.txt',
-                                          session=PipSession()):
-        log.debug('found requirement: %s', requirement.name)
-        explicit.add(canonicalize_name(requirement.name))
+    for requirement in parse_requirements(
+        requirements_filename,
+        session=PipSession(),
+    ):
+        try:
+            requirement_name = requirement.name
+        # The type of "requirement" changed between pip versions.
+        # We exclude the "except" from coverage so that on any pip version we
+        # can report 100% coverage.
+        except AttributeError:  # pragma: no cover
+            from pip._internal.req.constructors import install_req_from_line
+            requirement_name = install_req_from_line(
+                requirement.requirement,
+            ).name
+
+        log.debug('found requirement: %s', requirement_name)
+        explicit.add(canonicalize_name(requirement_name))
 
     return [(name, used[name]) for name in used if name not in explicit]
 
@@ -121,7 +138,11 @@ def main():
 
     log.info('using pip_check_reqs-%s from %s', __version__, __file__)
 
-    missing = find_missing_reqs(options)
+    requirements_filename = 'requirements.txt'
+    missing = find_missing_reqs(
+        options=options,
+        requirements_filename=requirements_filename,
+    )
 
     if missing:
         log.warning('Missing requirements:')

requirements.txt

@@ -1,3 +1,3 @@
 packaging
-# These versions are matched in the GitHub workflows CI.
-pip >= 10.0.1, <= 19
+# Pinned pip versions are matched in the GitHub workflows CI matrix.
+pip >= 10.0.1

tests/test_common.py

@@ -1,9 +1,9 @@
 from __future__ import absolute_import
 
 import ast
-import collections
 import logging
 import os.path
+from pathlib import Path
 
 import pytest
 import pretend
@@ -157,25 +157,23 @@ def ignore_mods(module):
     (['spam*'], 'eggs', False),
     (['spam'], '/spam', True),
 ])
-def test_ignorer(monkeypatch, ignore_cfg, candidate, result):
+def test_ignorer(monkeypatch, tmp_path: Path, ignore_cfg, candidate, result):
     monkeypatch.setattr(os.path, 'relpath', lambda s: s.lstrip('/'))
     ignorer = common.ignorer(ignore_cfg)
     assert ignorer(candidate) == result
 
 
-def test_find_required_modules(monkeypatch):
+def test_find_required_modules(monkeypatch, tmp_path: Path):
     class options:
-        @staticmethod
-        def ignore_reqs(req):
-            if req.name == 'barfoo':
-                return True
-            return False
+        pass
+
+    options.ignore_reqs = common.ignorer(ignore_cfg=['barfoo'])
 
-    FakeReq = collections.namedtuple('FakeReq', ['name'])
-    requirements = [FakeReq('foobar'), FakeReq('barfoo')]
-    monkeypatch.setattr(
-        common, 'parse_requirements',
-        pretend.call_recorder(lambda a, session=None: requirements))
+    fake_requirements_file = tmp_path / 'requirements.txt'
+    fake_requirements_file.write_text('foobar==1\nbarfoo==2')
 
-    reqs = common.find_required_modules(options)
+    reqs = common.find_required_modules(
+        options=options,
+        requirements_filename=str(fake_requirements_file),
+    )
     assert reqs == set(['foobar'])

tests/test_find_extra_reqs.py

@@ -3,6 +3,7 @@
 import collections
 import logging
 import optparse
+from pathlib import Path
 
 import pytest
 import pretend
@@ -37,7 +38,7 @@ def parse_args(self):
     return FakeOptParse
 
 
-def test_find_extra_reqs(monkeypatch):
+def test_find_extra_reqs(monkeypatch, tmp_path: Path):
     imported_modules = dict(spam=common.FoundModule('spam',
                                                     'site-spam/spam.py',
                                                     [('ham.py', 1)]),
@@ -64,19 +65,19 @@ def test_find_extra_reqs(monkeypatch):
     monkeypatch.setattr(find_extra_reqs, 'search_packages_info',
                         pretend.call_recorder(lambda x: packages_info))
 
-    FakeReq = collections.namedtuple('FakeReq', ['name'])
-    requirements = [FakeReq('foobar')]
-    monkeypatch.setattr(
-        common, 'parse_requirements',
-        pretend.call_recorder(lambda a, session=None: requirements))
+    fake_requirements_file = tmp_path / 'requirements.txt'
+    fake_requirements_file.write_text('foobar==1')
 
     class options:
         def ignore_reqs(x, y):
             return False
 
     options = options()
 
-    result = find_extra_reqs.find_extra_reqs(options)
+    result = find_extra_reqs.find_extra_reqs(
+        options=options,
+        requirements_filename=str(fake_requirements_file),
+    )
     assert result == ['foobar']
 
 
@@ -86,7 +87,7 @@ def test_main_failure(monkeypatch, caplog, fake_opts):
     caplog.set_level(logging.WARN)
 
     monkeypatch.setattr(find_extra_reqs, 'find_extra_reqs',
-                        lambda x: ['extra'])
+                        lambda options, requirements_filename: ['extra'])
 
     with pytest.raises(SystemExit) as excinfo:
         find_extra_reqs.main()
@@ -145,7 +146,11 @@ def parse_args(self):
 
     monkeypatch.setattr(optparse, 'OptionParser', FakeOptParse)
 
-    monkeypatch.setattr(find_extra_reqs, 'find_extra_reqs', lambda x: [])
+    monkeypatch.setattr(
+        find_extra_reqs,
+        'find_extra_reqs',
+        lambda options, requirements_filename: [],
+    )
     find_extra_reqs.main()
 
     for event in [(logging.DEBUG, 'debug'), (logging.INFO, 'info'),