Merge pull request #89 from r1chardj0n3s/3.11-tests

Add support for Python 3.11

Author: adamtheturtle

.github/workflows/ci.yml

@@ -22,13 +22,13 @@ jobs:
         # These versions match the minimum and maximum versions of pip in
         # requirements.txt.
         # An empty string here represents the latest version.
-        pip-version: ['==10.0.1', '==21.2.4', '']
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        pip-version: ['==21.2.4', '']
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: "Set up Python"
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -48,7 +48,9 @@ jobs:
           # We use '--ignore-installed' to avoid GitHub's cache which can cause
           # issues - we have seen packages from this cache be cause trouble with
           # pip-extra-reqs.
-          python -m pip install --ignore-installed --upgrade --editable .[dev]
+          #
+          # We avoid "--upgrade" as we do version tests for the "pip" dependency.
+          python -m pip install --ignore-installed --editable .[dev]
           python -m pip install flake8
 
       - name: "Lint"
@@ -62,4 +64,4 @@ jobs:
           pytest -s -vvv --cov-fail-under 100 --cov=pip_check_reqs/ --cov=tests tests/ --cov-report=xml
 
       - name: "Upload coverage to Codecov"
-        uses: "codecov/codecov-action@v1"
+        uses: "codecov/codecov-action@v3"

pip_check_reqs/common.py

@@ -7,7 +7,6 @@
 import sys
 
 from pathlib import Path
-from typing import Container, Optional, List, cast
 
 from packaging.utils import canonicalize_name
 from packaging.markers import Marker
@@ -20,47 +19,6 @@
 except ImportError:  # pragma: no cover
     from pip._internal.download import PipSession
 from pip._internal.req.req_file import parse_requirements
-try:
-    from pip._internal.utils.misc import get_installed_distributions
-except ImportError:  # pip>=21.3
-    from pip._internal.utils.compat import stdlib_pkgs
-    from pip._internal.metadata import get_default_environment, get_environment
-    from pip._internal.metadata.pkg_resources import Distribution as _Dist
-    from pip._vendor.pkg_resources import Distribution
-
-    # get_installed_distributions was removed in pip 21.3.
-    # This is a copy from pip.
-    # See
-    # https://github.com/pypa/pip/commit/d051a00fc57037104fca85ad8ebf2cdbd1e32d24#diff-058e40cb3a9ea705f655937e48f3a053f5dc7c500b7f1b2aae76e9bd673faf64.
-    #
-    # This is mocked in all tests (unfortunately) and so we do not cover this
-    # function.
-    def get_installed_distributions(
-         local_only: bool = True,
-         skip: Container[str] = stdlib_pkgs,
-         include_editables: bool = True,
-         editables_only: bool = False,
-         user_only: bool = False,
-         paths: Optional[List[str]] = None,
-     ) -> List[Distribution]:  # pragma: no cover
-        """Return a list of installed Distribution objects.
-
-        Left for compatibility until direct pkg_resources uses are refactored
-        out.
-        """
-        if paths is None:
-            env = get_default_environment()
-        else:
-            env = get_environment(paths)
-
-        dists = env.iter_installed_distributions(
-            local_only=local_only,
-            skip=skip,
-            include_editables=include_editables,
-            editables_only=editables_only,
-            user_only=user_only,
-        )
-        return [cast(_Dist, dist)._dist for dist in dists]
 
 
 log = logging.getLogger(__name__)
@@ -120,8 +78,8 @@ def __addModule(self, modname, lineno):
 
             # we might have previously seen a useful path though...
             if modpath is None:  # pragma: no cover
-                # the sys module will hit this code path on py3k - possibly
-                # others will, but I've not discovered them
+                # the sys module will hit this code path, and os will on 3.11+.
+                # possibly others will, but I've not discovered them.
                 modpath = last_modpath
                 break
 

pip_check_reqs/find_extra_reqs.py

@@ -1,4 +1,5 @@
 import collections
+import importlib.metadata
 import logging
 import pathlib
 import optparse
@@ -8,7 +9,7 @@
 from packaging.utils import canonicalize_name
 from pip._internal.commands.show import search_packages_info
 from pip_check_reqs import common
-from pip_check_reqs.common import get_installed_distributions, version_info
+from pip_check_reqs.common import version_info
 
 log = logging.getLogger(__name__)
 
@@ -20,7 +21,11 @@ def find_extra_reqs(options, requirements_filename):
 
     # 2. find which packages provide which files
     installed_files = {}
-    all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
+    all_pkgs = (
+        dist.metadata["Name"] for dist
+        in importlib.metadata.distributions()
+    )
+
     for package in search_packages_info(all_pkgs):
         if isinstance(package, dict):  # pragma: no cover
             package_name = package['name']

pip_check_reqs/find_missing_reqs.py

@@ -1,4 +1,5 @@
 import collections
+import importlib.metadata
 import logging
 import optparse
 import os
@@ -15,7 +16,7 @@
 from pip._internal.req.req_file import parse_requirements
 
 from pip_check_reqs import common
-from pip_check_reqs.common import get_installed_distributions, version_info
+from pip_check_reqs.common import version_info
 
 log = logging.getLogger(__name__)
 
@@ -27,7 +28,11 @@ def find_missing_reqs(options, requirements_filename):
 
     # 2. find which packages provide which files
     installed_files = {}
-    all_pkgs = (pkg.project_name for pkg in get_installed_distributions())
+    all_pkgs = (
+        dist.metadata["Name"] for dist
+        in importlib.metadata.distributions()
+    )
+
     for package in search_packages_info(all_pkgs):
         if isinstance(package, dict):  # pragma: no cover
             package_name = package['name']

requirements.txt

@@ -1,3 +1,3 @@
 packaging >= 16.0
 # Pinned pip versions are matched in the GitHub workflows CI matrix.
-pip >= 10.0.1
+pip >= 21.2.4

tests/test_common.py

@@ -89,9 +89,20 @@ def test_pyfiles_package(monkeypatch):
         ['spam/__init__.py', 'spam/ham.py', 'spam/dub/bass.py']
 
 
+# Beware - using "sys" or "os" here can have weird results.
+# See the comment in the implementation.
+# We don't mind so much as we only really use this for third party packages.
 @pytest.mark.parametrize(["ignore_ham", "ignore_hashlib", "expect", "locs"], [
-    (False, False, ['ast', 'os', 'hashlib'], [('spam.py', 2), ('ham.py', 2)]),
-    (False, True, ['ast', 'os'], [('spam.py', 2), ('ham.py', 2)]),
+    (
+        False,
+        False,
+        ['ast', 'pathlib', 'hashlib'],
+        [
+            ('spam.py', 2),
+            ('ham.py', 2),
+        ],
+    ),
+    (False, True, ['ast', 'pathlib'], [('spam.py', 2), ('ham.py', 2)]),
     (True, False, ['ast'], [('spam.py', 2)]),
     (True, True, ['ast'], [('spam.py', 2)]),
 ])
@@ -110,7 +121,7 @@ def test_find_imported_modules(caplog, ignore_ham, ignore_hashlib,
     )
     ham_file_contents = textwrap.dedent(
         """\
-        from os import path
+        from pathlib import Path
         import ast, hashlib
         """,
     )

tests/test_find_extra_reqs.py

@@ -1,6 +1,8 @@
 from __future__ import absolute_import
+from dataclasses import dataclass
+import importlib
+from typing import Dict, Optional
 
-import collections
 import logging
 import optparse
 from pathlib import Path
@@ -51,10 +53,20 @@ def test_find_extra_reqs(monkeypatch, tmp_path: Path):
     monkeypatch.setattr(common, 'find_imported_modules',
                         pretend.call_recorder(lambda a: imported_modules))
 
-    FakeDist = collections.namedtuple('FakeDist', ['project_name'])
-    installed_distributions = map(FakeDist, ['spam', 'pass'])
-    monkeypatch.setattr(find_extra_reqs, 'get_installed_distributions',
-                        pretend.call_recorder(lambda: installed_distributions))
+    @dataclass
+    class FakePathDistribution:
+        metadata: Dict
+        name: Optional[str] = None
+
+    installed_distributions = map(
+        FakePathDistribution,
+        [{'Name': 'spam'}, {'Name': 'pass'}],
+    )
+    monkeypatch.setattr(
+        importlib.metadata,
+        'distributions',
+        pretend.call_recorder(lambda **kwargs: installed_distributions),
+    )
     packages_info = [
         dict(name='spam',
              location='site-spam',

tests/test_find_missing_reqs.py

@@ -1,6 +1,8 @@
 from __future__ import absolute_import
+from dataclasses import dataclass
+import importlib
+from typing import Dict, Optional
 
-import collections
 import logging
 import optparse
 from pathlib import Path
@@ -50,10 +52,20 @@ def test_find_missing_reqs(monkeypatch, tmp_path: Path):
     monkeypatch.setattr(common, 'find_imported_modules',
                         pretend.call_recorder(lambda a: imported_modules))
 
-    FakeDist = collections.namedtuple('FakeDist', ['project_name'])
-    installed_distributions = map(FakeDist, ['spam', 'pass'])
-    monkeypatch.setattr(find_missing_reqs, 'get_installed_distributions',
-                        pretend.call_recorder(lambda: installed_distributions))
+    @dataclass
+    class FakePathDistribution:
+        metadata: Dict
+        name: Optional[str] = None
+
+    installed_distributions = map(
+        FakePathDistribution,
+        [{'Name': 'spam'}, {'Name': 'pass'}],
+    )
+    monkeypatch.setattr(
+        importlib.metadata,
+        'distributions',
+        pretend.call_recorder(lambda **kwargs: installed_distributions),
+    )
     packages_info = [
         dict(name='spam',
              location='site-spam',