Merge branch 'main' into update-non-PEP-440-wheel-filename-deprecation-notice

notatallshaw · web-flow · commit 7ddafd074356 · 2024-10-08T09:08:56.000-04:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,6 +10,8 @@ on:
   pull_request:
   schedule:
     - cron: 0 0 * * MON # Run every Monday at 00:00 UTC
+  workflow_dispatch:
+    # allow manual runs on branches without a PR
 
 env:
   # The "FORCE_COLOR" variable, when set to 1,
@@ -55,6 +57,9 @@ jobs:
               - "src/**"
               - "tests/**"
               - "noxfile.py"
+              # The test suite should also run when cutting a release
+              # (which is the only time this file is modified).
+              - "NEWS.rst"
         if: github.event_name == 'pull_request'
 
   packaging:
@@ -107,7 +112,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-latest, macos-12]
+        os: [ubuntu-latest, macos-13, macos-latest]
         python:
           - "3.8"
           - "3.9"
@@ -130,8 +135,13 @@ jobs:
           sudo apt-get install bzr
 
       - name: Install MacOS dependencies
-        if: matrix.os == 'macos-12'
-        run: brew install breezy
+        if: runner.os == 'macOS'
+        run: |
+          DEPS=breezy
+          if ! which svn; then
+            DEPS="${DEPS} subversion"
+          fi
+          brew install ${DEPS}
 
       - run: pip install nox
 
diff --git a/news/12653.feature.rst b/news/12653.feature.rst
@@ -0,0 +1,2 @@
+Detect recursively referencing requirements files and help users identify
+the source.
diff --git a/news/12964.trivial.rst b/news/12964.trivial.rst
@@ -0,0 +1 @@
+A valid, but non-existent URL used in a test case was corrected to be a valid URL.
diff --git a/news/A454782D-6619-44D8-811F-2C1973A430BB.trivial.rst b/news/A454782D-6619-44D8-811F-2C1973A430BB.trivial.rst
diff --git a/news/F2CF2568-49B1-49EF-8B16-70491B9C8FDA.trivial.rst b/news/F2CF2568-49B1-49EF-8B16-70491B9C8FDA.trivial.rst
diff --git a/news/urllib3.vendor.rst b/news/urllib3.vendor.rst
@@ -1 +1 @@
-Upgrade urllib3 to 1.26.19
+Upgrade urllib3 to 1.26.20
diff --git a/pyproject.toml b/pyproject.toml
@@ -181,7 +181,7 @@ select = [
     "PLR0",
     "W",
     "RUF100",
-    "UP032",
+    "UP",
 ]
 
 [tool.ruff.lint.isort]
diff --git a/src/pip/_internal/cli/index_command.py b/src/pip/_internal/cli/index_command.py
@@ -54,7 +54,7 @@ class SessionCommandMixin(CommandContextMixIn):
 
     def __init__(self) -> None:
         super().__init__()
-        self._session: Optional["PipSession"] = None
+        self._session: Optional[PipSession] = None
 
     @classmethod
     def _get_index_urls(cls, options: Values) -> Optional[List[str]]:
diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py
@@ -176,7 +176,7 @@ def run(self, options: Values, args: List[str]) -> int:
         if options.excludes:
             skip.update(canonicalize_name(n) for n in options.excludes)
 
-        packages: "_ProcessedDists" = [
+        packages: _ProcessedDists = [
             cast("_DistWithLatestInfo", d)
             for d in get_environment(options.path).iter_installed_distributions(
                 local_only=options.local,
diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py
@@ -89,7 +89,7 @@ def transform_hits(hits: List[Dict[str, str]]) -> List["TransformedHit"]:
     packages with the list of versions stored inline. This converts the
     list from pypi into one we can use.
     """
-    packages: Dict[str, "TransformedHit"] = OrderedDict()
+    packages: Dict[str, TransformedHit] = OrderedDict()
     for hit in hits:
         name = hit["name"]
         summary = hit["summary"]
diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py
@@ -429,7 +429,7 @@ class HashErrors(InstallationError):
     """Multiple HashError instances rolled into one for reporting"""
 
     def __init__(self) -> None:
-        self.errors: List["HashError"] = []
+        self.errors: List[HashError] = []
 
     def append(self, error: "HashError") -> None:
         self.errors.append(error)
diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py
@@ -80,7 +80,7 @@ def _set_requirement_extras(req: Requirement, new_extras: Set[str]) -> Requireme
     assert (
         pre is not None and post is not None
     ), f"regex group selection for requirement {req} failed, this should never happen"
-    extras: str = "[%s]" % ",".join(sorted(new_extras)) if new_extras else ""
+    extras: str = "[{}]".format(",".join(sorted(new_extras)) if new_extras else "")
     return get_requirement(f"{pre}{extras}{post}")
 
 
diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py
@@ -324,11 +324,15 @@ def __init__(
     ) -> None:
         self._session = session
         self._line_parser = line_parser
+        self._parsed_files: dict[str, Optional[str]] = {}
 
     def parse(
         self, filename: str, constraint: bool
     ) -> Generator[ParsedLine, None, None]:
         """Parse a given file, yielding parsed lines."""
+        self._parsed_files[os.path.abspath(filename)] = (
+            None  # The primary requirements file passed
+        )
         yield from self._parse_and_recurse(filename, constraint)
 
     def _parse_and_recurse(
@@ -353,11 +357,25 @@ def _parse_and_recurse(
                 # original file and nested file are paths
                 elif not SCHEME_RE.search(req_path):
                     # do a join so relative paths work
-                    req_path = os.path.join(
-                        os.path.dirname(filename),
-                        req_path,
+                    # and then abspath so that we can identify recursive references
+                    req_path = os.path.abspath(
+                        os.path.join(
+                            os.path.dirname(filename),
+                            req_path,
+                        )
                     )
-
+                if req_path in self._parsed_files:
+                    initial_file = self._parsed_files[req_path]
+                    tail = (
+                        f" and again in {initial_file}"
+                        if initial_file is not None
+                        else ""
+                    )
+                    raise RequirementsFileParseError(
+                        f"{req_path} recursively references itself in {filename}{tail}"
+                    )
+                # Keeping a track where was each file first included in
+                self._parsed_files[req_path] = filename
                 yield from self._parse_and_recurse(req_path, nested_constraint)
             else:
                 yield line
diff --git a/src/pip/_vendor/urllib3/_version.py b/src/pip/_vendor/urllib3/_version.py
@@ -1,2 +1,2 @@
 # This file is protected via CODEOWNERS
-__version__ = "1.26.19"
+__version__ = "1.26.20"
diff --git a/src/pip/_vendor/urllib3/connectionpool.py b/src/pip/_vendor/urllib3/connectionpool.py
@@ -423,12 +423,13 @@ def _make_request(
             pass
         except IOError as e:
             # Python 2 and macOS/Linux
-            # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS
+            # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE/ECONNRESET are needed on macOS
             # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
             if e.errno not in {
                 errno.EPIPE,
                 errno.ESHUTDOWN,
                 errno.EPROTOTYPE,
+                errno.ECONNRESET,
             }:
                 raise
 
diff --git a/src/pip/_vendor/urllib3/util/ssl_.py b/src/pip/_vendor/urllib3/util/ssl_.py
@@ -1,11 +1,11 @@
 from __future__ import absolute_import
 
+import hashlib
 import hmac
 import os
 import sys
 import warnings
 from binascii import hexlify, unhexlify
-from hashlib import md5, sha1, sha256
 
 from ..exceptions import (
     InsecurePlatformWarning,
@@ -24,7 +24,10 @@
 ALPN_PROTOCOLS = ["http/1.1"]
 
 # Maps the length of a digest to a possible hash function producing this digest
-HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256}
+HASHFUNC_MAP = {
+    length: getattr(hashlib, algorithm, None)
+    for length, algorithm in ((32, "md5"), (40, "sha1"), (64, "sha256"))
+}
 
 
 def _const_compare_digest_backport(a, b):
@@ -191,9 +194,15 @@ def assert_fingerprint(cert, fingerprint):
 
     fingerprint = fingerprint.replace(":", "").lower()
     digest_length = len(fingerprint)
-    hashfunc = HASHFUNC_MAP.get(digest_length)
-    if not hashfunc:
+    if digest_length not in HASHFUNC_MAP:
         raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint))
+    hashfunc = HASHFUNC_MAP.get(digest_length)
+    if hashfunc is None:
+        raise SSLError(
+            "Hash function implementation unavailable for fingerprint length: {0}".format(
+                digest_length
+            )
+        )
 
     # We need encode() here for py32; works on py2 and p33.
     fingerprint_bytes = unhexlify(fingerprint.encode())
diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt
@@ -8,7 +8,7 @@ pyproject-hooks==1.0.0
 requests==2.32.3
     certifi==2024.7.4
     idna==3.7
-    urllib3==1.26.19
+    urllib3==1.26.20
 rich==13.7.1
     pygments==2.18.0
     typing_extensions==4.12.2
diff --git a/tests/functional/test_bad_url.py b/tests/functional/test_bad_url.py
@@ -7,8 +7,8 @@
 def test_filenotfound_error_message(script: Any) -> None:
     # Test the error message returned when using a bad 'file:' URL.
     # make pip to fail and get an error message
-    # by running "pip install -r file:nonexistent_file"
-    proc = script.pip("install", "-r", "file:unexistent_file", expect_error=True)
+    # by running "pip install -r file:///nonexistent_file"
+    proc = script.pip("install", "-r", "file:///unexistent_file", expect_error=True)
     assert proc.returncode == 1
     expect = (
         "ERROR: 404 Client Error: FileNotFoundError for url: file:///unexistent_file"
diff --git a/tests/functional/test_search.py b/tests/functional/test_search.py
@@ -45,7 +45,7 @@ def test_pypi_xml_transformation() -> None:
             "version": "1.0",
         },
     ]
-    expected: List["TransformedHit"] = [
+    expected: List[TransformedHit] = [
         {
             "versions": ["1.0", "2.0"],
             "name": "foo",
@@ -159,7 +159,7 @@ def test_latest_prerelease_install_message(
     """
     Test documentation for installing pre-release packages is displayed
     """
-    hits: List["TransformedHit"] = [
+    hits: List[TransformedHit] = [
         {
             "name": "ni",
             "summary": "For knights who say Ni!",
@@ -188,7 +188,7 @@ def test_search_print_results_should_contain_latest_versions(
     """
     Test that printed search results contain the latest package versions
     """
-    hits: List["TransformedHit"] = [
+    hits: List[TransformedHit] = [
         {
             "name": "testlib1",
             "summary": "Test library 1.",
diff --git a/tests/functional/test_uninstall_user.py b/tests/functional/test_uninstall_user.py
@@ -2,6 +2,8 @@
 tests specific to uninstalling --user installs
 """
 
+import platform
+import sys
 from os.path import isdir, isfile, normcase
 
 import pytest
@@ -74,6 +76,12 @@ def test_uninstall_from_usersite_with_dist_in_global_site(
         dist_info_folder = script.base_path / script.site_packages / "pkg-0.1.dist-info"
         assert isdir(dist_info_folder)
 
+    @pytest.mark.xfail(
+        sys.platform == "darwin"
+        and platform.machine() == "arm64"
+        and sys.version_info[:2] in {(3, 8), (3, 9)},
+        reason="Unexpected egg-link install path",
+    )
     def test_uninstall_editable_from_usersite(
         self, script: PipTestEnvironment, data: TestData
     ) -> None:
diff --git a/tests/requirements.txt b/tests/requirements.txt
@@ -1,4 +1,3 @@
-cffi >= 1.17.0rc1 ; python_version > "3.12" # Temporary workaround for Python 3.13 until final CFFI release
 cryptography
 freezegun
 installer
@@ -8,8 +7,10 @@ pytest-rerunfailures
 pytest-xdist
 scripttest
 setuptools
-virtualenv < 20.0 ; python_version < '3.10'
-virtualenv >= 20.0 ; python_version >= '3.10'
+# macOS (darwin) arm64 always uses virtualenv >= 20.0
+# for other platforms, it depends on python version
+virtualenv < 20.0 ; python_version < '3.10' and (sys_platform != 'darwin' or platform_machine != 'arm64')
+virtualenv >= 20.0 ; python_version >= '3.10' or (sys_platform == 'darwin' and platform_machine == 'arm64')
 werkzeug
 wheel
 tomli-w
diff --git a/tests/unit/resolution_resolvelib/test_provider.py b/tests/unit/resolution_resolvelib/test_provider.py
@@ -19,7 +19,7 @@ def build_requirement_information(
     install_requirement = install_req_from_req_string(name)
     # RequirementInformation is typed as a tuple, but it is a namedtupled.
     # https://github.com/sarugaku/resolvelib/blob/7bc025aa2a4e979597c438ad7b17d2e8a08a364e/src/resolvelib/resolvers.pyi#L20-L22
-    requirement_information: "PreferenceInformation" = RequirementInformation(
+    requirement_information: PreferenceInformation = RequirementInformation(
         requirement=SpecifierRequirement(install_requirement),  # type: ignore[call-arg]
         parent=parent,
     )
diff --git a/tests/unit/resolution_resolvelib/test_resolver.py b/tests/unit/resolution_resolvelib/test_resolver.py
@@ -38,7 +38,7 @@ def _make_graph(
 ) -> "DirectedGraph[Optional[str]]":
     """Build graph from edge declarations."""
 
-    graph: "DirectedGraph[Optional[str]]" = DirectedGraph()
+    graph: DirectedGraph[Optional[str]] = DirectedGraph()
     for parent, child in edges:
         parent = cast(str, canonicalize_name(parent)) if parent else None
         child = cast(str, canonicalize_name(child)) if child else None
diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py
@@ -1,6 +1,7 @@
 import collections
 import logging
 import os
+import re
 import textwrap
 from optparse import Values
 from pathlib import Path
@@ -345,6 +346,63 @@ def test_nested_constraints_file(
         assert reqs[0].name == req_name
         assert reqs[0].constraint
 
+    def test_recursive_requirements_file(
+        self, tmpdir: Path, session: PipSession
+    ) -> None:
+        req_files: list[Path] = []
+        req_file_count = 4
+        for i in range(req_file_count):
+            req_file = tmpdir / f"{i}.txt"
+            req_file.write_text(f"-r {(i+1) % req_file_count}.txt")
+            req_files.append(req_file)
+
+        # When the passed requirements file recursively references itself
+        with pytest.raises(
+            RequirementsFileParseError,
+            match=(
+                f"{re.escape(str(req_files[0]))} recursively references itself"
+                f" in {re.escape(str(req_files[req_file_count - 1]))}"
+            ),
+        ):
+            list(parse_requirements(filename=str(req_files[0]), session=session))
+
+        # When one of other the requirements file recursively references itself
+        req_files[req_file_count - 1].write_text(
+            # Just name since they are in the same folder
+            f"-r {req_files[req_file_count - 2].name}"
+        )
+        with pytest.raises(
+            RequirementsFileParseError,
+            match=(
+                f"{re.escape(str(req_files[req_file_count - 2]))} recursively"
+                " references itself in"
+                f" {re.escape(str(req_files[req_file_count - 1]))} and again in"
+                f" {re.escape(str(req_files[req_file_count - 3]))}"
+            ),
+        ):
+            list(parse_requirements(filename=str(req_files[0]), session=session))
+
+    def test_recursive_relative_requirements_file(
+        self, tmpdir: Path, session: PipSession
+    ) -> None:
+        root_req_file = tmpdir / "root.txt"
+        (tmpdir / "nest" / "nest").mkdir(parents=True)
+        level_1_req_file = tmpdir / "nest" / "level_1.txt"
+        level_2_req_file = tmpdir / "nest" / "nest" / "level_2.txt"
+
+        root_req_file.write_text("-r nest/level_1.txt")
+        level_1_req_file.write_text("-r nest/level_2.txt")
+        level_2_req_file.write_text("-r ../../root.txt")
+
+        with pytest.raises(
+            RequirementsFileParseError,
+            match=(
+                f"{re.escape(str(root_req_file))} recursively references itself in"
+                f" {re.escape(str(level_2_req_file))}"
+            ),
+        ):
+            list(parse_requirements(filename=str(root_req_file), session=session))
+
     def test_options_on_a_requirement_line(self, line_processor: LineProcessor) -> None:
         line = (
             'SomeProject --global-option="yo3" --global-option "yo4" '

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Detect recursively referencing requirements files and help users identify`
	`2`	`+the source.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+A valid, but non-existent URL used in a test case was corrected to be a valid URL.`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-Upgrade urllib3 to 1.26.19`
	`1`	`+Upgrade urllib3 to 1.26.20`
Original file line number	Diff line number	Diff line change
`@@ -181,7 +181,7 @@ select = [`
`181`	`181`	`"PLR0",`
`182`	`182`	`"W",`
`183`	`183`	`"RUF100",`
`184`		`- "UP032",`
	`184`	`+ "UP",`
`185`	`185`	`]`
`186`	`186`
`187`	`187`	`[tool.ruff.lint.isort]`