Merge branch 'main' into release/22.1.2

Author: pradyunsg

docs/html/index.md

@@ -48,3 +48,5 @@ lists or chat rooms:
 [packaging-discourse]: https://discuss.python.org/c/packaging/14
 [irc-pypa]: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa
 [irc-pypa-dev]: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa-dev
+
+If you find any security issues, please report to [[email protected]](mailto:[email protected])

docs/html/topics/configuration.md

@@ -213,9 +213,9 @@ Use `no`, `false` or `0` instead.
 
 ## Precedence / Override order
 
-Command line options have override environment variables, which override the
+Command line options override environment variables, which override the
 values in a configuration file. Within the configuration file, values in
-command-specific sections over values in the global section.
+command-specific sections override values in the global section.
 
 Examples:
 

news/10979.bugfix.rst

@@ -0,0 +1 @@
+Revert `#10979 <https://github.com/pypa/pip/issues/10979>`_ since it introduced a regression in certain edge cases.

news/11082.feature.rst

@@ -0,0 +1,3 @@
+Add support to use `truststore <https://pypi.org/project/truststore/>`_ as an alternative SSL certificate verification backend. The backend can be enabled on Python 3.10 and later by installing ``truststore`` into the environment, and adding the ``--use-feature=truststore`` flag to various pip commands.
+
+``truststore`` differs from the current default verification backend (provided by ``certifi``) in it uses the operating system’s trust store, which can be better controlled and augmented to better support non-standard certificates. Depending on feedback, pip may switch to this as the default certificate verification backend in the future.

news/11099.process.rst

@@ -0,0 +1 @@
+Remove reliance on the stdlib cgi module, which is deprecated in Python 3.11.

news/11136.bugfix.rst

@@ -0,0 +1 @@
+Fix an incorrect assertion in the logging logic, that prevented the upgrade prompt from being presented.

src/pip/_internal/cli/cmdoptions.py

@@ -1000,7 +1000,7 @@ def check_list_path_option(options: Values) -> None:
     metavar="feature",
     action="append",
     default=[],
-    choices=["2020-resolver", "fast-deps"],
+    choices=["2020-resolver", "fast-deps", "truststore"],
     help="Enable new functionality, that may be backward incompatible.",
 )
 

src/pip/_internal/cli/req_command.py

@@ -10,7 +10,7 @@
 import sys
 from functools import partial
 from optparse import Values
-from typing import Any, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, List, Optional, Tuple
 
 from pip._internal.cache import WheelCache
 from pip._internal.cli import cmdoptions
@@ -42,9 +42,33 @@
 )
 from pip._internal.utils.virtualenv import running_under_virtualenv
 
+if TYPE_CHECKING:
+    from ssl import SSLContext
+
 logger = logging.getLogger(__name__)
 
 
+def _create_truststore_ssl_context() -> Optional["SSLContext"]:
+    if sys.version_info < (3, 10):
+        raise CommandError("The truststore feature is only available for Python 3.10+")
+
+    try:
+        import ssl
+    except ImportError:
+        logger.warning("Disabling truststore since ssl support is missing")
+        return None
+
+    try:
+        import truststore
+    except ImportError:
+        raise CommandError(
+            "To use the truststore feature, 'truststore' must be installed into "
+            "pip's current environment."
+        )
+
+    return truststore.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+
+
 class SessionCommandMixin(CommandContextMixIn):
 
     """
@@ -84,15 +108,27 @@ def _build_session(
         options: Values,
         retries: Optional[int] = None,
         timeout: Optional[int] = None,
+        fallback_to_certifi: bool = False,
     ) -> PipSession:
-        assert not options.cache_dir or os.path.isabs(options.cache_dir)
+        cache_dir = options.cache_dir
+        assert not cache_dir or os.path.isabs(cache_dir)
+
+        if "truststore" in options.features_enabled:
+            try:
+                ssl_context = _create_truststore_ssl_context()
+            except Exception:
+                if not fallback_to_certifi:
+                    raise
+                ssl_context = None
+        else:
+            ssl_context = None
+
         session = PipSession(
-            cache=(
-                os.path.join(options.cache_dir, "http") if options.cache_dir else None
-            ),
+            cache=os.path.join(cache_dir, "http") if cache_dir else None,
             retries=retries if retries is not None else options.retries,
             trusted_hosts=options.trusted_hosts,
             index_urls=self._get_index_urls(options),
+            ssl_context=ssl_context,
         )
 
         # Handle custom ca-bundles from the user
@@ -142,7 +178,14 @@ def handle_pip_version_check(self, options: Values) -> None:
 
         # Otherwise, check if we're using the latest version of pip available.
         session = self._build_session(
-            options, retries=0, timeout=min(5, options.timeout)
+            options,
+            retries=0,
+            timeout=min(5, options.timeout),
+            # This is set to ensure the function does not fail when truststore is
+            # specified in use-feature but cannot be loaded. This usually raises a
+            # CommandError and shows a nice user-facing error, but this function is not
+            # called in that try-except block.
+            fallback_to_certifi=True,
         )
         with session:
             pip_self_version_check(session, options)

src/pip/_internal/index/collector.py

@@ -2,8 +2,8 @@
 The main purpose of this module is to expose LinkCollector.collect_sources().
 """
 
-import cgi
 import collections
+import email.message
 import functools
 import itertools
 import logging
@@ -155,9 +155,11 @@ def _get_html_response(url: str, session: PipSession) -> Response:
 def _get_encoding_from_headers(headers: ResponseHeaders) -> Optional[str]:
     """Determine if we have any encoding information in our headers."""
     if headers and "Content-Type" in headers:
-        content_type, params = cgi.parse_header(headers["Content-Type"])
-        if "charset" in params:
-            return params["charset"]
+        m = email.message.Message()
+        m["content-type"] = headers["Content-Type"]
+        charset = m.get_param("charset")
+        if charset:
+            return str(charset)
     return None
 
 

src/pip/_internal/network/download.py

@@ -1,6 +1,6 @@
 """Download files with progress indicators.
 """
-import cgi
+import email.message
 import logging
 import mimetypes
 import os
@@ -81,12 +81,13 @@ def parse_content_disposition(content_disposition: str, default_filename: str) -
     Parse the "filename" value from a Content-Disposition header, and
     return the default filename if the result is empty.
     """
-    _type, params = cgi.parse_header(content_disposition)
-    filename = params.get("filename")
+    m = email.message.Message()
+    m["content-type"] = content_disposition
+    filename = m.get_param("filename")
     if filename:
         # We need to sanitize the filename to prevent directory traversal
         # in case the filename contains ".." path parts.
-        filename = sanitize_content_filename(filename)
+        filename = sanitize_content_filename(str(filename))
     return filename or default_filename