Release 2.15.900 (#300)

Ousret · web-flow · commit 74f34299967c · 2025-12-16T08:03:36.000+01:00
2.15.900 (2025-12-16) ===================== - Improved pre-check for socket liveness probe before connection reuse from pool. - Backported "HTTPHeaderDict bytes key handling" from upstream urllib3#3653 - Backported "Expand environment variable of SSLKEYLOGFILE" from upstream urllib3#3705 - Backported "Fix redirect handling when an integer is passed to a pool manager" from upstream urllib3#3655 - Backported "Improved the performance of content decoding by optimizing ``BytesQueueBuffer`` class." from upstream urllib3#3711 - Backported "GHSA-gm62-xv2j-4w53" security patch for "attacker could compose an HTTP response with virtually unlimited links in the ``Content-Encoding`` header" from upstream urllib3@24d7b67
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -44,7 +44,7 @@ jobs:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t"]
         os:
-          - macos-13
+          - macos-14-large
           - windows-2025
           - ubuntu-24.04
         nox-session: ['']
@@ -122,10 +122,12 @@ jobs:
           - python-version: "3.8"
             os: ubuntu-24.04
           - python-version: "3.7"
-            os: macos-13
+            os: macos-14-large
+          - python-version: "3.8"
+            os: macos-14-large
 
     runs-on: ${{ matrix.os }}
-    name: ${{ fromJson('{"macos-13":"macOS","windows-2025":"Windows","ubuntu-24.04":"Ubuntu"}')[matrix.os] }} ${{ matrix.python-version }} ${{ matrix.nox-session }}
+    name: ${{ fromJson('{"macos-14-large":"macOS","windows-2025":"Windows","ubuntu-24.04":"Ubuntu"}')[matrix.os] }} ${{ matrix.python-version }} ${{ matrix.nox-session }}
     continue-on-error: ${{ matrix.experimental }}
     timeout-minutes: 35
     steps:
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -24,7 +24,7 @@ jobs:
       - name: "Setup Python"
         uses: "actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c"
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: "Install dependencies"
         run: python -m pip install --upgrade nox
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
@@ -21,9 +21,12 @@ jobs:
       matrix:
         python-version: [ "3.7", "3.13" ]
         os:
-          - macos-13
+          - macos-15
           - windows-2022
           - ubuntu-22.04
+        exclude:
+          - python-version: "3.7"
+            os: macos-15
 
     steps:
       - name: "Checkout repository"
@@ -58,9 +61,12 @@ jobs:
       matrix:
         python-version: [ "3.7", "3.13" ]
         os:
-          - macos-13
+          - macos-15
           - windows-2022
           - ubuntu-22.04
+        exclude:
+          - python-version: "3.7"
+            os: macos-15
 
     steps:
       - name: "Checkout repository"
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -1,3 +1,13 @@
+2.15.900 (2025-12-16)
+=====================
+
+- Improved pre-check for socket liveness probe before connection reuse from pool.
+- Backported "HTTPHeaderDict bytes key handling" from upstream https://github.com/urllib3/urllib3/pull/3653
+- Backported "Expand environment variable of SSLKEYLOGFILE" from upstream https://github.com/urllib3/urllib3/pull/3705
+- Backported "Fix redirect handling when an integer is passed to a pool manager" from upstream https://github.com/urllib3/urllib3/pull/3655
+- Backported "Improved the performance of content decoding by optimizing ``BytesQueueBuffer`` class." from upstream https://github.com/urllib3/urllib3/pull/3711
+- Backported "GHSA-gm62-xv2j-4w53" security patch for "attacker could compose an HTTP response with virtually unlimited links in the ``Content-Encoding`` header" from upstream https://github.com/urllib3/urllib3/commit/24d7b67eac89f94e11003424bcf0d8f7b72222a8
+
 2.14.908 (2025-11-27)
 =====================
 
diff --git a/src/urllib3/_async/poolmanager.py b/src/urllib3/_async/poolmanager.py
@@ -105,22 +105,18 @@ def __init__(
     ) -> None:
         super().__init__(headers)
 
+        # PoolManager handles redirects itself in PoolManager.urlopen().
+        # It always passes redirect=False to the underlying connection pool to
+        # suppress per-pool redirect handling. If the user supplied a non-Retry
+        # value (int/bool/etc) for retries and we let the pool normalize it
+        # while redirect=False, the resulting Retry object would have redirect
+        # handling disabled, which can interfere with PoolManager's own
+        # redirect logic. Normalize here so redirects remain governed solely by
+        # PoolManager logic.
         if "retries" in connection_pool_kw:
             retries = connection_pool_kw["retries"]
             if not isinstance(retries, Retry):
-                # When Retry is initialized, raise_on_redirect is based
-                # on a redirect boolean value.
-                # But requests made via a pool manager always set
-                # redirect to False, and raise_on_redirect always ends
-                # up being False consequently.
-                # Here we fix the issue by setting raise_on_redirect to
-                # a value needed by the pool manager without considering
-                # the redirect boolean.
-
-                raise_on_redirect = retries is not False
-
-                retries = Retry.from_int(retries, redirect=False)
-                retries.raise_on_redirect = raise_on_redirect
+                retries = Retry.from_int(retries)
 
                 connection_pool_kw = connection_pool_kw.copy()
                 connection_pool_kw["retries"] = retries
diff --git a/src/urllib3/_collections.py b/src/urllib3/_collections.py
@@ -260,13 +260,19 @@ def __setitem__(self, key: str, val: str) -> None:
         self._container[_lower_wrapper(key)] = [key, val]
 
     def __getitem__(self, key: str) -> str:
+        if isinstance(key, bytes):
+            key = key.decode("latin-1")
         val = self._container[_lower_wrapper(key)]
         return ", ".join(val[1:])
 
     def __delitem__(self, key: str) -> None:
+        if isinstance(key, bytes):
+            key = key.decode("latin-1")
         del self._container[_lower_wrapper(key)]
 
     def __contains__(self, key: object) -> bool:
+        if isinstance(key, bytes):
+            key = key.decode("latin-1")
         if isinstance(key, str):
             return _lower_wrapper(key) in self._container
         return False
@@ -377,6 +383,8 @@ def getlist(
     ) -> list[str] | _DT:
         """Returns a list of all the values for the named field. Returns an
         empty list if the key doesn't exist."""
+        if isinstance(key, bytes):
+            key = key.decode("latin-1")
         try:
             vals = self._container[_lower_wrapper(key)]
         except KeyError:
diff --git a/src/urllib3/_version.py b/src/urllib3/_version.py
@@ -1,4 +1,4 @@
 # This file is protected via CODEOWNERS
 from __future__ import annotations
 
-__version__ = "2.14.908"
+__version__ = "2.15.900"
diff --git a/src/urllib3/poolmanager.py b/src/urllib3/poolmanager.py
@@ -242,22 +242,18 @@ def __init__(
     ) -> None:
         super().__init__(headers)
 
+        # PoolManager handles redirects itself in PoolManager.urlopen().
+        # It always passes redirect=False to the underlying connection pool to
+        # suppress per-pool redirect handling. If the user supplied a non-Retry
+        # value (int/bool/etc) for retries and we let the pool normalize it
+        # while redirect=False, the resulting Retry object would have redirect
+        # handling disabled, which can interfere with PoolManager's own
+        # redirect logic. Normalize here so redirects remain governed solely by
+        # PoolManager logic.
         if "retries" in connection_pool_kw:
             retries = connection_pool_kw["retries"]
             if not isinstance(retries, Retry):
-                # When Retry is initialized, raise_on_redirect is based
-                # on a redirect boolean value.
-                # But requests made via a pool manager always set
-                # redirect to False, and raise_on_redirect always ends
-                # up being False consequently.
-                # Here we fix the issue by setting raise_on_redirect to
-                # a value needed by the pool manager without considering
-                # the redirect boolean.
-
-                raise_on_redirect = retries is not False
-
-                retries = Retry.from_int(retries, redirect=False)
-                retries.raise_on_redirect = raise_on_redirect
+                retries = Retry.from_int(retries)
 
                 connection_pool_kw = connection_pool_kw.copy()
                 connection_pool_kw["retries"] = retries
diff --git a/src/urllib3/response.py b/src/urllib3/response.py
@@ -206,8 +206,20 @@ class MultiDecoder(ContentDecoder):
         they were applied.
     """
 
+    # Maximum allowed number of chained HTTP encodings in the
+    # Content-Encoding header.
+    max_decode_links = 5
+
     def __init__(self, modes: str) -> None:
-        self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")]
+        encodings = [m.strip() for m in modes.split(",")]
+
+        if len(encodings) > self.max_decode_links:
+            raise DecodeError(
+                "Too many content encodings in the chain: "
+                f"{len(encodings)} > {self.max_decode_links}"
+            )
+
+        self._decoders = [_get_decoder(e) for e in encodings]
 
     def flush(self) -> bytes:
         return self._decoders[0].flush()
diff --git a/src/urllib3/util/response.py b/src/urllib3/util/response.py
@@ -63,13 +63,10 @@ class BytesQueueBuffer:
 
      * self.buffer, which contains the full data
      * the largest chunk that we will copy in get()
-
-    The worst case scenario is a single chunk, in which case we'll make a full copy of
-    the data inside get().
     """
 
     def __init__(self) -> None:
-        self.buffer: typing.Deque[bytes] = collections.deque()
+        self.buffer: typing.Deque[bytes | memoryview[bytes]] = collections.deque()
         self._size: int = 0
 
     def __len__(self) -> int:
@@ -87,13 +84,18 @@ def get(self, n: int) -> bytes:
         elif n < 0:
             raise ValueError("n should be > 0")
 
+        if len(self.buffer[0]) == n and isinstance(self.buffer[0], bytes):
+            self._size -= n
+            return self.buffer.popleft()
+
         fetched = 0
         ret = io.BytesIO()
         while fetched < n:
             remaining = n - fetched
             chunk = self.buffer.popleft()
             chunk_length = len(chunk)
             if remaining < chunk_length:
+                chunk = memoryview(chunk)
                 left_chunk, right_chunk = chunk[:remaining], chunk[remaining:]
                 ret.write(left_chunk)
                 self.buffer.appendleft(right_chunk)
diff --git a/src/urllib3/util/socket_state.py b/src/urllib3/util/socket_state.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import errno
 import sys
 import socket
 import struct
@@ -12,6 +13,17 @@
 IS_NT = sys.platform in {"win32", "cygwin", "msys"}
 IS_DARWIN_OR_BSD = not IS_NT and (sys.platform == "darwin" or "bsd" in sys.platform)
 IS_LINUX = not IS_DARWIN_OR_BSD and sys.platform == "linux"
+SOCKET_CLOSED_ERRNOS: frozenset[int] = frozenset(
+    filter(
+        None,
+        (
+            getattr(errno, "EBADF", None),
+            getattr(errno, "ENOTSOCK", None),
+            getattr(errno, "EINVAL", None),
+            getattr(errno, "ENOTCONN", None),
+        ),
+    )
+)
 
 # Of course, Windows don't have any nice shortcut
 # through getsockopt, why make it simple when a
@@ -84,6 +96,15 @@ def is_established(sock: socket.socket | AsyncSocket | SSLTransport) -> bool:
     if sock.fileno() == -1:
         return False
 
+    # catch earlier the most catastrophic states
+    # this pre-check avoid wasting time on TCP probing
+    try:
+        err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+        if err != 0:
+            return False
+    except OSError:
+        return False
+
     # Well... If we're on UDP (or anything else),
     if sock.type is not socket.SOCK_STREAM:
         return True
@@ -96,7 +117,11 @@ def is_established(sock: socket.socket | AsyncSocket | SSLTransport) -> bool:
 
         try:
             info = sock.getsockopt(socket.IPPROTO_TCP, TCP_CONNECTION_INFO, 1024)
-        except OSError:
+        except OSError as e:
+            # EBADF, ENOTSOCK, EINVAL are expected for invalid/closed sockets
+            # Other errors might indicate real issues
+            if e.errno in SOCKET_CLOSED_ERRNOS:
+                return False
             return True
 
         state: int = struct.unpack("B", info[0:1])[0]
diff --git a/src/urllib3/util/ssl_.py b/src/urllib3/util/ssl_.py
@@ -585,7 +585,11 @@ def create_urllib3_context(
     # Enable logging of TLS session keys via defacto standard environment variable
     # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values.
     if hasattr(context, "keylog_filename"):
-        sslkeylogfile = os.environ.get("SSLKEYLOGFILE")
+        if "SSLKEYLOGFILE" in os.environ:
+            sslkeylogfile = os.path.expandvars(os.environ.get("SSLKEYLOGFILE"))
+        else:
+            sslkeylogfile = None
+
         if sslkeylogfile:
             context.keylog_filename = sslkeylogfile
 
diff --git a/test/test_collections.py b/test/test_collections.py
@@ -227,6 +227,10 @@ def test_delitem(self, d: HTTPHeaderDict) -> None:
         assert "cookie" not in d
         assert "COOKIE" not in d
 
+    def test_delitem_with_bytes_key(self, d: HTTPHeaderDict) -> None:
+        del d[b"cookie"]  # type: ignore[arg-type]
+        assert "cookie" not in d
+
     def test_add_well_known_multiheader(self, d: HTTPHeaderDict) -> None:
         d.add("COOKIE", "asdf")
         assert d.getlist("cookie") == ["foo", "bar", "asdf"]
@@ -312,6 +316,20 @@ def test_getlist(self, d: HTTPHeaderDict) -> None:
         d.add("b", "asdf")
         assert d.getlist("b") == ["asdf"]
 
+    def test_getlist_with_bytes_key(self, d: HTTPHeaderDict) -> None:
+        assert d.getlist(b"cookie") == ["foo", "bar"]  # type: ignore[call-overload]
+
+    def test_getitem_with_bytes(self, d: HTTPHeaderDict) -> None:
+        d["Content-Type"] = "application/json"
+        d.add("Content-Type", "charset=utf-8")
+        result = d[b"Content-Type"]  # type: ignore[index]
+        assert result == "application/json, charset=utf-8"
+
+    def test_contains_with_bytes(self, d: HTTPHeaderDict) -> None:
+        d["Content-Type"] = "application/json"
+        assert b"Content-Type" in d  # type: ignore[comparison-overlap]
+        assert b"X-Not-There" not in d  # type: ignore[comparison-overlap]
+
     def test_getlist_after_copy(self, d: HTTPHeaderDict) -> None:
         assert d.getlist("cookie") == HTTPHeaderDict(d).getlist("cookie")
 
diff --git a/test/test_response.py b/test/test_response.py
@@ -411,6 +411,17 @@ def test_read_multi_decoding_deflate_deflate(self) -> None:
         assert r.read(9 * 37) == b"foobarbaz" * 37
         assert r.read() == b""
 
+    def test_read_multi_decoding_too_many_links(self) -> None:
+        fp = BytesIO(b"foo")
+
+        with pytest.raises(
+            DecodeError, match="Too many content encodings in the chain: 6 > 5"
+        ):
+            HTTPResponse(
+                fp,
+                headers={"content-encoding": "gzip, deflate, br, zstd, gzip, deflate"},
+            )
+
     def test_body_blob(self) -> None:
         resp = HTTPResponse(b"foo")
         assert resp.data == b"foo"
diff --git a/test/with_dummyserver/asynchronous/test_poolmanager.py b/test/with_dummyserver/asynchronous/test_poolmanager.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import typing
+
 from test import LONG_TIMEOUT
 
 import pytest
@@ -21,8 +23,12 @@ def setup_class(cls) -> None:
         cls.base_url = f"http://{cls.host}:{cls.port}"
         cls.base_url_alt = f"http://{cls.host_alt}:{cls.port}"
 
-    async def test_redirect(self) -> None:
-        async with AsyncPoolManager() as http:
+    @pytest.mark.parametrize(
+        "pool_manager_kwargs",
+        ({}, {"retries": None}, {"retries": 1}, {"retries": Retry(1)}),
+    )
+    async def test_redirect(self, pool_manager_kwargs: dict[str, typing.Any]) -> None:
+        async with AsyncPoolManager(**pool_manager_kwargs) as http:
             r = await http.request(
                 "GET",
                 f"{self.base_url}/redirect",
@@ -63,8 +69,14 @@ async def test_redirect_with_alt_top_level(self) -> None:
             assert r.status == 200
             assert await r.data == b"Dummy server!"
 
-    async def test_redirect_twice(self) -> None:
-        async with AsyncPoolManager() as http:
+    @pytest.mark.parametrize(
+        "pool_manager_kwargs",
+        ({}, {"retries": None}, {"retries": 2}, {"retries": Retry(2)}),
+    )
+    async def test_redirect_twice(
+        self, pool_manager_kwargs: dict[str, typing.Any]
+    ) -> None:
+        async with AsyncPoolManager(**pool_manager_kwargs) as http:
             r = await http.request(
                 "GET",
                 f"{self.base_url}/redirect",
diff --git a/test/with_dummyserver/test_https.py b/test/with_dummyserver/test_https.py
diff --git a/test/with_dummyserver/test_poolmanager.py b/test/with_dummyserver/test_poolmanager.py
