[PR #11634/cde03b9 backport][3.13] Fix blocking I/O to load netrc when creating requests (#11680)

bdraco · web-flow · commit b2ddca1bca01 · 2025-10-16T08:11:39.000-10:00
diff --git a/CHANGES/11634.bugfix.rst b/CHANGES/11634.bugfix.rst
@@ -0,0 +1 @@
+Fixed blocking I/O in the event loop when using netrc authentication by moving netrc file lookup to an executor -- by :user:`bdraco`.
diff --git a/aiohttp/client.py b/aiohttp/client.py
@@ -98,7 +98,9 @@
     EMPTY_BODY_METHODS,
     BasicAuth,
     TimeoutHandle,
+    basicauth_from_netrc,
     get_env_proxy_for_url,
+    netrc_from_env,
     sentinel,
     strip_auth_from_url,
 )
@@ -657,6 +659,20 @@ async def _request(
                         )
                     ):
                         auth = self._default_auth
+
+                    # Try netrc if auth is still None and trust_env is enabled.
+                    # Only check if NETRC environment variable is set to avoid
+                    # creating an expensive executor job unnecessarily.
+                    if (
+                        auth is None
+                        and self._trust_env
+                        and url.host is not None
+                        and os.environ.get("NETRC")
+                    ):
+                        auth = await self._loop.run_in_executor(
+                            None, self._get_netrc_auth, url.host
+                        )
+
                     # It would be confusing if we support explicit
                     # Authorization header with auth argument
                     if (
@@ -1211,6 +1227,19 @@ def _prepare_headers(self, headers: Optional[LooseHeaders]) -> "CIMultiDict[str]
                     added_names.add(key)
         return result
 
+    def _get_netrc_auth(self, host: str) -> Optional[BasicAuth]:
+        """
+        Get auth from netrc for the given host.
+
+        This method is designed to be called in an executor to avoid
+        blocking I/O in the event loop.
+        """
+        netrc_obj = netrc_from_env()
+        try:
+            return basicauth_from_netrc(netrc_obj, host)
+        except LookupError:
+            return None
+
     if sys.version_info >= (3, 11) and TYPE_CHECKING:
 
         def get(
diff --git a/aiohttp/client_reqrep.py b/aiohttp/client_reqrep.py
@@ -53,8 +53,6 @@
     BasicAuth,
     HeadersMixin,
     TimerNoop,
-    basicauth_from_netrc,
-    netrc_from_env,
     noop,
     reify,
     sentinel,
@@ -1164,10 +1162,6 @@ def update_auth(self, auth: Optional[BasicAuth], trust_env: bool = False) -> Non
         """Set basic auth."""
         if auth is None:
             auth = self.auth
-        if auth is None and trust_env and self.url.host is not None:
-            netrc_obj = netrc_from_env()
-            with contextlib.suppress(LookupError):
-                auth = basicauth_from_netrc(netrc_obj, self.url.host)
         if auth is None:
             return
 
diff --git a/tests/conftest.py b/tests/conftest.py
@@ -69,10 +69,6 @@ def blockbuster(request: pytest.FixtureRequest) -> Iterator[None]:
     with blockbuster_ctx(
         "aiohttp", excluded_modules=["aiohttp.pytest_plugin", "aiohttp.test_utils"]
     ) as bb:
-        # TODO: Fix blocking call in ClientRequest's constructor.
-        # https://github.com/aio-libs/aiohttp/issues/10435
-        for func in ["io.TextIOWrapper.read", "os.stat"]:
-            bb.functions[func].can_block_in("aiohttp/client_reqrep.py", "update_auth")
         for func in [
             "os.getcwd",
             "os.readlink",
@@ -285,7 +281,35 @@ def netrc_contents(
 
 
 @pytest.fixture
-def start_connection():
+def netrc_default_contents(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
+    """Create a temporary netrc file with default test credentials and set NETRC env var."""
+    netrc_file = tmp_path / ".netrc"
+    netrc_file.write_text("default login netrc_user password netrc_pass\n")
+
+    monkeypatch.setenv("NETRC", str(netrc_file))
+
+    return netrc_file
+
+
+@pytest.fixture
+def no_netrc(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Ensure NETRC environment variable is not set."""
+    monkeypatch.delenv("NETRC", raising=False)
+
+
+@pytest.fixture
+def netrc_other_host(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:
+    """Create a temporary netrc file with credentials for a different host and set NETRC env var."""
+    netrc_file = tmp_path / ".netrc"
+    netrc_file.write_text("machine other.example.com login user password pass\n")
+
+    monkeypatch.setenv("NETRC", str(netrc_file))
+
+    return netrc_file
+
+
+@pytest.fixture
+def start_connection() -> Iterator[mock.Mock]:
     with mock.patch(
         "aiohttp.connector.aiohappyeyeballs.start_connection",
         autospec=True,
diff --git a/tests/test_client_functional.py b/tests/test_client_functional.py
@@ -77,8 +77,25 @@ def fname(here):
     return here / "conftest.py"
 
 
-async def test_keepalive_two_requests_success(aiohttp_client) -> None:
-    async def handler(request):
+@pytest.fixture
+def headers_echo_client(
+    aiohttp_client: AiohttpClient,
+) -> Callable[..., Awaitable[TestClient[web.Request, web.Application]]]:
+    """Create a client with an app that echoes request headers as JSON."""
+
+    async def factory(**kwargs: Any) -> TestClient[web.Request, web.Application]:
+        async def handler(request: web.Request) -> web.Response:
+            return web.json_response({"headers": dict(request.headers)})
+
+        app = web.Application()
+        app.router.add_get("/", handler)
+        return await aiohttp_client(app, **kwargs)
+
+    return factory
+
+
+async def test_keepalive_two_requests_success(aiohttp_client: AiohttpClient) -> None:
+    async def handler(request: web.Request) -> web.Response:
         body = await request.read()
         assert b"" == body
         return web.Response(body=b"OK")
@@ -3712,29 +3729,25 @@ async def handler(request):
     assert not ctx._coro.cr_running
 
 
-async def test_session_auth(aiohttp_client) -> None:
-    async def handler(request):
-        return web.json_response({"headers": dict(request.headers)})
-
-    app = web.Application()
-    app.router.add_get("/", handler)
-
-    client = await aiohttp_client(app, auth=aiohttp.BasicAuth("login", "pass"))
+async def test_session_auth(
+    headers_echo_client: Callable[
+        ..., Awaitable[TestClient[web.Request, web.Application]]
+    ],
+) -> None:
+    client = await headers_echo_client(auth=aiohttp.BasicAuth("login", "pass"))
 
     r = await client.get("/")
     assert r.status == 200
     content = await r.json()
     assert content["headers"]["Authorization"] == "Basic bG9naW46cGFzcw=="
 
 
-async def test_session_auth_override(aiohttp_client) -> None:
-    async def handler(request):
-        return web.json_response({"headers": dict(request.headers)})
-
-    app = web.Application()
-    app.router.add_get("/", handler)
-
-    client = await aiohttp_client(app, auth=aiohttp.BasicAuth("login", "pass"))
+async def test_session_auth_override(
+    headers_echo_client: Callable[
+        ..., Awaitable[TestClient[web.Request, web.Application]]
+    ],
+) -> None:
+    client = await headers_echo_client(auth=aiohttp.BasicAuth("login", "pass"))
 
     r = await client.get("/", auth=aiohttp.BasicAuth("other_login", "pass"))
     assert r.status == 200
@@ -3756,30 +3769,77 @@ async def handler(request):
         await client.get("/", headers=headers)
 
 
-async def test_session_headers(aiohttp_client) -> None:
-    async def handler(request):
-        return web.json_response({"headers": dict(request.headers)})
+@pytest.mark.usefixtures("netrc_default_contents")
+async def test_netrc_auth_from_env(  # type: ignore[misc]
+    headers_echo_client: Callable[
+        ..., Awaitable[TestClient[web.Request, web.Application]]
+    ],
+) -> None:
+    """Test that netrc authentication works when NETRC env var is set and trust_env=True."""
+    client = await headers_echo_client(trust_env=True)
+    async with client.get("/") as r:
+        assert r.status == 200
+        content = await r.json()
+    # Base64 encoded "netrc_user:netrc_pass" is "bmV0cmNfdXNlcjpuZXRyY19wYXNz"
+    assert content["headers"]["Authorization"] == "Basic bmV0cmNfdXNlcjpuZXRyY19wYXNz"
 
-    app = web.Application()
-    app.router.add_get("/", handler)
 
-    client = await aiohttp_client(app, headers={"X-Real-IP": "192.168.0.1"})
+@pytest.mark.usefixtures("no_netrc")
+async def test_netrc_auth_skipped_without_env_var(  # type: ignore[misc]
+    headers_echo_client: Callable[
+        ..., Awaitable[TestClient[web.Request, web.Application]]
+    ],
+) -> None:
+    """Test that netrc authentication is skipped when NETRC env var is not set."""
+    client = await headers_echo_client(trust_env=True)
+    async with client.get("/") as r:
+        assert r.status == 200
+        content = await r.json()
+    # No Authorization header should be present
+    assert "Authorization" not in content["headers"]
+
+
+@pytest.mark.usefixtures("netrc_default_contents")
+async def test_netrc_auth_overridden_by_explicit_auth(  # type: ignore[misc]
+    headers_echo_client: Callable[
+        ..., Awaitable[TestClient[web.Request, web.Application]]
+    ],
+) -> None:
+    """Test that explicit auth parameter overrides netrc authentication."""
+    client = await headers_echo_client(trust_env=True)
+    # Make request with explicit auth (should override netrc)
+    async with client.get(
+        "/", auth=aiohttp.BasicAuth("explicit_user", "explicit_pass")
+    ) as r:
+        assert r.status == 200
+        content = await r.json()
+    # Base64 encoded "explicit_user:explicit_pass" is "ZXhwbGljaXRfdXNlcjpleHBsaWNpdF9wYXNz"
+    assert (
+        content["headers"]["Authorization"]
+        == "Basic ZXhwbGljaXRfdXNlcjpleHBsaWNpdF9wYXNz"
+    )
+
+
+async def test_session_headers(
+    headers_echo_client: Callable[
+        ..., Awaitable[TestClient[web.Request, web.Application]]
+    ],
+) -> None:
+    client = await headers_echo_client(headers={"X-Real-IP": "192.168.0.1"})
 
     r = await client.get("/")
     assert r.status == 200
     content = await r.json()
     assert content["headers"]["X-Real-IP"] == "192.168.0.1"
 
 
-async def test_session_headers_merge(aiohttp_client) -> None:
-    async def handler(request):
-        return web.json_response({"headers": dict(request.headers)})
-
-    app = web.Application()
-    app.router.add_get("/", handler)
-
-    client = await aiohttp_client(
-        app, headers=[("X-Real-IP", "192.168.0.1"), ("X-Sent-By", "requests")]
+async def test_session_headers_merge(
+    headers_echo_client: Callable[
+        ..., Awaitable[TestClient[web.Request, web.Application]]
+    ],
+) -> None:
+    client = await headers_echo_client(
+        headers=[("X-Real-IP", "192.168.0.1"), ("X-Sent-By", "requests")]
     )
 
     r = await client.get("/", headers={"X-Sent-By": "aiohttp"})
diff --git a/tests/test_client_request.py b/tests/test_client_request.py
@@ -15,7 +15,7 @@
 from yarl import URL
 
 import aiohttp
-from aiohttp import BaseConnector, hdrs, helpers, payload
+from aiohttp import BaseConnector, hdrs, payload
 from aiohttp.abc import AbstractStreamWriter
 from aiohttp.client_exceptions import ClientConnectionError
 from aiohttp.client_reqrep import (
@@ -1545,26 +1545,6 @@ def test_gen_default_accept_encoding(
             assert _gen_default_accept_encoding() == expected
 
 
-@pytest.mark.parametrize(
-    ("netrc_contents", "expected_auth"),
-    [
-        (
-            "machine example.com login username password pass\n",
-            helpers.BasicAuth("username", "pass"),
-        )
-    ],
-    indirect=("netrc_contents",),
-)
-@pytest.mark.usefixtures("netrc_contents")
-def test_basicauth_from_netrc_present(
-    make_request: Any,
-    expected_auth: Optional[helpers.BasicAuth],
-):
-    """Test appropriate Authorization header is sent when netrc is not empty."""
-    req = make_request("get", "http://example.com", trust_env=True)
-    assert req.headers[hdrs.AUTHORIZATION] == expected_auth.encode()
-
-
 @pytest.mark.parametrize(
     "netrc_contents",
     ("machine example.com login username password pass\n",),
diff --git a/tests/test_client_session.py b/tests/test_client_session.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Fixed blocking I/O in the event loop when using netrc authentication by moving netrc file lookup to an executor -- by :user:`bdraco`.