Parameterize some codspeed benchmarks by connection type (tcp vs ssl) (aio-libs#12823)

Author: tarasko

CHANGES/12823.misc.rst

@@ -0,0 +1,2 @@
+Parameterized some codspeed benchmarks by connection type (SSL + TCP).
+Previously, benchmarks only ran for TCP connection type. -- by :user:`tarasko`.

docs/spelling_wordlist.txt

@@ -77,6 +77,7 @@ cmd
 codecov
 codebase
 codec
+codspeed
 Codings
 committer
 committers

tests/test_benchmarks_client.py

@@ -1,15 +1,17 @@
 """codspeed benchmarks for HTTP client."""
 
 import asyncio
-from collections.abc import Iterator
-from typing import TYPE_CHECKING, Any
+import ssl
+from collections.abc import Awaitable, Callable, Iterator
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, TypedDict
 
 import pytest
 from pytest_aiohttp import AiohttpClient, AiohttpServer
 from yarl import URL
 
 from aiohttp import hdrs, request, web
-from aiohttp.test_utils import TestServer
+from aiohttp.test_utils import TestClient, TestServer
 
 if TYPE_CHECKING:
     from pytest_codspeed import BenchmarkFixture
@@ -18,6 +20,56 @@
     BenchmarkFixture = pytest_codspeed.BenchmarkFixture
 
 
+@pytest.fixture
+def aiohttp_client_sync(
+    event_loop: asyncio.AbstractEventLoop,
+) -> Iterator[
+    Callable[[web.Application], Awaitable[TestClient[web.Request, web.Application]]]
+]:
+    clients = []
+
+    async def go(
+        app: web.Application,
+        *,
+        server_kwargs: dict[str, Any] | None = None,
+    ) -> TestClient[web.Request, web.Application]:
+        server = TestServer(app)
+        client = TestClient(server)
+        await server.start_server(**(server_kwargs or {}))
+        await client.start_server()
+        clients.append(client)
+        return client
+
+    yield go
+
+    while clients:
+        event_loop.run_until_complete(clients.pop().close())
+
+
+class _ConnArgs(TypedDict, total=False):
+    ssl: ssl.SSLContext
+
+
+@dataclass(frozen=True)
+class ConnectionType:
+    s_kwargs: _ConnArgs
+    c_kwargs: _ConnArgs
+
+
+@pytest.fixture(params=("tcp", "ssl"), ids=("tcp", "ssl"))
+def conn_type(
+    request: pytest.FixtureRequest,
+    ssl_ctx: ssl.SSLContext,
+    client_ssl_ctx: ssl.SSLContext,
+) -> ConnectionType:
+    if request.param == "ssl":
+        return ConnectionType(
+            s_kwargs={"ssl": ssl_ctx},
+            c_kwargs={"ssl": client_ssl_ctx},
+        )
+    return ConnectionType(s_kwargs={}, c_kwargs={})
+
+
 @pytest.fixture
 def aiohttp_server_sync(
     event_loop: asyncio.AbstractEventLoop,
@@ -45,8 +97,9 @@ async def go(
 
 def test_one_hundred_simple_get_requests(
     event_loop: asyncio.AbstractEventLoop,
-    aiohttp_client: AiohttpClient,
+    aiohttp_client_sync: AiohttpClient,
     benchmark: BenchmarkFixture,
+    conn_type: ConnectionType,
 ) -> None:
     """Benchmark 100 simple GET requests."""
     message_count = 100
@@ -58,9 +111,9 @@ async def handler(request: web.Request) -> web.Response:
     app.router.add_route("GET", "/", handler)
 
     async def run_client_benchmark() -> None:
-        client = await aiohttp_client(app)
+        client = await aiohttp_client_sync(app, server_kwargs=conn_type.s_kwargs)
         for _ in range(message_count):
-            await client.get("/")
+            await client.get("/", **conn_type.c_kwargs)
         await client.close()
 
     @benchmark
@@ -70,7 +123,7 @@ def _run() -> None:
 
 def test_one_hundred_simple_get_requests_alternating_clients(
     event_loop: asyncio.AbstractEventLoop,
-    aiohttp_client: AiohttpClient,
+    aiohttp_client_sync: AiohttpClient,
     benchmark: BenchmarkFixture,
 ) -> None:
     """Benchmark 100 simple GET requests with alternating clients."""
@@ -83,8 +136,8 @@ async def handler(request: web.Request) -> web.Response:
     app.router.add_route("GET", "/", handler)
 
     async def run_client_benchmark() -> None:
-        client1 = await aiohttp_client(app)
-        client2 = await aiohttp_client(app)
+        client1 = await aiohttp_client_sync(app)
+        client2 = await aiohttp_client_sync(app)
         for i in range(message_count):
             if i % 2 == 0:
                 await client1.get("/")
@@ -154,8 +207,9 @@ def _run() -> None:
 
 def test_one_hundred_get_requests_with_1024_chunked_payload(
     event_loop: asyncio.AbstractEventLoop,
-    aiohttp_client: AiohttpClient,
+    aiohttp_client_sync: AiohttpClient,
     benchmark: BenchmarkFixture,
+    conn_type: ConnectionType,
 ) -> None:
     """Benchmark 100 GET requests with a small payload of 1024 bytes."""
     message_count = 100
@@ -170,9 +224,9 @@ async def handler(request: web.Request) -> web.Response:
     app.router.add_route("GET", "/", handler)
 
     async def run_client_benchmark() -> None:
-        client = await aiohttp_client(app)
+        client = await aiohttp_client_sync(app, server_kwargs=conn_type.s_kwargs)
         for _ in range(message_count):
-            resp = await client.get("/")
+            resp = await client.get("/", **conn_type.c_kwargs)
             await resp.read()
         await client.close()
 
@@ -212,8 +266,9 @@ def _run() -> None:
 
 def test_one_hundred_get_requests_with_1mb_chunked_payload(
     event_loop: asyncio.AbstractEventLoop,
-    aiohttp_client: AiohttpClient,
+    aiohttp_client_sync: AiohttpClient,
     benchmark: BenchmarkFixture,
+    conn_type: ConnectionType,
 ) -> None:
     """Benchmark 100 GET requests with a 1 MiB chunked payload using read."""
     message_count = 100
@@ -228,9 +283,9 @@ async def handler(request: web.Request) -> web.Response:
     app.router.add_route("GET", "/", handler)
 
     async def run_client_benchmark() -> None:
-        client = await aiohttp_client(app)
+        client = await aiohttp_client_sync(app, server_kwargs=conn_type.s_kwargs)
         for _ in range(message_count):
-            resp = await client.get("/")
+            resp = await client.get("/", **conn_type.c_kwargs)
             await resp.read()
         await client.close()
 

tests/test_benchmarks_client_ws.py

@@ -1,8 +1,10 @@
 """codspeed benchmarks for websocket client."""
 
 import asyncio
+import ssl
 from collections.abc import Awaitable, Callable, Iterator
-from typing import TYPE_CHECKING, Any
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, TypedDict
 
 import pytest
 from pytest_aiohttp import AiohttpClient
@@ -34,10 +36,12 @@ async def go(
         server_kwargs: dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> TestClient[web.Request, web.Application]:
-        server_kwargs = server_kwargs or {}
+        server_kwargs = dict(server_kwargs or {})
+        server_ssl_context = server_kwargs.pop("ssl", None)
         server = TestServer(__param, **server_kwargs)
         client = aiohttp_client_cls(server, **kwargs)
 
+        await server.start_server(ssl=server_ssl_context)
         await client.start_server()
         clients.append(client)
         return client
@@ -48,6 +52,30 @@ async def go(
         event_loop.run_until_complete(clients.pop().close())
 
 
+class _ConnArgs(TypedDict, total=False):
+    ssl: ssl.SSLContext
+
+
+@dataclass(frozen=True)
+class ConnectionType:
+    s_kwargs: _ConnArgs
+    c_kwargs: _ConnArgs
+
+
+@pytest.fixture(params=("tcp", "ssl"), ids=("tcp", "ssl"))
+def conn_type(
+    request: pytest.FixtureRequest,
+    ssl_ctx: ssl.SSLContext,
+    client_ssl_ctx: ssl.SSLContext,
+) -> ConnectionType:
+    if request.param == "ssl":
+        return ConnectionType(
+            s_kwargs={"ssl": ssl_ctx},
+            c_kwargs={"ssl": client_ssl_ctx},
+        )
+    return ConnectionType(s_kwargs={}, c_kwargs={})
+
+
 def test_one_thousand_round_trip_websocket_text_messages(
     event_loop: asyncio.AbstractEventLoop,
     aiohttp_client_sync: AiohttpClient,
@@ -84,6 +112,7 @@ def test_one_thousand_round_trip_websocket_binary_messages(
     event_loop: asyncio.AbstractEventLoop,
     aiohttp_client_sync: AiohttpClient,
     benchmark: BenchmarkFixture,
+    conn_type: ConnectionType,
     msg_size: int,
 ) -> None:
     """Benchmark round trip of 1000 WebSocket binary messages."""
@@ -102,8 +131,8 @@ async def handler(request: web.Request) -> web.WebSocketResponse:
     app.router.add_route("GET", "/", handler)
 
     async def run_websocket_benchmark() -> None:
-        client = await aiohttp_client_sync(app)
-        resp = await client.ws_connect("/")
+        client = await aiohttp_client_sync(app, server_kwargs=conn_type.s_kwargs)
+        resp = await client.ws_connect("/", **conn_type.c_kwargs)
         for _ in range(message_count):
             await resp.receive()
         await resp.close()

tests/test_benchmarks_web_fileresponse.py

@@ -2,8 +2,10 @@
 
 import asyncio
 import pathlib
+import ssl
 from collections.abc import Awaitable, Callable, Iterator
-from typing import TYPE_CHECKING, Any
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any, TypedDict
 
 import pytest
 from multidict import CIMultiDict
@@ -35,10 +37,12 @@ async def go(
         server_kwargs: dict[str, Any] | None = None,
         **kwargs: Any,
     ) -> TestClient[web.Request, web.Application]:
-        server_kwargs = server_kwargs or {}
+        server_kwargs = dict(server_kwargs or {})
+        server_ssl_context = server_kwargs.pop("ssl", None)
         server = TestServer(__param, **server_kwargs)
         client = aiohttp_client_cls(server, **kwargs)
 
+        await server.start_server(ssl=server_ssl_context)
         await client.start_server()
         clients.append(client)
         return client
@@ -49,10 +53,35 @@ async def go(
         event_loop.run_until_complete(clients.pop().close())
 
 
+class _ConnArgs(TypedDict, total=False):
+    ssl: ssl.SSLContext
+
+
+@dataclass(frozen=True)
+class ConnectionType:
+    s_kwargs: _ConnArgs
+    c_kwargs: _ConnArgs
+
+
+@pytest.fixture(params=("tcp", "ssl"), ids=("tcp", "ssl"))
+def conn_type(
+    request: pytest.FixtureRequest,
+    ssl_ctx: ssl.SSLContext,
+    client_ssl_ctx: ssl.SSLContext,
+) -> ConnectionType:
+    if request.param == "ssl":
+        return ConnectionType(
+            s_kwargs={"ssl": ssl_ctx},
+            c_kwargs={"ssl": client_ssl_ctx},
+        )
+    return ConnectionType(s_kwargs={}, c_kwargs={})
+
+
 def test_simple_web_file_response(
     event_loop: asyncio.AbstractEventLoop,
-    aiohttp_client: AiohttpClient,
+    aiohttp_client_sync: AiohttpClient,
     benchmark: BenchmarkFixture,
+    conn_type: ConnectionType,
 ) -> None:
     """Benchmark creating 100 simple web.FileResponse."""
     response_count = 100
@@ -65,9 +94,9 @@ async def handler(request: web.Request) -> web.FileResponse:
     app.router.add_route("GET", "/", handler)
 
     async def run_file_response_benchmark() -> None:
-        client = await aiohttp_client(app)
+        client = await aiohttp_client_sync(app, server_kwargs=conn_type.s_kwargs)
         for _ in range(response_count):
-            await client.get("/")
+            await client.get("/", **conn_type.c_kwargs)
         await client.close()
 
     @benchmark
@@ -77,8 +106,9 @@ def _run() -> None:
 
 def test_simple_web_file_sendfile_fallback_response(
     event_loop: asyncio.AbstractEventLoop,
-    aiohttp_client: AiohttpClient,
+    aiohttp_client_sync: AiohttpClient,
     benchmark: BenchmarkFixture,
+    conn_type: ConnectionType,
 ) -> None:
     """Benchmark creating 100 simple web.FileResponse without sendfile."""
     response_count = 100
@@ -94,9 +124,9 @@ async def handler(request: web.Request) -> web.FileResponse:
     app.router.add_route("GET", "/", handler)
 
     async def run_file_response_benchmark() -> None:
-        client = await aiohttp_client(app)
+        client = await aiohttp_client_sync(app, server_kwargs=conn_type.s_kwargs)
         for _ in range(response_count):
-            await client.get("/")
+            await client.get("/", **conn_type.c_kwargs)
         await client.close()
 
     @benchmark