using new docker client

Author: Andrei Neagu

packages/service-library/src/servicelib/docker_utils.py

@@ -1,6 +1,6 @@
 import logging
-from collections.abc import Awaitable, Callable
-from contextlib import AsyncExitStack
+from collections.abc import AsyncIterator, Awaitable, Callable
+from contextlib import AsyncExitStack, asynccontextmanager
 from dataclasses import dataclass
 from datetime import datetime
 from functools import cached_property
@@ -10,6 +10,7 @@
 import aiohttp
 import arrow
 from aiohttp import ClientSession
+from fastapi import FastAPI
 from models_library.docker import DockerGenericTag
 from models_library.generated_models.docker_rest_api import ProgressDetail
 from models_library.utils.change_case import snake_to_camel
@@ -286,14 +287,15 @@ async def pull_image(
             )
 
 
-async def create_remote_docker_client(
-    settings: DockerApiProxysettings,
-) -> aiodocker.Docker:
-    """Allows to attach to a remote docker client istnace and use it in place
-    of the docker socket.
+@asynccontextmanager
+async def lifespan_remote_docker_client(app: FastAPI) -> AsyncIterator[None]:
+    """Ensures `setup` and `teardown` for the docker client.
 
-    NOTE: Usually this is required to use a manager node's docker client for swarm commands.
+    NOTE: the `DockerApiProxysettings` must be placed as `DOCKER_API_PROXY`
+    on the Application's Settings object
     """
+    settings: DockerApiProxysettings = app.state.settings.DOCKER_API_PROXY
+
     session: ClientSession | None = None
     if settings.DOCKER_API_PROXY_USER and settings.DOCKER_API_PROXY_PASSWORD:
         session = ClientSession(
@@ -303,4 +305,26 @@ async def create_remote_docker_client(
             )
         )
 
-    return aiodocker.Docker(url=settings.base_url, session=session)
+    async with AsyncExitStack() as exit_stack:
+        if settings.DOCKER_API_PROXY_USER and settings.DOCKER_API_PROXY_PASSWORD:
+            await exit_stack.enter_async_context(
+                ClientSession(
+                    auth=aiohttp.BasicAuth(
+                        login=settings.DOCKER_API_PROXY_USER,
+                        password=settings.DOCKER_API_PROXY_PASSWORD.get_secret_value(),
+                    )
+                )
+            )
+
+        client = await exit_stack.enter_async_context(
+            aiodocker.Docker(url=settings.base_url, session=session)
+        )
+
+        app.state.remote_docker_client = client
+
+        yield
+
+
+def get_remote_docker_client(app: FastAPI) -> aiodocker.Docker:
+    client: aiodocker.Docker = app.state.remote_docker_client
+    return client

services/docker-api-proxy/requirements/_test.in

@@ -6,8 +6,11 @@
 --requirement ../../../packages/service-library/requirements/_base.in
 
 aiodocker
+arrow
+asgi_lifespan
 docker
 faker
+fastapi
 flaky
 pytest
 pytest-asyncio
@@ -16,4 +19,3 @@ pytest-mock
 python-dotenv
 PyYAML
 tenacity
-arrow

services/docker-api-proxy/requirements/_test.txt

@@ -37,13 +37,16 @@ anyio==4.8.0
     # via
     #   fast-depends
     #   faststream
+    #   starlette
 arrow==1.3.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   -r requirements/_test.in
-attrs==24.3.0
+asgi-lifespan==2.1.0
+    # via -r requirements/_test.in
+attrs==25.1.0
     # via
     #   aiohttp
     #   jsonschema
@@ -69,7 +72,7 @@ click==8.1.8
     # via typer
 coverage==7.6.10
     # via pytest-cov
-deprecated==1.2.15
+deprecated==1.2.18
     # via
     #   opentelemetry-api
     #   opentelemetry-exporter-otlp-proto-grpc
@@ -87,6 +90,8 @@ faker==35.0.0
     # via -r requirements/_test.in
 fast-depends==2.4.12
     # via faststream
+fastapi==0.115.7
+    # via -r requirements/_test.in
 faststream==0.5.34
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 flaky==3.8.1
@@ -99,7 +104,7 @@ googleapis-common-protos==1.66.0
     # via
     #   opentelemetry-exporter-otlp-proto-grpc
     #   opentelemetry-exporter-otlp-proto-http
-grpcio==1.69.0
+grpcio==1.70.0
     # via opentelemetry-exporter-otlp-proto-grpc
 idna==3.10
     # via
@@ -215,7 +220,7 @@ protobuf==5.29.3
     #   opentelemetry-proto
 psutil==6.1.1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
-pydantic==2.10.5
+pydantic==2.10.6
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -241,6 +246,7 @@ pydantic==2.10.5
     #   -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
     #   fast-depends
+    #   fastapi
     #   pydantic-extra-types
     #   pydantic-settings
 pydantic-core==2.27.2
@@ -329,8 +335,20 @@ redis==5.2.1
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/service-library/requirements/_base.in
-referencing==0.36.1
+referencing==0.35.1
     # via
+    #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
     #   jsonschema
     #   jsonschema-specifications
 requests==2.32.3
@@ -351,7 +369,24 @@ shellingham==1.5.4
 six==1.17.0
     # via python-dateutil
 sniffio==1.3.1
-    # via anyio
+    # via
+    #   anyio
+    #   asgi-lifespan
+starlette==0.45.3
+    # via
+    #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   fastapi
 tenacity==9.0.0
     # via
     #   -r requirements/../../../packages/service-library/requirements/_base.in
@@ -371,12 +406,12 @@ typing-extensions==4.12.2
     #   aiodebug
     #   anyio
     #   faker
+    #   fastapi
     #   faststream
     #   opentelemetry-sdk
     #   pydantic
     #   pydantic-core
     #   pydantic-extra-types
-    #   referencing
     #   typer
 urllib3==2.3.0
     # via

services/docker-api-proxy/requirements/_tools.txt

@@ -1,6 +1,6 @@
 astroid==3.3.8
     # via pylint
-black==24.10.0
+black==25.1.0
     # via -r requirements/../../../requirements/devenv.txt
 build==1.2.2.post1
     # via pip-tools
@@ -21,7 +21,7 @@ filelock==3.17.0
     # via virtualenv
 identify==2.6.6
     # via pre-commit
-isort==5.13.2
+isort==6.0.0
     # via
     #   -r requirements/../../../requirements/devenv.txt
     #   pylint
@@ -42,7 +42,7 @@ packaging==24.2
     #   build
 pathspec==0.12.1
     # via black
-pip==24.3.1
+pip==25.0
     # via pip-tools
 pip-tools==7.4.1
     # via -r requirements/../../../requirements/devenv.txt
@@ -53,7 +53,7 @@ platformdirs==4.3.6
     #   virtualenv
 pre-commit==4.1.0
     # via -r requirements/../../../requirements/devenv.txt
-pylint==3.3.3
+pylint==3.3.4
     # via -r requirements/../../../requirements/devenv.txt
 pyproject-hooks==1.2.0
     # via
@@ -64,7 +64,7 @@ pyyaml==6.0.2
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_test.txt
     #   pre-commit
-ruff==0.9.2
+ruff==0.9.3
     # via -r requirements/../../../requirements/devenv.txt
 setuptools==75.8.0
     # via pip-tools

services/docker-api-proxy/tests/integration/conftest.py

@@ -1,5 +1,23 @@
 # pylint:disable=unrecognized-options
 
+from collections.abc import AsyncIterator, Callable
+from contextlib import AbstractAsyncContextManager, asynccontextmanager
+from typing import Annotated
+
+import aiodocker
+import pytest
+from asgi_lifespan import LifespanManager
+from fastapi import FastAPI
+from pydantic import Field
+from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict
+from servicelib.docker_utils import (
+    get_remote_docker_client,
+    lifespan_remote_docker_client,
+)
+from servicelib.fastapi.lifespan_utils import combine_lfiespan_context_managers
+from settings_library.application import BaseApplicationSettings
+from settings_library.docker_api_proxy import DockerApiProxysettings
+
 pytest_plugins = [
     "pytest_simcore.docker_api_proxy",
     "pytest_simcore.docker_compose",
@@ -12,3 +30,34 @@
 def pytest_configure(config):
     # Set asyncio_mode to "auto"
     config.option.asyncio_mode = "auto"
+
+
+def _get_test_app() -> FastAPI:
+    class ApplicationSetting(BaseApplicationSettings):
+        DOCKER_API_PROXY: Annotated[
+            DockerApiProxysettings,
+            Field(json_schema_extra={"auto_default_from_env": True}),
+        ]
+
+    app = FastAPI(
+        lifespan=combine_lfiespan_context_managers(lifespan_remote_docker_client)
+    )
+    app.state.settings = ApplicationSetting.create_from_envs()
+
+    return app
+
+
+@pytest.fixture
+async def setup_docker_client(
+    monkeypatch: pytest.MonkeyPatch,
+) -> Callable[[EnvVarsDict], AbstractAsyncContextManager[aiodocker.Docker]]:
+    @asynccontextmanager
+    async def _(env_vars: EnvVarsDict) -> AsyncIterator[aiodocker.Docker]:
+        setenvs_from_dict(monkeypatch, env_vars)
+
+        app = _get_test_app()
+
+        async with LifespanManager(app):
+            yield get_remote_docker_client(app)
+
+    return _

services/docker-api-proxy/tests/integration/test_docker_api_proxy.py

@@ -1,8 +1,13 @@
+# pylint: disable=unused-argument
+
 import json
 import sys
+from collections.abc import Callable
+from contextlib import AbstractAsyncContextManager
 from pathlib import Path
 
-from servicelib.docker_utils import create_remote_docker_client
+import aiodocker
+from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict
 from settings_library.docker_api_proxy import DockerApiProxysettings
 
 HERE = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
@@ -13,9 +18,16 @@
 
 
 async def test_unauthenticated_docker_client(
+    docker_swarm: None,
     docker_api_proxy_settings: DockerApiProxysettings,
+    setup_docker_client: Callable[
+        [EnvVarsDict], AbstractAsyncContextManager[aiodocker.Docker]
+    ],
 ):
-    docker = await create_remote_docker_client(docker_api_proxy_settings)
-    async with docker:
-        info = await docker.system.info()
+    envs = {
+        "DOCKER_API_PROXY_HOST": "127.0.0.1",
+        "DOCKER_API_PROXY_PORT": "8014",
+    }
+    async with setup_docker_client(envs) as working_docker:
+        info = await working_docker.system.info()
         print(json.dumps(info, indent=2))