adding docker network creation interface

Author: Andrei Neagu

packages/models-library/src/models_library/docker.py

@@ -21,6 +21,8 @@
 from .projects_nodes_io import NodeID
 from .users import UserID
 
+DockerNetworkID: TypeAlias = str
+
 
 class DockerLabelKey(ConstrainedStr):
     # NOTE: https://docs.docker.com/config/labels-custom-metadata/#key-format-recommendations
@@ -37,7 +39,15 @@ def from_key(cls, key: str) -> "DockerLabelKey":
     str, StringConstraints(pattern=DOCKER_GENERIC_TAG_KEY_RE)
 ]
 
-DockerPlacementConstraint: TypeAlias = Annotated[str, StringConstraints(strip_whitespace = True, pattern = re.compile(r"^(?!-)(?![.])(?!.*--)(?!.*[.][.])[a-zA-Z0-9.-]*(?<!-)(?<![.])(!=|==)[a-zA-Z0-9_. -]*$"))]
+DockerPlacementConstraint: TypeAlias = Annotated[
+    str,
+    StringConstraints(
+        strip_whitespace=True,
+        pattern=re.compile(
+            r"^(?!-)(?![.])(?!.*--)(?!.*[.][.])[a-zA-Z0-9.-]*(?<!-)(?<![.])(!=|==)[a-zA-Z0-9_. -]*$"
+        ),
+    ),
+]
 
 _SIMCORE_RUNTIME_DOCKER_LABEL_PREFIX: Final[str] = "io.simcore.runtime."
 _BACKWARDS_COMPATIBILITY_SIMCORE_RUNTIME_DOCKER_LABELS_MAP: Final[dict[str, str]] = {

services/director-v2/src/simcore_service_director_v2/api/dependencies/docker.py

@@ -0,0 +1,8 @@
+from aiodocker import Docker
+from fastapi import Request
+
+from ...modules.docker_client import SharedDockerClient
+
+
+def get_shared_docker_client(request: Request) -> Docker:
+    return SharedDockerClient.docker_instance(request.app)

services/director-v2/src/simcore_service_director_v2/api/entrypoints.py

@@ -4,6 +4,7 @@
 from .routes import (
     computations,
     computations_tasks,
+    docker_networks,
     dynamic_scheduler,
     dynamic_services,
     health,
@@ -26,6 +27,9 @@
 v2_router.include_router(
     dynamic_services.router, tags=["dynamic services"], prefix="/dynamic_services"
 )
+v2_router.include_router(
+    docker_networks.router, tags=["docker networks"], prefix="/docker/networks"
+)
 
 v2_router.include_router(
     dynamic_scheduler.router, tags=["dynamic scheduler"], prefix="/dynamic_scheduler"

services/director-v2/src/simcore_service_director_v2/api/routes/docker_networks.py

@@ -0,0 +1,38 @@
+from typing import Annotated
+
+from aiodocker import Docker
+from fastapi import APIRouter, Depends, status
+from models_library.docker import DockerNetworkID
+from models_library.generated_models.docker_rest_api import Network
+
+from ..dependencies.docker import get_shared_docker_client
+
+router = APIRouter()
+
+
+@router.post(
+    "/",
+    summary="create a docker network given the input parameters",
+    status_code=status.HTTP_200_OK,
+)
+async def create_docker_network(
+    docker_network: Network,
+    shared_docker_client: Annotated[Docker, Depends(get_shared_docker_client)],
+) -> DockerNetworkID:
+    created_network = await shared_docker_client.networks.create(
+        docker_network.model_dump(mode="json")
+    )
+    return created_network.id
+
+
+@router.delete(
+    "/{network_id}",
+    summary="removes an existing docker network",
+    status_code=status.HTTP_204_NO_CONTENT,
+)
+async def remove_docker_network(
+    network_id: DockerNetworkID,
+    shared_docker_client: Annotated[Docker, Depends(get_shared_docker_client)],
+):
+    created_network = await shared_docker_client.networks.get(network_id)
+    await created_network.delete()

services/director-v2/src/simcore_service_director_v2/core/application.py

@@ -23,6 +23,7 @@
     dask_clients_pool,
     db,
     director_v0,
+    docker_client,
     dynamic_services,
     dynamic_sidecar,
     instrumentation,
@@ -206,6 +207,8 @@ def init_app(settings: AppSettings | None = None) -> FastAPI:
     if settings.DIRECTOR_V2_PROFILING:
         app.add_middleware(ProfilerMiddleware)
 
+    docker_client.setup(app)
+
     # setup app --
     app.add_event_handler("startup", on_startup)
     app.add_event_handler("shutdown", on_shutdown)

services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/__init__.py

@@ -13,7 +13,9 @@ class DynamicServicesSettings(BaseCustomSettings):
         default=True, description="Enables/Disables the dynamic_sidecar submodule"
     )
 
-    DYNAMIC_SIDECAR: DynamicSidecarSettings = Field(json_schema_extra={"auto_default_from_env": True})
+    DYNAMIC_SIDECAR: DynamicSidecarSettings = Field(
+        json_schema_extra={"auto_default_from_env": True}
+    )
 
     DYNAMIC_SCHEDULER: DynamicServicesSchedulerSettings = Field(
         json_schema_extra={"auto_default_from_env": True}
@@ -31,4 +33,6 @@ class DynamicServicesSettings(BaseCustomSettings):
         json_schema_extra={"auto_default_from_env": True}
     )
 
-    WEBSERVER_SETTINGS: WebServerSettings = Field(json_schema_extra={"auto_default_from_env": True})
+    WEBSERVER_SETTINGS: WebServerSettings = Field(
+        json_schema_extra={"auto_default_from_env": True}
+    )

services/director-v2/src/simcore_service_director_v2/modules/docker_client.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+
+import aiodocker
+from aiodocker import Docker
+from fastapi import FastAPI
+from servicelib.fastapi.app_state import SingletonInAppStateMixin
+
+
+@dataclass
+class SharedDockerClient(SingletonInAppStateMixin):
+    app_state_name: str = "shared_docker_client"
+
+    docker_client: Docker | None = None
+
+    @classmethod
+    def docker_instance(cls, app: FastAPI) -> Docker:
+        docker_client = cls.get_from_app_state(app).docker_client
+        assert docker_client is not None  # nosec
+        return docker_client
+
+    async def setup(self) -> None:
+        self.docker_client = aiodocker.Docker()
+
+    async def shutdown(self) -> None:
+        if self.docker_client is not None:
+            await self.docker_client.close()
+
+
+def setup(app: FastAPI) -> None:
+    async def on_startup() -> None:
+        shared_client = SharedDockerClient()
+        await shared_client.setup()
+        shared_client.set_to_app_state(app)
+
+    async def on_shutdown() -> None:
+        shared_client = SharedDockerClient.pop_from_app_state(app)
+        await shared_client.shutdown()
+
+    app.add_event_handler("startup", on_startup)
+    app.add_event_handler("shutdown", on_shutdown)

services/director-v2/tests/unit/test_api_routes_docker_networks.py

@@ -0,0 +1,65 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+
+from collections.abc import AsyncIterable
+
+import pytest
+from aiodocker import Docker, DockerError
+from faker import Faker
+from fastapi import status
+from models_library.docker import DockerNetworkID
+from models_library.generated_models.docker_rest_api import Network
+from pytest_simcore.helpers.typing_env import EnvVarsDict
+from starlette.testclient import TestClient
+
+
+@pytest.fixture
+def mock_env(
+    mock_exclusive: None,
+    disable_rabbitmq: None,
+    disable_postgres: None,
+    mock_env: EnvVarsDict,
+    monkeypatch: pytest.MonkeyPatch,
+    faker: Faker,
+) -> None:
+    monkeypatch.setenv("DIRECTOR_V2_DOCKER_ENTRYPOINT_ACCESS_TOKEN", "adminadmin")
+
+    monkeypatch.setenv("SC_BOOT_MODE", "default")
+    monkeypatch.setenv("DIRECTOR_ENABLED", "false")
+    monkeypatch.setenv("COMPUTATIONAL_BACKEND_ENABLED", "false")
+    monkeypatch.setenv("COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED", "false")
+
+    monkeypatch.setenv("DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED", "false")
+
+    monkeypatch.setenv("R_CLONE_PROVIDER", "MINIO")
+    monkeypatch.setenv("S3_ENDPOINT", faker.url())
+    monkeypatch.setenv("S3_ACCESS_KEY", faker.pystr())
+    monkeypatch.setenv("S3_REGION", faker.pystr())
+    monkeypatch.setenv("S3_SECRET_KEY", faker.pystr())
+    monkeypatch.setenv("S3_BUCKET_NAME", faker.pystr())
+
+
+@pytest.fixture
+async def docker_client() -> AsyncIterable[Docker]:
+    async with Docker() as client:
+        yield client
+
+
+async def test_routes_are_protected(client: TestClient, docker_client: Docker):
+    network_name = "a_test_network"
+    network = Network(name=network_name)
+
+    response = client.post("/v2/docker/networks/", json=network.model_dump(mode="json"))
+    assert response.status_code == status.HTTP_200_OK, response.text
+    network_id: DockerNetworkID = response.json()
+
+    response = client.delete(f"/v2/docker/networks/{network_id}")
+    assert response.status_code == status.HTTP_204_NO_CONTENT, response.text
+
+    # check it is not here
+
+    for name_or_id in (network_name, network_id):
+        with pytest.raises(DockerError) as exc:
+            await docker_client.networks.get(name_or_id)
+
+        assert exc.value.status == status.HTTP_404_NOT_FOUND