Merge remote-tracking branch 'upstream/master' into pr-osparc-docker-socket-via-http-interface

Author: Andrei Neagu

.github/workflows/ci-testing-pull-request.yml

@@ -31,13 +31,15 @@ jobs:
           enable-cache: false
       - name: checkout source branch
         uses: actions/checkout@v4
-      - name: Regenerate specs and check
+      - name: Generate openapi specs
         run: |
           make devenv
           source .venv/bin/activate
           make openapi-specs
+      - name: Check openapi specs are up to date
+        run: |
           if ! ./ci/github/helpers/openapi-specs-diff.bash diff \
-            https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/refs/heads/${{ github.event.pull_request.head.ref }} \
+            https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.after }} \
             .; then \
             echo "::error:: OAS are not up to date. Run 'make openapi-specs' to update them"; exit 1; \
           fi
@@ -57,7 +59,7 @@ jobs:
       - name: check api-server backwards compatibility
         run: |
           ./scripts/openapi-diff.bash breaking --fail-on ERR\
-            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.event.pull_request.base.ref }}/services/api-server/openapi.json \
+            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/${{ github.event.after }}/services/api-server/openapi.json \
             /specs/services/api-server/openapi.json
 
   all-oas-breaking:
@@ -76,5 +78,5 @@ jobs:
       - name: Check openapi-specs backwards compatibility
         run: |
           ./ci/github/helpers/openapi-specs-diff.bash breaking \
-            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/refs/heads/${{ github.event.pull_request.base.ref }} \
+            https://raw.githubusercontent.com/${{ github.event.pull_request.base.repo.full_name }}/${{ github.event.after }} \
             .

packages/service-integration/README.md

@@ -5,3 +5,11 @@ This is the o2sparc's service integration library or ``ooil`` in short
 
 
 SEE how is it used in Makefiles in https://github.com/ITISFoundation/cookiecutter-osparc-service
+
+
+#### What is the .osparc folder and its content?
+'osparc config' is a set of stardard file forms (yaml) that the user fills provides in order to describe how her service works and integrates with osparc. It may contain:
+  - config files are stored under '.osparc/' folder in the root repo folder (analogous to other configs like .github, .vscode, etc)
+  - configs are parsed and validated into pydantic models
+  - models can be serialized/deserialized into label annotations on images. This way, the config is attached to the service during it's entire lifetime.
+  - config should provide enough information about that context to allow building an image and running a container on a single command call.

packages/service-integration/src/service_integration/cli/_compose_spec.py

@@ -192,6 +192,14 @@ def create_compose(
             if file_path.exists():
                 configs_kwargs_map[config_name][arg_name] = file_path
 
+    # warn about subfolders without metadata.yml
+    for subdir in filter(lambda p: p.is_dir(), basedir.rglob("*")):
+        if not (subdir / "metadata.yml").exists():
+            relative_subdir = subdir.relative_to(basedir)
+            rich.print(
+                f"[warning] Subfolder '{relative_subdir}' does not contain a 'metadata.yml' file. Skipping."
+            )
+
     if not configs_kwargs_map:
         rich.print(f"[warning] No config files were found in '{config_path}'")
 

packages/service-library/requirements/_fastapi.in

@@ -7,6 +7,7 @@
 
 
 fastapi
+fastapi-lifespan-manager
 httpx
 opentelemetry-instrumentation-fastapi
 opentelemetry-instrumentation-httpx

packages/service-library/requirements/_fastapi.txt

@@ -23,6 +23,10 @@ deprecated==1.2.17
     #   opentelemetry-api
     #   opentelemetry-semantic-conventions
 fastapi==0.115.7
+    # via
+    #   -r requirements/_fastapi.in
+    #   fastapi-lifespan-manager
+fastapi-lifespan-manager==0.1.4
     # via -r requirements/_fastapi.in
 h11==0.14.0
     # via

packages/service-library/src/servicelib/docker_utils.py

@@ -1,18 +1,19 @@
 import asyncio
 import logging
 from collections.abc import AsyncIterator, Awaitable, Callable
-from contextlib import AsyncExitStack, asynccontextmanager
+from contextlib import AsyncExitStack
 from dataclasses import dataclass
 from datetime import datetime
 from functools import cached_property
-from typing import Any, AsyncContextManager, Final, Literal
+from typing import Any, Final, Literal
 
 import aiodocker
 import aiohttp
 import arrow
 import tenacity
 from aiohttp import ClientSession
 from fastapi import FastAPI
+from fastapi_lifespan_manager import State
 from models_library.docker import DockerGenericTag
 from models_library.generated_models.docker_rest_api import ProgressDetail
 from models_library.utils.change_case import snake_to_camel
@@ -24,6 +25,7 @@
     TypeAdapter,
     ValidationError,
 )
+from servicelib.fastapi.lifespan_utils import LifespanGenerator
 from settings_library.docker_api_proxy import DockerApiProxysettings
 from settings_library.docker_registry import RegistrySettings
 from yarl import URL
@@ -300,7 +302,7 @@ async def pull_image(
 
 def get_lifespan_remote_docker_client(
     docker_api_proxy_settings_property_name: str,
-) -> Callable[[FastAPI], AsyncContextManager[None]]:
+) -> LifespanGenerator:
     """Ensures `setup` and `teardown` for the remote docker client.
 
     Arguments:
@@ -311,8 +313,7 @@ def get_lifespan_remote_docker_client(
         docker client lifespan manager
     """
 
-    @asynccontextmanager
-    async def _(app: FastAPI) -> AsyncIterator[None]:
+    async def _(app: FastAPI) -> AsyncIterator[State]:
         settings: DockerApiProxysettings = getattr(
             app.state.settings, docker_api_proxy_settings_property_name
         )
@@ -345,7 +346,7 @@ async def _(app: FastAPI) -> AsyncIterator[None]:
 
             await wait_till_docker_api_proxy_is_responsive(app)
 
-            yield
+            yield {}
 
     return _
 

packages/service-library/src/servicelib/fastapi/lifespan_utils.py

@@ -1,24 +1,20 @@
-from collections.abc import AsyncGenerator, Callable
-from contextlib import AsyncExitStack, asynccontextmanager
-from typing import AsyncContextManager, TypeAlias
+from collections.abc import AsyncIterator
+from typing import Protocol
 
 from fastapi import FastAPI
+from fastapi_lifespan_manager import LifespanManager, State
 
-LifespanContextManager: TypeAlias = Callable[[FastAPI], AsyncContextManager[None]]
 
+class LifespanGenerator(Protocol):
+    def __call__(self, app: FastAPI) -> AsyncIterator["State"]:
+        ...
 
-def combine_lfiespan_context_managers(
-    *context_managers: LifespanContextManager,
-) -> LifespanContextManager:
-    """the first entry has its `setup` called first and its `teardown` called last
-    With `setup` and `teardown` referring to the code before and after the `yield`
-    """
 
-    @asynccontextmanager
-    async def _(app: FastAPI) -> AsyncGenerator[None, None]:
-        async with AsyncExitStack() as stack:
-            for context_manager in context_managers:
-                await stack.enter_async_context(context_manager(app))
-            yield
+def combine_lifespans(*generators: LifespanGenerator) -> LifespanManager:
 
-    return _
+    manager = LifespanManager()
+
+    for generator in generators:
+        manager.add(generator)
+
+    return manager

packages/service-library/src/servicelib/fastapi/profiler_middleware.py renamed to packages/service-library/src/servicelib/fastapi/profiler.py

@@ -1,5 +1,6 @@
 from typing import Any, Final
 
+from fastapi import FastAPI
 from servicelib.aiohttp import status
 from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON
 from starlette.requests import Request
@@ -13,7 +14,7 @@
 )
 
 
-def is_last_response(response_headers: dict[bytes, bytes], message: dict[str, Any]):
+def _is_last_response(response_headers: dict[bytes, bytes], message: dict[str, Any]):
     if (
         content_type := response_headers.get(b"content-type")
     ) and content_type == MIMETYPE_APPLICATION_JSON.encode():
@@ -79,7 +80,7 @@ async def _send_wrapper(message):
                         response_headers = dict(message.get("headers"))
                         message["headers"] = check_response_headers(response_headers)
                     elif message["type"] == "http.response.body":
-                        if is_last_response(response_headers, message):
+                        if _is_last_response(response_headers, message):
                             _profiler.stop()
                             profile_text = _profiler.output_text(
                                 unicode=True, color=True, show_all=True
@@ -96,3 +97,8 @@ async def _send_wrapper(message):
 
         finally:
             _profiler.reset()
+
+
+def initialize_profiler(app: FastAPI) -> None:
+    # NOTE: this cannot be ran once the application is started
+    app.add_middleware(ProfilerMiddleware)

packages/service-library/src/servicelib/fastapi/prometheus_instrumentation.py

@@ -1,28 +1,62 @@
 # pylint: disable=protected-access
 
 
+from collections.abc import AsyncIterator
+
 from fastapi import FastAPI
+from fastapi_lifespan_manager import State
 from prometheus_client import CollectorRegistry
 from prometheus_fastapi_instrumentator import Instrumentator
 
 
-def setup_prometheus_instrumentation(app: FastAPI) -> Instrumentator:
+def initialize_prometheus_instrumentation(app: FastAPI) -> None:
+    # NOTE: this cannot be ran once the application is started
+
     # NOTE: use that registry to prevent having a global one
     app.state.prometheus_registry = registry = CollectorRegistry(auto_describe=True)
-    instrumentator = Instrumentator(
+    app.state.prometheus_instrumentator = Instrumentator(
         should_instrument_requests_inprogress=False,  # bug in https://github.com/trallnag/prometheus-fastapi-instrumentator/issues/317
         inprogress_labels=False,
         registry=registry,
-    ).instrument(app)
+    )
+    app.state.prometheus_instrumentator.instrument(app)
+
+
+def _startup(app: FastAPI) -> None:
+    assert isinstance(app.state.prometheus_instrumentator, Instrumentator)  # nosec
+    app.state.prometheus_instrumentator.expose(app, include_in_schema=False)
+
+
+def _shutdown(app: FastAPI) -> None:
+    assert isinstance(app.state.prometheus_registry, CollectorRegistry)  # nosec
+    registry = app.state.prometheus_registry
+    for collector in list(registry._collector_to_names.keys()):  # noqa: SLF001
+        registry.unregister(collector)
+
+
+def get_prometheus_instrumentator(app: FastAPI) -> Instrumentator:
+    assert isinstance(app.state.prometheus_instrumentator, Instrumentator)  # nosec
+    return app.state.prometheus_instrumentator
+
+
+def setup_prometheus_instrumentation(app: FastAPI) -> Instrumentator:
+    initialize_prometheus_instrumentation(app)
 
     async def _on_startup() -> None:
-        instrumentator.expose(app, include_in_schema=False)
+        _startup(app)
 
-    def _unregister() -> None:
-        # NOTE: avoid registering collectors multiple times when running unittests consecutively (https://stackoverflow.com/a/62489287)
-        for collector in list(registry._collector_to_names.keys()):  # noqa: SLF001
-            registry.unregister(collector)
+    def _on_shutdown() -> None:
+        _shutdown(app)
 
     app.add_event_handler("startup", _on_startup)
-    app.add_event_handler("shutdown", _unregister)
-    return instrumentator
+    app.add_event_handler("shutdown", _on_shutdown)
+
+    return get_prometheus_instrumentator(app)
+
+
+async def lifespan_prometheus_instrumentation(app: FastAPI) -> AsyncIterator[State]:
+    # NOTE: requires ``initialize_prometheus_instrumentation`` to be called before the
+    # lifespan of the applicaiton runs, usually rigth after the ``FastAPI`` instance is created
+    _startup(app)
+    yield {}
+    _shutdown(app)

packages/service-library/src/servicelib/fastapi/tracing.py

@@ -3,8 +3,10 @@
 """
 
 import logging
+from collections.abc import AsyncIterator
 
 from fastapi import FastAPI
+from fastapi_lifespan_manager import State
 from httpx import AsyncClient, Client
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
@@ -15,6 +17,7 @@
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from servicelib.fastapi.lifespan_utils import LifespanGenerator
 from servicelib.logging_utils import log_context
 from settings_library.tracing import TracingSettings
 from yarl import URL
@@ -131,5 +134,15 @@ def setup_tracing(
             RequestsInstrumentor().instrument()
 
 
+def get_lifespan_tracing(
+    tracing_settings: TracingSettings, service_name: str
+) -> LifespanGenerator:
+    async def _(app: FastAPI) -> AsyncIterator[State]:
+        setup_tracing(app, tracing_settings, service_name)
+        yield {}
+
+    return _
+
+
 def setup_httpx_client_tracing(client: AsyncClient | Client):
     HTTPXClientInstrumentor.instrument_client(client)