✨Adding dynamic services monitoring dashboard (⚠️devops)

.env-devel

@@ -128,6 +128,7 @@ DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
 DYNAMIC_SCHEDULER_PROFILING=1
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
 DYNAMIC_SCHEDULER_TRACING={}
+DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin
 
 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'
 

Makefile

@@ -323,6 +323,7 @@ printf "$$rows" "oSparc platform" "http://$(get_my_ip).nip.io:9081";\
 printf "$$rows" "oSparc public API doc" "http://$(get_my_ip).nip.io:8006/dev/doc";\
 printf "$$rows" "oSparc web API doc" "http://$(get_my_ip).nip.io:9081/dev/doc";\
 printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\
+printf "$$rows" "Dy-scheduler Dashboard" "http://$(get_my_ip).nip.io:8012";\
 printf "$$rows" "Docker Registry" "http://$${REGISTRY_URL}/v2/_catalog" $${REGISTRY_USER} $${REGISTRY_PW};\
 printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVITATIONS_USERNAME} $${INVITATIONS_PASSWORD};\
 printf "$$rows" "Jaeger" "http://$(get_my_ip).nip.io:16686";\

packages/models-library/tests/test_service_settings_labels.py

@@ -9,6 +9,7 @@
 from typing import Any, Final, NamedTuple
 
 import pytest
+from common_library.json_serialization import json_dumps
 from models_library.basic_types import PortInt
 from models_library.osparc_variable_identifier import (
     OsparcVariableIdentifier,
@@ -32,7 +33,6 @@
 from models_library.services_resources import DEFAULT_SINGLE_SERVICE_NAME
 from models_library.utils.string_substitution import TextTemplate
 from pydantic import BaseModel, TypeAdapter, ValidationError
-from pydantic.json import pydantic_encoder
 
 
 class _Parametrization(NamedTuple):
@@ -558,11 +558,6 @@ def test_can_parse_labels_with_osparc_identifiers(
     assert "$" not in service_meta_str
 
 
-def servicelib__json_serialization__json_dumps(obj: Any, **kwargs):
-    # Analogous to 'models_library.utils.json_serialization.json_dumps'
-    return json.dumps(obj, default=pydantic_encoder, **kwargs)
-
-
 def test_resolving_some_service_labels_at_load_time(
     vendor_environments: dict[str, Any], service_labels: dict[str, str]
 ):
@@ -579,9 +574,7 @@ def test_resolving_some_service_labels_at_load_time(
         ("settings", SimcoreServiceSettingsLabel),
     ):
         to_serialize = getattr(service_meta, attribute_name)
-        template = TextTemplate(
-            servicelib__json_serialization__json_dumps(to_serialize)
-        )
+        template = TextTemplate(json_dumps(to_serialize))
         assert template.is_valid()
         resolved_label: str = template.safe_substitute(vendor_environments)
         to_restore = TypeAdapter(pydantic_model).validate_json(resolved_label)

packages/models-library/tests/test_utils_fastapi_encoders.py

@@ -4,36 +4,25 @@
 # pylint: disable=too-many-arguments
 
 import json
-from typing import Any
 from uuid import uuid4
 
-import pytest
+from common_library.json_serialization import json_dumps
 from faker import Faker
 from models_library.utils.fastapi_encoders import servicelib_jsonable_encoder
-from pydantic.json import pydantic_encoder
-
-
-def servicelib__json_serialization__json_dumps(obj: Any, **kwargs):
-    # Analogous to 'models_library.utils.json_serialization.json_dumps'
-    return json.dumps(obj, default=pydantic_encoder, **kwargs)
 
 
 def test_using_uuids_as_keys(faker: Faker):
     uuid_key = uuid4()
 
-    with pytest.raises(TypeError):
-        # IMPORTANT NOTE: we cannot serialize UUID objects as keys.
-        # We have to convert them to strings but then the class information is lost upon deserialization i.e. it is not reversable!
-        # NOTE: This could  potentially be solved using 'orjson' !!
-        #
-        servicelib__json_serialization__json_dumps({uuid_key: "value"}, indent=1)
+    # this was previously failing
+    assert json_dumps({uuid_key: "value"}, indent=1)
 
-    # use encoder
+    # uuid keys now serialize without raising to the expected format string
     data = servicelib_jsonable_encoder({uuid_key: "value"})
     assert data == {f"{uuid_key}": "value"}
 
     # serialize w/o raising
-    dumped_data = servicelib__json_serialization__json_dumps(data, indent=1)
+    dumped_data = json_dumps(data, indent=1)
 
     # deserialize w/o raising
     loaded_data = json.loads(dumped_data)

services/docker-compose.yml

@@ -567,6 +567,7 @@ services:
       DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT}
       DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING}
       DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING}
+      DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET}
       TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}
       TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT}
   static-webserver:

services/dynamic-scheduler/Dockerfile

@@ -146,7 +146,7 @@ HEALTHCHECK --interval=30s \
   --timeout=20s \
   --start-period=30s \
   --retries=3 \
-  CMD ["python3", "services/dynamic-scheduler/docker/healthcheck.py", "http://localhost:8000/"]
+  CMD ["python3", "services/dynamic-scheduler/docker/healthcheck.py", "http://localhost:8000/health"]
 
 ENTRYPOINT [ "/bin/sh", "services/dynamic-scheduler/docker/entrypoint.sh" ]
 CMD ["/bin/sh", "services/dynamic-scheduler/docker/boot.sh"]

services/dynamic-scheduler/requirements/_base.in

@@ -18,6 +18,7 @@
 arrow
 fastapi
 httpx
+nicegui
 packaging
 python-socketio
 typer[all]

services/dynamic-scheduler/requirements/_base.txt

@@ -7,7 +7,9 @@ aiodebug==2.3.0
 aiodocker==0.23.0
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 aiofiles==24.1.0
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
+    # via
+    #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   nicegui
 aiohappyeyeballs==2.4.3
     # via aiohttp
 aiohttp==3.10.10
@@ -27,6 +29,8 @@ aiohttp==3.10.10
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   aiodocker
+    #   nicegui
+    #   python-socketio
 aiormq==6.8.1
     # via aio-pika
 aiosignal==1.3.1
@@ -79,6 +83,7 @@ certifi==2024.8.30
     #   -c requirements/../../../requirements/constraints.txt
     #   httpcore
     #   httpx
+    #   nicegui
     #   requests
 charset-normalizer==3.4.0
     # via requests
@@ -94,6 +99,8 @@ deprecated==1.2.14
     #   opentelemetry-semantic-conventions
 dnspython==2.7.0
     # via email-validator
+docutils==0.21.2
+    # via nicegui
 email-validator==2.2.0
     # via pydantic
 fast-depends==2.4.12
@@ -102,6 +109,7 @@ fastapi==0.115.5
     # via
     #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
     #   -r requirements/_base.in
+    #   nicegui
 faststream==0.5.28
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 frozenlist==1.4.1
@@ -143,15 +151,37 @@ httpx==0.27.2
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
     #   -r requirements/_base.in
+    #   nicegui
 idna==3.10
     # via
     #   anyio
     #   email-validator
     #   httpx
     #   requests
     #   yarl
+ifaddr==0.2.0
+    # via nicegui
 importlib-metadata==8.4.0
     # via opentelemetry-api
+itsdangerous==2.2.0
+    # via nicegui
+jinja2==3.1.4
+    # via
+    #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   nicegui
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
@@ -177,14 +207,20 @@ mako==1.3.5
     #   alembic
 markdown-it-py==3.0.0
     # via rich
+markdown2==2.5.1
+    # via nicegui
 markupsafe==3.0.1
-    # via mako
+    # via
+    #   jinja2
+    #   mako
 mdurl==0.1.2
     # via markdown-it-py
 multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
+nicegui==2.7.0
+    # via -r requirements/_base.in
 opentelemetry-api==1.27.0
     # via
     #   -r requirements/../../../packages/service-library/requirements/_base.in
@@ -279,6 +315,7 @@ orjson==3.10.7
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in
+    #   nicegui
 packaging==24.1
     # via -r requirements/_base.in
 pamqp==3.3.0
@@ -295,6 +332,8 @@ protobuf==4.25.5
     # via
     #   googleapis-common-protos
     #   opentelemetry-proto
+pscript==0.7.7
+    # via vbuild
 psutil==6.0.0
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 psycopg2-binary==2.9.10
@@ -352,7 +391,9 @@ pydantic-settings==2.6.1
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
 pygments==2.18.0
-    # via rich
+    # via
+    #   nicegui
+    #   rich
 pyinstrument==5.0.0
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 python-dateutil==2.9.0.post0
@@ -363,8 +404,12 @@ python-dotenv==1.0.1
     #   uvicorn
 python-engineio==4.10.1
     # via python-socketio
+python-multipart==0.0.17
+    # via nicegui
 python-socketio==5.11.4
-    # via -r requirements/_base.in
+    # via
+    #   -r requirements/_base.in
+    #   nicegui
 pyyaml==6.0.2
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -408,7 +453,9 @@ referencing==0.29.3
 repro-zipfile==0.3.1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 requests==2.32.3
-    # via opentelemetry-exporter-otlp-proto-http
+    # via
+    #   nicegui
+    #   opentelemetry-exporter-otlp-proto-http
 rich==13.9.2
     # via
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
@@ -485,6 +532,7 @@ typing-extensions==4.12.2
     #   alembic
     #   fastapi
     #   faststream
+    #   nicegui
     #   opentelemetry-sdk
     #   pydantic
     #   pydantic-core
@@ -507,15 +555,21 @@ urllib3==2.2.3
     #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
+    #   nicegui
     #   requests
 uvicorn==0.32.0
     # via
     #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
     #   -r requirements/_base.in
+    #   nicegui
 uvloop==0.21.0
     # via uvicorn
+vbuild==0.8.2
+    # via nicegui
 watchfiles==0.24.0
-    # via uvicorn
+    # via
+    #   nicegui
+    #   uvicorn
 websockets==13.1
     # via uvicorn
 wrapt==1.16.0

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_dependencies.py → services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/dependencies.py

@@ -6,7 +6,7 @@
 from settings_library.redis import RedisDatabase
 from simcore_service_dynamic_scheduler.services.redis import get_all_redis_clients
 
-from ...services.rabbitmq import get_rabbitmq_client, get_rabbitmq_rpc_server
+from ..services.rabbitmq import get_rabbitmq_client, get_rabbitmq_rpc_server
 
 assert get_app  # nosec
 assert get_reverse_url_mapper  # nosec

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/__init__.py

@@ -0,0 +1,3 @@
+from ._setup import setup_frontend
+
+__all__: tuple[str, ...] = ("setup_frontend",)

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_setup.py

@@ -0,0 +1,19 @@
+import nicegui
+from fastapi import FastAPI
+
+from ...core.settings import ApplicationSettings
+from ._utils import set_parent_app
+from .routes import router
+
+
+def setup_frontend(app: FastAPI) -> None:
+    settings: ApplicationSettings = app.state.settings
+
+    nicegui.app.include_router(router)
+
+    nicegui.ui.run_with(
+        app,
+        mount_path="/",
+        storage_secret=settings.DYNAMIC_SCHEDULER_UI_STORAGE_SECRET.get_secret_value(),
+    )
+    set_parent_app(app)

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/_utils.py

@@ -0,0 +1,11 @@
+import nicegui
+from fastapi import FastAPI
+
+
+def set_parent_app(parent_app: FastAPI) -> None:
+    nicegui.app.state.parent_app = parent_app
+
+
+def get_parent_app(app: FastAPI) -> FastAPI:
+    parent_app: FastAPI = app.state.parent_app
+    return parent_app

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/frontend/routes/__init__.py

@@ -0,0 +1,10 @@
+from nicegui import APIRouter
+
+from . import _index, _service
+
+router = APIRouter()
+
+router.include_router(_index.router)
+router.include_router(_service.router)
+
+__all__: tuple[str, ...] = ("router",)