✨Adding dynamic services monitoring dashboard (⚠️devops) (#6784)

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

.env-devel

@@ -128,6 +128,7 @@ DYNAMIC_SCHEDULER_LOGLEVEL=DEBUG
 DYNAMIC_SCHEDULER_PROFILING=1
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
 DYNAMIC_SCHEDULER_TRACING={}
+DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin
 
 FUNCTION_SERVICES_AUTHORS='{"UN": {"name": "Unknown", "email": "[email protected]", "affiliation": "unknown"}}'
 

Makefile

@@ -322,6 +322,7 @@ printf "$$rows" "oSparc platform" "http://$(get_my_ip).nip.io:9081";\
 printf "$$rows" "oSparc public API doc" "http://$(get_my_ip).nip.io:8006/dev/doc";\
 printf "$$rows" "oSparc web API doc" "http://$(get_my_ip).nip.io:9081/dev/doc";\
 printf "$$rows" "Dask Dashboard" "http://$(get_my_ip).nip.io:8787";\
+printf "$$rows" "Dy-scheduler Dashboard" "http://$(get_my_ip).nip.io:8012";\
 printf "$$rows" "Docker Registry" "http://$${REGISTRY_URL}/v2/_catalog" $${REGISTRY_USER} $${REGISTRY_PW};\
 printf "$$rows" "Invitations" "http://$(get_my_ip).nip.io:8008/dev/doc" $${INVITATIONS_USERNAME} $${INVITATIONS_PASSWORD};\
 printf "$$rows" "Jaeger" "http://$(get_my_ip).nip.io:16686";\

packages/models-library/tests/test_service_settings_labels.py

@@ -8,8 +8,8 @@
 from pprint import pformat
 from typing import Any, Final, NamedTuple
 
-import pydantic_core
 import pytest
+from common_library.json_serialization import json_dumps
 from models_library.basic_types import PortInt
 from models_library.osparc_variable_identifier import (
     OsparcVariableIdentifier,
@@ -558,11 +558,6 @@ def test_can_parse_labels_with_osparc_identifiers(
     assert "$" not in service_meta_str
 
 
-def servicelib__json_serialization__json_dumps(obj: Any, **kwargs):
-    # Analogous to 'models_library.utils.json_serialization.json_dumps'
-    return json.dumps(obj, default=pydantic_core.to_jsonable_python, **kwargs)
-
-
 def test_resolving_some_service_labels_at_load_time(
     vendor_environments: dict[str, Any], service_labels: dict[str, str]
 ):
@@ -579,9 +574,7 @@ def test_resolving_some_service_labels_at_load_time(
         ("settings", SimcoreServiceSettingsLabel),
     ):
         to_serialize = getattr(service_meta, attribute_name)
-        template = TextTemplate(
-            servicelib__json_serialization__json_dumps(to_serialize)
-        )
+        template = TextTemplate(json_dumps(to_serialize))
         assert template.is_valid()
         resolved_label: str = template.safe_substitute(vendor_environments)
         to_restore = TypeAdapter(pydantic_model).validate_json(resolved_label)

packages/models-library/tests/test_utils_fastapi_encoders.py

@@ -4,36 +4,25 @@
 # pylint: disable=too-many-arguments
 
 import json
-from typing import Any
 from uuid import uuid4
 
-import pytest
+from common_library.json_serialization import json_dumps
 from faker import Faker
 from models_library.utils.fastapi_encoders import servicelib_jsonable_encoder
-from pydantic.json import pydantic_encoder
-
-
-def servicelib__json_serialization__json_dumps(obj: Any, **kwargs):
-    # Analogous to 'models_library.utils.json_serialization.json_dumps'
-    return json.dumps(obj, default=pydantic_encoder, **kwargs)
 
 
 def test_using_uuids_as_keys(faker: Faker):
     uuid_key = uuid4()
 
-    with pytest.raises(TypeError):
-        # IMPORTANT NOTE: we cannot serialize UUID objects as keys.
-        # We have to convert them to strings but then the class information is lost upon deserialization i.e. it is not reversable!
-        # NOTE: This could  potentially be solved using 'orjson' !!
-        #
-        servicelib__json_serialization__json_dumps({uuid_key: "value"}, indent=1)
+    # this was previously failing
+    assert json_dumps({uuid_key: "value"}, indent=1)
 
-    # use encoder
+    # uuid keys now serialize without raising to the expected format string
     data = servicelib_jsonable_encoder({uuid_key: "value"})
     assert data == {f"{uuid_key}": "value"}
 
     # serialize w/o raising
-    dumped_data = servicelib__json_serialization__json_dumps(data, indent=1)
+    dumped_data = json_dumps(data, indent=1)
 
     # deserialize w/o raising
     loaded_data = json.loads(dumped_data)

services/docker-compose.yml

@@ -566,6 +566,7 @@ services:
       DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT}
       DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING}
       DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING}
+      DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET}
       TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT}
       TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT}
   static-webserver:

services/dynamic-scheduler/Dockerfile

@@ -146,7 +146,7 @@ HEALTHCHECK --interval=30s \
   --timeout=20s \
   --start-period=30s \
   --retries=3 \
-  CMD ["python3", "services/dynamic-scheduler/docker/healthcheck.py", "http://localhost:8000/"]
+  CMD ["python3", "services/dynamic-scheduler/docker/healthcheck.py", "http://localhost:8000/health"]
 
 ENTRYPOINT [ "/bin/sh", "services/dynamic-scheduler/docker/entrypoint.sh" ]
 CMD ["/bin/sh", "services/dynamic-scheduler/docker/boot.sh"]

services/dynamic-scheduler/requirements/_base.in

@@ -18,6 +18,7 @@
 arrow
 fastapi
 httpx
+nicegui
 packaging
 python-socketio
 typer[all]

services/dynamic-scheduler/requirements/_base.txt

@@ -7,7 +7,9 @@ aiodebug==2.3.0
 aiodocker==0.24.0
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 aiofiles==24.1.0
-    # via -r requirements/../../../packages/service-library/requirements/_base.in
+    # via
+    #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   nicegui
 aiohappyeyeballs==2.4.3
     # via aiohttp
 aiohttp==3.11.7
@@ -27,6 +29,8 @@ aiohttp==3.11.7
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   aiodocker
+    #   nicegui
+    #   python-socketio
 aiormq==6.8.1
     # via aio-pika
 aiosignal==1.3.1
@@ -77,6 +81,7 @@ certifi==2024.8.30
     #   -c requirements/../../../requirements/constraints.txt
     #   httpcore
     #   httpx
+    #   nicegui
     #   requests
 charset-normalizer==3.4.0
     # via requests
@@ -92,6 +97,8 @@ deprecated==1.2.15
     #   opentelemetry-semantic-conventions
 dnspython==2.7.0
     # via email-validator
+docutils==0.21.2
+    # via nicegui
 email-validator==2.2.0
     # via pydantic
 exceptiongroup==1.2.2
@@ -102,6 +109,7 @@ fastapi==0.115.5
     # via
     #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
     #   -r requirements/_base.in
+    #   nicegui
 faststream==0.5.30
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 frozenlist==1.5.0
@@ -143,15 +151,37 @@ httpx==0.27.2
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
     #   -r requirements/_base.in
+    #   nicegui
 idna==3.10
     # via
     #   anyio
     #   email-validator
     #   httpx
     #   requests
     #   yarl
+ifaddr==0.2.0
+    # via nicegui
 importlib-metadata==8.5.0
     # via opentelemetry-api
+itsdangerous==2.2.0
+    # via nicegui
+jinja2==3.1.4
+    # via
+    #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   nicegui
 jsonschema==4.23.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
@@ -177,14 +207,20 @@ mako==1.3.6
     #   alembic
 markdown-it-py==3.0.0
     # via rich
+markdown2==2.5.1
+    # via nicegui
 markupsafe==3.0.2
-    # via mako
+    # via
+    #   jinja2
+    #   mako
 mdurl==0.1.2
     # via markdown-it-py
 multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
+nicegui==2.7.0
+    # via -r requirements/_base.in
 opentelemetry-api==1.28.2
     # via
     #   -r requirements/../../../packages/service-library/requirements/_base.in
@@ -280,6 +316,7 @@ orjson==3.10.12
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/_base.in
+    #   nicegui
 packaging==24.2
     # via
     #   -r requirements/_base.in
@@ -300,6 +337,8 @@ protobuf==5.28.3
     # via
     #   googleapis-common-protos
     #   opentelemetry-proto
+pscript==0.7.7
+    # via vbuild
 psutil==6.1.0
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 psycopg2-binary==2.9.10
@@ -357,7 +396,9 @@ pydantic-settings==2.6.1
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
 pygments==2.18.0
-    # via rich
+    # via
+    #   nicegui
+    #   rich
 pyinstrument==5.0.0
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 python-dateutil==2.9.0.post0
@@ -368,8 +409,12 @@ python-dotenv==1.0.1
     #   uvicorn
 python-engineio==4.10.1
     # via python-socketio
+python-multipart==0.0.17
+    # via nicegui
 python-socketio==5.11.4
-    # via -r requirements/_base.in
+    # via
+    #   -r requirements/_base.in
+    #   nicegui
 pyyaml==6.0.2
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -412,7 +457,9 @@ referencing==0.35.1
 repro-zipfile==0.3.1
     # via -r requirements/../../../packages/service-library/requirements/_base.in
 requests==2.32.3
-    # via opentelemetry-exporter-otlp-proto-http
+    # via
+    #   nicegui
+    #   opentelemetry-exporter-otlp-proto-http
 rich==13.9.4
     # via
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
@@ -487,6 +534,7 @@ typing-extensions==4.12.2
     #   alembic
     #   fastapi
     #   faststream
+    #   nicegui
     #   opentelemetry-sdk
     #   pydantic
     #   pydantic-core
@@ -510,15 +558,21 @@ urllib3==2.2.3
     #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
+    #   nicegui
     #   requests
 uvicorn==0.32.1
     # via
     #   -r requirements/../../../packages/service-library/requirements/_fastapi.in
     #   -r requirements/_base.in
+    #   nicegui
 uvloop==0.21.0
     # via uvicorn
+vbuild==0.8.2
+    # via nicegui
 watchfiles==1.0.0
-    # via uvicorn
+    # via
+    #   nicegui
+    #   uvicorn
 websockets==14.1
     # via uvicorn
 wrapt==1.17.0

services/dynamic-scheduler/requirements/_test.in

@@ -15,6 +15,8 @@ asgi_lifespan
 coverage
 docker
 faker
+hypercorn
+playwright
 pytest
 pytest-asyncio
 pytest-cov

services/dynamic-scheduler/requirements/_test.txt

@@ -23,10 +23,20 @@ docker==7.1.0
     # via -r requirements/_test.in
 faker==33.0.0
     # via -r requirements/_test.in
+greenlet==3.1.1
+    # via
+    #   -c requirements/_base.txt
+    #   playwright
 h11==0.14.0
     # via
     #   -c requirements/_base.txt
     #   httpcore
+    #   hypercorn
+    #   wsproto
+h2==4.1.0
+    # via hypercorn
+hpack==4.0.0
+    # via h2
 httpcore==1.0.7
     # via
     #   -c requirements/_base.txt
@@ -36,6 +46,10 @@ httpx==0.27.2
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   respx
+hypercorn==0.17.3
+    # via -r requirements/_test.in
+hyperframe==6.0.1
+    # via h2
 icdiff==2.0.7
     # via pytest-icdiff
 idna==3.10
@@ -51,10 +65,16 @@ packaging==24.2
     #   -c requirements/_base.txt
     #   pytest
     #   pytest-sugar
+playwright==1.49.0
+    # via -r requirements/_test.in
 pluggy==1.5.0
     # via pytest
 pprintpp==0.4.0
     # via pytest-icdiff
+priority==2.0.0
+    # via hypercorn
+pyee==12.0.0
+    # via playwright
 pytest==8.3.3
     # via
     #   -r requirements/_test.in
@@ -107,9 +127,14 @@ typing-extensions==4.12.2
     # via
     #   -c requirements/_base.txt
     #   faker
+    #   pyee
 urllib3==2.2.3
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   docker
     #   requests
+wsproto==1.2.0
+    # via
+    #   -c requirements/_base.txt
+    #   hypercorn