re-added monitoring

Author: sanderegg

services/director/src/simcore_service_director/instrumentation.py

@@ -1,10 +1,57 @@
+from dataclasses import dataclass, field
+from typing import cast
+
 from fastapi import FastAPI
+from prometheus_client import CollectorRegistry, Counter
 from servicelib.fastapi.prometheus_instrumentation import (
     setup_prometheus_instrumentation,
 )
+from servicelib.instrumentation import MetricsBase, get_metrics_namespace
 
+from ._meta import APP_NAME
+from .core.errors import ConfigurationError
 from .core.settings import get_application_settings
 
+MONITOR_SERVICE_STARTED_LABELS: list[str] = [
+    "service_key",
+    "service_tag",
+    "simcore_user_agent",
+]
+
+MONITOR_SERVICE_STOPPED_LABELS: list[str] = [
+    "service_key",
+    "service_tag",
+    "result",
+    "simcore_user_agent",
+]
+
+
+@dataclass(slots=True, kw_only=True)
+class DirectorV0Instrumentation(MetricsBase):
+    registry: CollectorRegistry
+
+    services_started: Counter = field(init=False)
+    services_stopped: Counter = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.services_started = Counter(
+            name="services_started_total",
+            documentation="Counts the services started",
+            labelnames=MONITOR_SERVICE_STARTED_LABELS,
+            namespace=get_metrics_namespace(APP_NAME),
+            subsystem=self.subsystem,
+            registry=self.registry,
+        )
+
+        self.services_stopped = Counter(
+            name="services_stopped_total",
+            documentation="Counts the services stopped",
+            labelnames=MONITOR_SERVICE_STOPPED_LABELS,
+            namespace=get_metrics_namespace(APP_NAME),
+            subsystem=self.subsystem,
+            registry=self.registry,
+        )
+
 
 def setup(app: FastAPI) -> None:
     app_settings = get_application_settings(app)
@@ -15,15 +62,10 @@ def setup(app: FastAPI) -> None:
     instrumentator = setup_prometheus_instrumentation(app)
 
     async def on_startup() -> None:
-        # metrics_subsystem = (
-        #     "dynamic" if app_settings.AUTOSCALING_NODES_MONITORING else "computational"
-        # )
-        # app.state.instrumentation = (
-        #     AutoscalingInstrumentation(  # pylint: disable=unexpected-keyword-arg
-        #         registry=instrumentator.registry, subsystem=metrics_subsystem
-        #     )
-        # )
-        ...
+        metrics_subsystem = ""
+        app.state.instrumentation = DirectorV0Instrumentation(
+            registry=instrumentator.registry, subsystem=metrics_subsystem
+        )
 
     async def on_shutdown() -> None:
         ...
@@ -32,12 +74,12 @@ async def on_shutdown() -> None:
     app.add_event_handler("shutdown", on_shutdown)
 
 
-# def get_instrumentation(app: FastAPI) -> AutoscalingInstrumentation:
-#     if not app.state.instrumentation:
-#         raise ConfigurationError(
-#             msg="Instrumentation not setup. Please check the configuration."
-#         )
-#     return cast(AutoscalingInstrumentation, app.state.instrumentation)
+def get_instrumentation(app: FastAPI) -> DirectorV0Instrumentation:
+    if not app.state.instrumentation:
+        raise ConfigurationError(
+            msg="Instrumentation not setup. Please check the configuration."
+        )
+    return cast(DirectorV0Instrumentation, app.state.instrumentation)
 
 
 def has_instrumentation(app: FastAPI) -> bool:

services/director/src/simcore_service_director/monitoring.py

@@ -25,8 +25,6 @@
 from fastapi import FastAPI
 from packaging.version import Version
 from servicelib.async_utils import run_sequentially_in_context
-
-# from servicelib.monitor_services import service_started, service_stopped
 from settings_library.docker_registry import RegistrySettings
 from tenacity import retry
 from tenacity.retry import retry_if_exception_type
@@ -44,6 +42,7 @@
 )
 from .core.settings import ApplicationSettings, get_application_settings
 from .exceptions import ServiceStateSaveError
+from .instrumentation import get_instrumentation
 from .services_common import ServicesCommonSettings
 from .system_utils import get_system_extra_hosts_raw
 from .utils import parse_as_datetime
@@ -1033,15 +1032,12 @@ async def start_service(
         )
         node_details = containers_meta_data[0]
         if app_settings.DIRECTOR_MONITORING_ENABLED:
-            ...
-            # TODO: is monitoring necessary?
-            # service_started(
-            #     app,
-            #     "undefined_user",  # NOTE: to prevent high cardinality metrics this is disabled
-            #     service_key,
-            #     service_tag,
-            #     "DYNAMIC",
-            # )
+            get_instrumentation(app).services_started.labels(
+                service_key=service_key,
+                service_tag=service_tag,
+                simcore_user_agent="undefined_user",
+            ).inc()
+
         # we return only the info of the main service
         return node_details
 
@@ -1294,13 +1290,9 @@ async def stop_service(app: FastAPI, *, node_uuid: str, save_state: bool) -> Non
         log.debug("removed network")
 
         if app_settings.DIRECTOR_MONITORING_ENABLED:
-            ...
-            # TODO: is it necessary still?
-            # service_stopped(
-            #     app,
-            #     "undefined_user",
-            #     service_details["service_key"],
-            #     service_details["service_version"],
-            #     "DYNAMIC",
-            #     "SUCCESS",
-            # )
+            get_instrumentation(app).services_stopped.labels(
+                service_key=service_details["service_key"],
+                service_tag=service_details["service_version"],
+                simcore_user_agent="undefined_user",
+                result="SUCCESS",
+            ).inc()