added instrumentation for prometheus metrics

Author: Andrei Neagu

services/agent/src/simcore_service_agent/core/application.py

@@ -5,9 +5,6 @@
     get_common_oas_options,
     override_fastapi_openapi_method,
 )
-from servicelib.fastapi.prometheus_instrumentation import (
-    setup_prometheus_instrumentation,
-)
 from servicelib.logging_utils import config_all_loggers
 
 from .._meta import (
@@ -18,6 +15,7 @@
     SUMMARY,
     VERSION,
 )
+from ..services.instrumentation import setup_instrumentation
 from ..services.rabbitmq import setup_rabbitmq
 from ..services.volumes_manager import setup_volume_manager
 from .api.rest.routes import setup_rest_api
@@ -53,8 +51,7 @@ def create_app() -> FastAPI:
     override_fastapi_openapi_method(app)
     app.state.settings = settings
 
-    if app.state.settings.AGENT_PROMETHEUS_INSTRUMENTATION_ENABLED:
-        setup_prometheus_instrumentation(app)
+    setup_instrumentation(app)
 
     setup_rabbitmq(app)
     setup_volume_manager(app)

services/agent/src/simcore_service_agent/services/docker_utils.py

@@ -12,9 +12,11 @@
 )
 from servicelib.docker_constants import PREFIX_DYNAMIC_SIDECAR_VOLUMES
 from servicelib.logging_utils import log_catch, log_context
+from simcore_service_agent.core.settings import ApplicationSettings
 from starlette import status
 
 from .backup import backup_volume
+from .instrumentation import get_instrumentation
 from .models import VolumeDetails
 
 _logger = logging.getLogger(__name__)
@@ -79,6 +81,10 @@ async def _backup_volume(app: FastAPI, docker: Docker, *, volume_name: str) -> N
             _logger, logging.INFO, f"backup '{volume_name}'", log_duration=True
         ):
             volume_details = await get_volume_details(docker, volume_name=volume_name)
+            settings: ApplicationSettings = app.state.settings
+            get_instrumentation(app).agent_metrics.backedup_volumes(
+                settings.AGENT_DOCKER_NODE_ID
+            )
             await backup_volume(app, volume_details, volume_name)
     else:
         _logger.debug("No backup is required for '%s'", volume_name)
@@ -95,3 +101,8 @@ async def remove_volume(
             await _backup_volume(app, docker, volume_name=volume_name)
 
         await DockerVolume(docker, volume_name).delete()
+
+        settings: ApplicationSettings = app.state.settings
+        get_instrumentation(app).agent_metrics.remove_volumes(
+            settings.AGENT_DOCKER_NODE_ID
+        )

services/agent/src/simcore_service_agent/services/instrumentation/__init__.py

@@ -0,0 +1,6 @@
+from ._setup import get_instrumentation, setup_instrumentation
+
+__all__: tuple[str, ...] = (
+    "get_instrumentation",
+    "setup_instrumentation",
+)

services/agent/src/simcore_service_agent/services/instrumentation/_models.py

@@ -0,0 +1,52 @@
+from dataclasses import dataclass, field
+from typing import Final
+
+from prometheus_client import CollectorRegistry, Counter
+from servicelib.instrumentation import MetricsBase, get_metrics_namespace
+
+from ..._meta import APP_NAME
+
+_METRICS_NAMESPACE: Final[str] = get_metrics_namespace(APP_NAME)
+_LABELS_COUNTERS: Final[tuple[str, ...]] = ("docker_node_id",)
+
+
+@dataclass(slots=True, kw_only=True)
+class AgentMetrics(MetricsBase):
+    volumes_removed: Counter = field(init=False)
+    volumes_backedup: Counter = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.volumes_removed = Counter(
+            "volumes_removed_total",
+            "Number of removed volumes by the agent",
+            labelnames=_LABELS_COUNTERS,
+            namespace=_METRICS_NAMESPACE,
+            subsystem=self.subsystem,
+            registry=self.registry,
+        )
+
+        self.volumes_backedup = Counter(
+            "volumes_backedup_total",
+            "Number of removed volumes who's content was uplaoded by the agent",
+            labelnames=_LABELS_COUNTERS,
+            namespace=_METRICS_NAMESPACE,
+            subsystem=self.subsystem,
+            registry=self.registry,
+        )
+
+    def remove_volumes(self, docker_node_id: str) -> None:
+        self.volumes_removed.labels(docker_node_id=docker_node_id).inc()
+
+    def backedup_volumes(self, docker_node_id: str) -> None:
+        self.volumes_backedup.labels(docker_node_id=docker_node_id).inc()
+
+
+@dataclass(slots=True, kw_only=True)
+class AgentInstrumentation:
+    registry: CollectorRegistry
+    agent_metrics: AgentMetrics = field(init=False)
+
+    def __post_init__(self) -> None:
+        self.agent_metrics = AgentMetrics(  # pylint: disable=unexpected-keyword-arg
+            subsystem="agent", registry=self.registry
+        )

services/agent/src/simcore_service_agent/services/instrumentation/_setup.py

@@ -0,0 +1,30 @@
+from fastapi import FastAPI
+from servicelib.fastapi.prometheus_instrumentation import (
+    setup_prometheus_instrumentation,
+)
+from simcore_service_agent.core.settings import ApplicationSettings
+
+from ._models import AgentInstrumentation
+
+
+def setup_instrumentation(app: FastAPI) -> None:
+    settings: ApplicationSettings = app.state.settings
+    if not settings.AGENT_PROMETHEUS_INSTRUMENTATION_ENABLED:
+        return
+
+    instrumentator = setup_prometheus_instrumentation(app)
+
+    async def on_startup() -> None:
+        app.state.instrumentation = AgentInstrumentation(
+            registry=instrumentator.registry
+        )
+
+    app.add_event_handler("startup", on_startup)
+
+
+def get_instrumentation(app: FastAPI) -> AgentInstrumentation:
+    assert (
+        app.state.instrumentation
+    ), "Instrumentation not setup. Please check the configuration"  # nosec
+    instrumentation: AgentInstrumentation = app.state.instrumentation
+    return instrumentation