OpenTelemetry monitoring

app.py

@@ -1,13 +1,31 @@
+import os
+import logging
 from fastapi import FastAPI, HTTPException, responses, status, Request
 from pydantic import BaseModel
 from typing import Union, Dict, Optional
 
 import submodules.model.business_objects.general as general
 from controller import stats
 from controller import integration
+from submodules.model import telemetry
 
-# API creation and description
-app = FastAPI()
+
+OTLP_GRPC_ENDPOINT = os.getenv("OTLP_GRPC_ENDPOINT", "tempo:4317")
+
+app_name = "refinery-weak-supervisor"
+app = FastAPI(title=app_name)
+
+if telemetry.ENABLE_TELEMETRY:
+    print("WARNING:  Running telemetry.", flush=True)
+    telemetry.setting_app_name(app_name)
+    telemetry.setting_otlp(app, app_name=app_name, endpoint=OTLP_GRPC_ENDPOINT)
+    app.add_middleware(telemetry.PrometheusMiddleware, app_name=app_name)
+    app.add_route("/metrics", telemetry.metrics)
+
+    # Filter out /metrics
+    logging.getLogger("uvicorn.access").addFilter(
+        lambda record: "GET /metrics" not in record.getMessage()
+    )
 
 
 class WeakSupervisionRequest(BaseModel):

start

@@ -2,11 +2,13 @@
 
 DEBUG_MODE=false
 DEBUG_PORT=15677
+ENABLE_TELEMETRY=false
 
-while getopts d flag
+while getopts dg flag
 do
     case "${flag}" in
         d) DEBUG_MODE=true;;
+        g) ENABLE_TELEMETRY=true;;
     esac
 done
 
@@ -46,6 +48,7 @@ docker run -d --rm \
 -p 7054:80 \
 -e POSTGRES=postgresql://postgres:kern@graphql-postgres:5432 \
 -e WS_NOTIFY_ENDPOINT="http://refinery-websocket:8080" \
+-e ENABLE_TELEMETRY=$ENABLE_TELEMETRY \
 --mount type=bind,source="$(pwd)"/,target=/app \
 -v /var/run/docker.sock:/var/run/docker.sock \
 -v "$INFERENCE_DIR":/inference \