OpenTelemetry monitoring

main.py

@@ -1,6 +1,24 @@
 from fastapi import FastAPI, Response, status, responses
+import os
+import logging
+import telemetry
 
-app = FastAPI()
+
+OTLP_GRPC_ENDPOINT = os.getenv("OTLP_GRPC_ENDPOINT", "tempo:4317")
+
+app_name = "refinery-authorizer"
+app = FastAPI(title=app_name)
+
+if telemetry.ENABLE_TELEMETRY:
+    print("WARNING:  Running telemetry.", flush=True)
+    telemetry.setting_otlp(app, app_name=app_name, endpoint=OTLP_GRPC_ENDPOINT)
+    app.add_middleware(telemetry.PrometheusMiddleware, app_name=app_name)
+    app.add_route("/metrics", telemetry.metrics)
+
+    # Filter out /metrics
+    logging.getLogger("uvicorn.access").addFilter(
+        lambda record: "GET /metrics" not in record.getMessage()
+    )
 
 
 @app.get("/health")

requirements.txt

@@ -6,65 +6,130 @@
 #
 annotated-types==0.7.0
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   pydantic
 anyio==4.9.0
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   starlette
+asgiref==3.9.2
+    # via opentelemetry-instrumentation-asgi
 certifi==2025.7.14
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   requests
 charset-normalizer==3.4.2
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   requests
 click==8.2.1
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   uvicorn
 fastapi==0.116.1
-    # via -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    # via -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
+googleapis-common-protos==1.70.0
+    # via opentelemetry-exporter-otlp-proto-grpc
+grpcio==1.75.0
+    # via opentelemetry-exporter-otlp-proto-grpc
 h11==0.16.0
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   uvicorn
 idna==3.10
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   anyio
     #   requests
+importlib-metadata==8.7.0
+    # via opentelemetry-api
+opentelemetry-api==1.37.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asgi
+    #   opentelemetry-instrumentation-fastapi
+    #   opentelemetry-instrumentation-logging
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-exporter-otlp-proto-common==1.37.0
+    # via opentelemetry-exporter-otlp-proto-grpc
+opentelemetry-exporter-otlp-proto-grpc==1.37.0
+    # via -r requirements/requirements.in
+opentelemetry-instrumentation==0.58b0
+    # via
+    #   opentelemetry-instrumentation-asgi
+    #   opentelemetry-instrumentation-fastapi
+    #   opentelemetry-instrumentation-logging
+opentelemetry-instrumentation-asgi==0.58b0
+    # via opentelemetry-instrumentation-fastapi
+opentelemetry-instrumentation-fastapi==0.58b0
+    # via -r requirements/requirements.in
+opentelemetry-instrumentation-logging==0.58b0
+    # via -r requirements/requirements.in
+opentelemetry-proto==1.37.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+opentelemetry-sdk==1.37.0
+    # via opentelemetry-exporter-otlp-proto-grpc
+opentelemetry-semantic-conventions==0.58b0
+    # via
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asgi
+    #   opentelemetry-instrumentation-fastapi
+    #   opentelemetry-sdk
+opentelemetry-util-http==0.58b0
+    # via
+    #   opentelemetry-instrumentation-asgi
+    #   opentelemetry-instrumentation-fastapi
+packaging==25.0
+    # via opentelemetry-instrumentation
+prometheus-client==0.23.1
+    # via -r requirements/requirements.in
+protobuf==6.32.1
+    # via
+    #   googleapis-common-protos
+    #   opentelemetry-proto
 pydantic==2.7.4
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   -r requirements/requirements.in
     #   fastapi
 pydantic-core==2.18.4
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   pydantic
 requests==2.32.4
-    # via -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    # via -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
 sniffio==1.3.1
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   anyio
 starlette==0.47.2
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   fastapi
 typing-extensions==4.14.1
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   anyio
     #   fastapi
+    #   grpcio
+    #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
     #   pydantic
     #   pydantic-core
     #   starlette
 urllib3==2.5.0
     # via
-    #   -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
     #   requests
 uvicorn==0.35.0
-    # via -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt
+    # via -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt
+wrapt==1.17.3
+    # via opentelemetry-instrumentation
+zipp==3.23.0
+    # via importlib-metadata

requirements/requirements.in

@@ -1,2 +1,6 @@
 -r mini-requirements.txt
-pydantic==2.7.4
+pydantic==2.7.4
+opentelemetry-exporter-otlp-proto-grpc==1.37.0
+opentelemetry-instrumentation-fastapi==0.58b0
+opentelemetry-instrumentation-logging==0.58b0
+prometheus-client==0.23.1

start

@@ -2,11 +2,13 @@
 
 DEBUG_MODE=false
 DEBUG_PORT=15672
+ENABLE_TELEMETRY=false
 
-while getopts d flag
+while getopts dg flag
 do
     case "${flag}" in
         d) DEBUG_MODE=true;;
+        g) ENABLE_TELEMETRY=true;;
     esac
 done
 
@@ -29,7 +31,10 @@ echo -ne 'starting...'
 docker run -d --rm \
 --name refinery-authorizer \
 -p $DEBUG_PORT:$DEBUG_PORT \
+-e ENABLE_TELEMETRY=$ENABLE_TELEMETRY \
 --network dev-setup_default \
+--log-driver=loki \
+--log-opt loki-url="http://$HOST_IP:3100/loki/api/v1/push" \
 refinery-authorizer-dev $CMD > /dev/null 2>&1
 echo -ne '\t\t\t [done]\n'
 

telemetry.py

@@ -0,0 +1,143 @@
+from typing import Tuple
+
+import time
+import os
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from prometheus_client import REGISTRY, Counter, Gauge, Histogram
+from prometheus_client.openmetrics.exposition import (
+    CONTENT_TYPE_LATEST,
+    generate_latest,
+)
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+from starlette.routing import Match
+from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR
+from starlette.types import ASGIApp
+
+ENABLE_TELEMETRY = os.getenv("ENABLE_TELEMETRY", "false") == "true"
+
+INFO = Gauge("fastapi_app_info", "FastAPI application information.", ["app_name"])
+REQUESTS = Counter(
+    "fastapi_requests_total",
+    "Total count of requests by method and path.",
+    ["method", "path", "app_name"],
+)
+RESPONSES = Counter(
+    "fastapi_responses_total",
+    "Total count of responses by method, path and status codes.",
+    ["method", "path", "status_code", "app_name"],
+)
+REQUESTS_PROCESSING_TIME = Histogram(
+    "fastapi_requests_duration_seconds",
+    "Histogram of requests processing time by path (in seconds)",
+    ["method", "path", "app_name"],
+)
+EXCEPTIONS = Counter(
+    "fastapi_exceptions_total",
+    "Total count of exceptions raised by path and exception type",
+    ["method", "path", "exception_type", "app_name"],
+)
+REQUESTS_IN_PROGRESS = Gauge(
+    "fastapi_requests_in_progress",
+    "Gauge of requests by method and path currently being processed",
+    ["method", "path", "app_name"],
+)
+
+
+class PrometheusMiddleware(BaseHTTPMiddleware):
+    def __init__(self, app: ASGIApp, app_name: str = "fastapi-app") -> None:
+        super().__init__(app)
+        self.app_name = app_name
+        INFO.labels(app_name=self.app_name).inc()
+
+    async def dispatch(
+        self, request: Request, call_next: RequestResponseEndpoint
+    ) -> Response:
+        method = request.method
+        path, is_handled_path = self.get_path(request)
+
+        if not is_handled_path:
+            return await call_next(request)
+
+        REQUESTS_IN_PROGRESS.labels(
+            method=method, path=path, app_name=self.app_name
+        ).inc()
+        REQUESTS.labels(method=method, path=path, app_name=self.app_name).inc()
+        before_time = time.perf_counter()
+        try:
+            response = await call_next(request)
+        except BaseException as e:
+            status_code = HTTP_500_INTERNAL_SERVER_ERROR
+            EXCEPTIONS.labels(
+                method=method,
+                path=path,
+                exception_type=type(e).__name__,
+                app_name=self.app_name,
+            ).inc()
+            raise e from None
+        else:
+            status_code = response.status_code
+            after_time = time.perf_counter()
+            # retrieve trace id for exemplar
+            span = trace.get_current_span()
+            trace_id = trace.format_trace_id(span.get_span_context().trace_id)
+
+            REQUESTS_PROCESSING_TIME.labels(
+                method=method, path=path, app_name=self.app_name
+            ).observe(after_time - before_time, exemplar={"TraceID": trace_id})
+        finally:
+            RESPONSES.labels(
+                method=method,
+                path=path,
+                status_code=status_code,
+                app_name=self.app_name,
+            ).inc()
+            REQUESTS_IN_PROGRESS.labels(
+                method=method, path=path, app_name=self.app_name
+            ).dec()
+
+        return response
+
+    @staticmethod
+    def get_path(request: Request) -> Tuple[str, bool]:
+        for route in request.app.routes:
+            match, child_scope = route.matches(request.scope)
+            if match == Match.FULL:
+                return route.path, True
+
+        return request.url.path, False
+
+
+def metrics(request: Request) -> Response:
+    return Response(
+        generate_latest(REGISTRY), headers={"Content-Type": CONTENT_TYPE_LATEST}
+    )
+
+
+def setting_otlp(
+    app: ASGIApp, app_name: str, endpoint: str, log_correlation: bool = True
+) -> None:
+    # Setting OpenTelemetry
+    # set the service name to show in traces
+    resource = Resource.create(attributes={"service.name": app_name})
+
+    # set the tracer provider
+    tracer = TracerProvider(resource=resource)
+    trace.set_tracer_provider(tracer)
+
+    tracer.add_span_processor(
+        BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True))
+    )
+
+    if log_correlation:
+        LoggingInstrumentor().instrument(set_logging_format=True)
+
+    FastAPIInstrumentor.instrument_app(app, tracer_provider=tracer)