diff --git a/main.py b/main.py index 07afe9d..6e2a97c 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,24 @@ from fastapi import FastAPI, Response, status, responses +import os +import logging +import telemetry -app = FastAPI() + +OTLP_GRPC_ENDPOINT = os.getenv("OTLP_GRPC_ENDPOINT", "tempo:4317") + +app_name = "refinery-authorizer" +app = FastAPI(title=app_name) + +if telemetry.ENABLE_TELEMETRY: + print("WARNING: Running telemetry.", flush=True) + telemetry.setting_otlp(app, app_name=app_name, endpoint=OTLP_GRPC_ENDPOINT) + app.add_middleware(telemetry.PrometheusMiddleware, app_name=app_name) + app.add_route("/metrics", telemetry.metrics) + + # Filter out /metrics + logging.getLogger("uvicorn.access").addFilter( + lambda record: "GET /metrics" not in record.getMessage() + ) @app.get("/health") diff --git a/requirements.txt b/requirements.txt index 8cc38ef..f90ab59 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,65 +6,130 @@ # annotated-types==0.7.0 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # pydantic anyio==4.9.0 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # starlette +asgiref==3.9.2 + # via opentelemetry-instrumentation-asgi certifi==2025.7.14 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # requests charset-normalizer==3.4.2 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # requests click==8.2.1 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # uvicorn fastapi==0.116.1 - # via -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # via -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt +googleapis-common-protos==1.70.0 + # via opentelemetry-exporter-otlp-proto-grpc +grpcio==1.75.0 + # via opentelemetry-exporter-otlp-proto-grpc h11==0.16.0 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # uvicorn idna==3.10 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # anyio # requests +importlib-metadata==8.7.0 + # via opentelemetry-api +opentelemetry-api==1.37.0 + # via + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-logging + # opentelemetry-sdk + # opentelemetry-semantic-conventions +opentelemetry-exporter-otlp-proto-common==1.37.0 + # via opentelemetry-exporter-otlp-proto-grpc +opentelemetry-exporter-otlp-proto-grpc==1.37.0 + # via -r requirements/requirements.in +opentelemetry-instrumentation==0.58b0 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-instrumentation-logging +opentelemetry-instrumentation-asgi==0.58b0 + # via opentelemetry-instrumentation-fastapi +opentelemetry-instrumentation-fastapi==0.58b0 + # via -r requirements/requirements.in +opentelemetry-instrumentation-logging==0.58b0 + # via -r requirements/requirements.in +opentelemetry-proto==1.37.0 + # via + # opentelemetry-exporter-otlp-proto-common + # opentelemetry-exporter-otlp-proto-grpc +opentelemetry-sdk==1.37.0 + # via opentelemetry-exporter-otlp-proto-grpc +opentelemetry-semantic-conventions==0.58b0 + # via + # opentelemetry-instrumentation + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi + # opentelemetry-sdk +opentelemetry-util-http==0.58b0 + # via + # opentelemetry-instrumentation-asgi + # opentelemetry-instrumentation-fastapi +packaging==25.0 + # via opentelemetry-instrumentation +prometheus-client==0.23.1 + # via -r requirements/requirements.in +protobuf==6.32.1 + # via + # googleapis-common-protos + # opentelemetry-proto pydantic==2.7.4 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # -r requirements/requirements.in # fastapi pydantic-core==2.18.4 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # pydantic requests==2.32.4 - # via -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # via -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt sniffio==1.3.1 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # anyio starlette==0.47.2 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # fastapi typing-extensions==4.14.1 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # anyio # fastapi + # grpcio + # opentelemetry-api + # opentelemetry-exporter-otlp-proto-grpc + # opentelemetry-sdk + # opentelemetry-semantic-conventions # pydantic # pydantic-core # starlette urllib3==2.5.0 # via - # -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt # requests uvicorn==0.35.0 - # via -r /home/runner/work/refinery-submodule-parent-images/refinery-submodule-parent-images/refinery-authorizer/requirements/mini-requirements.txt + # via -r /Users/andhrelja/Projects/refinery-authorizer/requirements/mini-requirements.txt +wrapt==1.17.3 + # via opentelemetry-instrumentation +zipp==3.23.0 + # via importlib-metadata diff --git a/requirements/requirements.in b/requirements/requirements.in index aff1dd1..d712627 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,2 +1,6 @@ -r mini-requirements.txt -pydantic==2.7.4 \ No newline at end of file +pydantic==2.7.4 +opentelemetry-exporter-otlp-proto-grpc==1.37.0 +opentelemetry-instrumentation-fastapi==0.58b0 +opentelemetry-instrumentation-logging==0.58b0 +prometheus-client==0.23.1 \ No newline at end of file diff --git a/start b/start index 1b51302..062d9ce 100755 --- a/start +++ b/start @@ -2,11 +2,13 @@ DEBUG_MODE=false DEBUG_PORT=15672 +ENABLE_TELEMETRY=false -while getopts d flag +while getopts dg flag do case "${flag}" in d) DEBUG_MODE=true;; + g) ENABLE_TELEMETRY=true;; esac done @@ -29,7 +31,10 @@ echo -ne 'starting...' docker run -d --rm \ --name refinery-authorizer \ -p $DEBUG_PORT:$DEBUG_PORT \ +-e ENABLE_TELEMETRY=$ENABLE_TELEMETRY \ --network dev-setup_default \ +--log-driver=loki \ +--log-opt loki-url="http://$HOST_IP:3100/loki/api/v1/push" \ refinery-authorizer-dev $CMD > /dev/null 2>&1 echo -ne '\t\t\t [done]\n' diff --git a/telemetry.py b/telemetry.py new file mode 100644 index 0000000..4164283 --- /dev/null +++ b/telemetry.py @@ -0,0 +1,143 @@ +from typing import Tuple + +import time +import os + +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.instrumentation.logging import LoggingInstrumentor +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from prometheus_client import REGISTRY, Counter, Gauge, Histogram +from prometheus_client.openmetrics.exposition import ( + CONTENT_TYPE_LATEST, + generate_latest, +) +from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint +from starlette.requests import Request +from starlette.responses import Response +from starlette.routing import Match +from starlette.status import HTTP_500_INTERNAL_SERVER_ERROR +from starlette.types import ASGIApp + +ENABLE_TELEMETRY = os.getenv("ENABLE_TELEMETRY", "false") == "true" + +INFO = Gauge("fastapi_app_info", "FastAPI application information.", ["app_name"]) +REQUESTS = Counter( + "fastapi_requests_total", + "Total count of requests by method and path.", + ["method", "path", "app_name"], +) +RESPONSES = Counter( + "fastapi_responses_total", + "Total count of responses by method, path and status codes.", + ["method", "path", "status_code", "app_name"], +) +REQUESTS_PROCESSING_TIME = Histogram( + "fastapi_requests_duration_seconds", + "Histogram of requests processing time by path (in seconds)", + ["method", "path", "app_name"], +) +EXCEPTIONS = Counter( + "fastapi_exceptions_total", + "Total count of exceptions raised by path and exception type", + ["method", "path", "exception_type", "app_name"], +) +REQUESTS_IN_PROGRESS = Gauge( + "fastapi_requests_in_progress", + "Gauge of requests by method and path currently being processed", + ["method", "path", "app_name"], +) + + +class PrometheusMiddleware(BaseHTTPMiddleware): + def __init__(self, app: ASGIApp, app_name: str = "fastapi-app") -> None: + super().__init__(app) + self.app_name = app_name + INFO.labels(app_name=self.app_name).inc() + + async def dispatch( + self, request: Request, call_next: RequestResponseEndpoint + ) -> Response: + method = request.method + path, is_handled_path = self.get_path(request) + + if not is_handled_path: + return await call_next(request) + + REQUESTS_IN_PROGRESS.labels( + method=method, path=path, app_name=self.app_name + ).inc() + REQUESTS.labels(method=method, path=path, app_name=self.app_name).inc() + before_time = time.perf_counter() + try: + response = await call_next(request) + except BaseException as e: + status_code = HTTP_500_INTERNAL_SERVER_ERROR + EXCEPTIONS.labels( + method=method, + path=path, + exception_type=type(e).__name__, + app_name=self.app_name, + ).inc() + raise e from None + else: + status_code = response.status_code + after_time = time.perf_counter() + # retrieve trace id for exemplar + span = trace.get_current_span() + trace_id = trace.format_trace_id(span.get_span_context().trace_id) + + REQUESTS_PROCESSING_TIME.labels( + method=method, path=path, app_name=self.app_name + ).observe(after_time - before_time, exemplar={"TraceID": trace_id}) + finally: + RESPONSES.labels( + method=method, + path=path, + status_code=status_code, + app_name=self.app_name, + ).inc() + REQUESTS_IN_PROGRESS.labels( + method=method, path=path, app_name=self.app_name + ).dec() + + return response + + @staticmethod + def get_path(request: Request) -> Tuple[str, bool]: + for route in request.app.routes: + match, child_scope = route.matches(request.scope) + if match == Match.FULL: + return route.path, True + + return request.url.path, False + + +def metrics(request: Request) -> Response: + return Response( + generate_latest(REGISTRY), headers={"Content-Type": CONTENT_TYPE_LATEST} + ) + + +def setting_otlp( + app: ASGIApp, app_name: str, endpoint: str, log_correlation: bool = True +) -> None: + # Setting OpenTelemetry + # set the service name to show in traces + resource = Resource.create(attributes={"service.name": app_name}) + + # set the tracer provider + tracer = TracerProvider(resource=resource) + trace.set_tracer_provider(tracer) + + tracer.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=endpoint, insecure=True)) + ) + + if log_correlation: + LoggingInstrumentor().instrument(set_logging_format=True) + + FastAPIInstrumentor.instrument_app(app, tracer_provider=tracer)