OpenTelemetry monitoring

app.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from fastapi import FastAPI
 from starlette.middleware import Middleware
 
@@ -64,16 +65,19 @@
 )
 from util import security, clean_up
 from middleware import log_storage
-from submodules.model import session
+from submodules.model import session, telemetry
 from controller.sums_table import manager as sums_table_manager
 
 logging.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger(__name__)
 
+OTLP_GRPC_ENDPOINT = os.getenv("OTLP_GRPC_ENDPOINT", "tempo:4317")
+
 init_config()
 migrate_kratos_users()
-fastapi_app = FastAPI()
 
+app_name = "refinery-gateway"
+fastapi_app = FastAPI(title=app_name)
 
 fastapi_app.include_router(
     org_router, prefix=PREFIX_ORGANIZATION, tags=["organization"]
@@ -116,7 +120,10 @@
 fastapi_app.include_router(
     playground_router, prefix=PREFIX_PLAYGROUND, tags=["playground"]
 )
-fastapi_app_internal = FastAPI()
+
+app_name_internal = app_name + "-i"
+fastapi_app_internal = FastAPI(title=app_name_internal)
+
 fastapi_app_internal.include_router(
     task_execution_router, prefix=PREFIX_TASK_EXECUTION, tags=["task-execution"]
 )
@@ -142,6 +149,31 @@
 
 fastapi_app.middleware("http")(handle_db_session)
 
+if telemetry.ENABLE_TELEMETRY:
+    print("WARNING:  Running telemetry.", flush=True)
+    telemetry.setting_app_name(app_name)
+    telemetry.setting_otlp(fastapi_app, app_name=app_name, endpoint=OTLP_GRPC_ENDPOINT)
+    fastapi_app.add_middleware(telemetry.PrometheusMiddleware, app_name=app_name)
+    fastapi_app.add_route("/metrics", telemetry.metrics)
+
+    # -------- internal --------
+    app_name += "-i"
+    telemetry.setting_otlp(
+        fastapi_app_internal, app_name=app_name_internal, endpoint=OTLP_GRPC_ENDPOINT
+    )
+    fastapi_app_internal.add_middleware(
+        telemetry.PrometheusMiddleware, app_name=app_name_internal
+    )
+    fastapi_app_internal.add_route("/metrics", telemetry.metrics)
+
+    # Filter out /metrics
+    logging.getLogger("uvicorn.access").addFilter(
+        lambda record: not any(
+            item in record.getMessage()
+            for item in ("GET /api/metrics", "GET /internal/api/metrics")
+        )
+    )
+
 middleware = [Middleware(DatabaseSessionHandler)]
 app = Starlette(routes=routes, middleware=middleware)
 

requirements.txt

@@ -8,21 +8,21 @@ alembic==1.7.1
     # via -r requirements/requirements.in
 annotated-types==0.7.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   pydantic
 anyio==4.11.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   httpx
     #   openai
     #   starlette
 argon2-cffi==25.1.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   minio
 argon2-cffi-bindings==25.1.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   argon2-cffi
 asgiref==3.9.2
     # via
@@ -34,7 +34,7 @@ boto3==1.39.6
     # via -r requirements/common-requirements.txt
 botocore==1.39.17
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   boto3
     #   s3transfer
 catalogue==2.0.10
@@ -44,22 +44,22 @@ catalogue==2.0.10
     #   thinc
 certifi==2025.8.3
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   httpcore
     #   httpx
     #   minio
     #   requests
 cffi==2.0.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   argon2-cffi-bindings
 charset-normalizer==3.4.3
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   requests
 click==8.3.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   typer
     #   uvicorn
 cloudpathlib==0.22.0
@@ -93,7 +93,7 @@ grpcio==1.75.0
     #   opentelemetry-exporter-otlp-proto-grpc
 h11==0.16.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   httpcore
     #   uvicorn
 httpcore==1.0.9
@@ -102,7 +102,7 @@ httpx==0.28.1
     # via openai
 idna==3.10
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   anyio
     #   httpx
     #   requests
@@ -114,7 +114,7 @@ jinja2==3.1.6
     # via spacy
 jmespath==1.0.1
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   boto3
     #   botocore
 langcodes==3.5.0
@@ -134,15 +134,15 @@ markupsafe==3.0.2
 mdurl==0.1.2
     # via markdown-it-py
 minio==7.2.15
-    # via -r requirements/common-requirements.txt
+    # via -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
 murmurhash==1.0.13
     # via
     #   preshed
     #   spacy
     #   thinc
 numpy==1.26.2
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   blis
     #   pandas
     #   spacy
@@ -210,7 +210,7 @@ packaging==25.0
     #   thinc
     #   weasel
 pandas==1.5.1
-    # via -r requirements/common-requirements.txt
+    # via -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
 preshed==3.0.10
     # via
     #   spacy
@@ -226,16 +226,16 @@ psycopg2-binary==2.9.9
     # via -r requirements/common-requirements.txt
 pycparser==2.23
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   cffi
 pycryptodome==3.23.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   minio
     #   rncryptor
 pydantic==2.7.4
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   -r requirements/requirements.in
     #   confection
     #   fastapi
@@ -245,7 +245,7 @@ pydantic==2.7.4
     #   weasel
 pydantic-core==2.18.4
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   pydantic
 pygments==2.19.2
     # via rich
@@ -255,16 +255,16 @@ pyminizip==0.2.6
     # via -r requirements/requirements.in
 python-dateutil==2.9.0.post0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   botocore
     #   pandas
 pytz==2025.2
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   pandas
 requests==2.32.4
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   docker
     #   spacy
     #   weasel
@@ -274,19 +274,19 @@ rncryptor==3.3.0
     # via -r requirements/requirements.in
 s3transfer==0.13.1
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   boto3
 shellingham==1.5.4
     # via typer
 six==1.17.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   python-dateutil
 smart-open==7.3.1
     # via weasel
 sniffio==1.3.1
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   anyio
     #   openai
 spacy[ja]==3.7.5
@@ -297,7 +297,7 @@ spacy-loggers==1.0.5
     # via spacy
 sqlalchemy==1.4.42
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   alembic
 srsly==2.5.1
     # via
@@ -307,7 +307,7 @@ srsly==2.5.1
     #   weasel
 starlette==0.47.3
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   fastapi
 sudachidict-core==20250825
     # via spacy
@@ -327,7 +327,7 @@ typer==0.19.2
     #   weasel
 typing-extensions==4.15.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   anyio
     #   fastapi
     #   grpcio
@@ -343,13 +343,13 @@ typing-extensions==4.15.0
     #   typer
 urllib3==2.5.0
     # via
-    #   -r requirements/common-requirements.txt
+    #   -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
     #   botocore
     #   docker
     #   minio
     #   requests
 uvicorn==0.35.0
-    # via -r requirements/common-requirements.txt
+    # via -r /Users/andhrelja/Projects/refinery-gateway/requirements/common-requirements.txt
 wasabi==1.1.3
     # via
     #   spacy

requirements/requirements.in

@@ -10,4 +10,11 @@ spacy[ja]==3.7.5
 pyminizip==0.2.6
 rncryptor==3.3.0
 pydantic==2.7.4
-openai==1.31.0
+openai==1.31.0
+opentelemetry-api==1.37.0
+opentelemetry-sdk==1.37.0
+opentelemetry-exporter-otlp-proto-grpc==1.37.0 
+opentelemetry-instrumentation==0.58b0
+opentelemetry-instrumentation-fastapi==0.58b0
+opentelemetry-instrumentation-logging==0.58b0
+prometheus-client==0.23.1

start

@@ -2,11 +2,13 @@
 
 DEBUG_MODE=false
 DEBUG_PORT=15670
+ENABLE_TELEMETRY=false
 
-while getopts d flag
+while getopts dg flag
 do
     case "${flag}" in
         d) DEBUG_MODE=true;;
+        g) ENABLE_TELEMETRY=true;;
     esac
 done
 
@@ -58,13 +60,13 @@ fi
 
 MINIO_ENDPOINT="http://$HOST_IP:7053"
 
-INFERENCE_DIR=${DEV_SETUP_DIR}inference/
-LOG_DIR=${DEV_SETUP_DIR}logs/
+INFERENCE_DIR=${DEV_SETUP_DIR}etc/inference/
+LOG_DIR=${DEV_SETUP_DIR}etc/logs/
 
-CONFIG_DIR=${PWD%/*}/dev-setup/config/
+CONFIG_DIR=${PWD%/*}/dev-setup/etc/config/
 if [ ! -d "$CONFIG_DIR" ] 
 then
-    CONFIG_DIR=${PWD%/*/*}/dev-setup/config/
+    CONFIG_DIR=${PWD%/*/*}/dev-setup/etc/config/
     if [ ! -d "$CONFIG_DIR" ] 
     then
         # to include volume for local development, use the dev-setup inference folder:  
@@ -120,6 +122,7 @@ docker run -d --rm \
 -e KERN_S3_ENDPOINT=${MINIO_ENDPOINT} \
 -e SMTP_HOST=mailhog \
 -e SMTP_PORT=1025 \
+-e ENABLE_TELEMETRY=$ENABLE_TELEMETRY \
 -v "$INFERENCE_DIR":/inference \
 -v "$LOG_DIR":/logs \
 -v "$CONFIG_DIR":/config \