Merge pull request #357 from ansforge/converter/fix-prometheus-metrics-export

Converter/fix prometheus metrics export

Author: Clemsazert

converter/.gitignore

@@ -171,4 +171,7 @@ cython_debug/
 .pypirc
 
 # Request cache
-*_cache.sqlite
+*_cache.sqlite
+
+# prometheus metrics
+tmp/prometheus_metrics

converter/Dockerfile

@@ -51,5 +51,11 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
 
 ENV CONVERTER_VERSION=${CONVERTER_VERSION}
 
+# Flask Prometheus Exporter setup
+RUN mkdir /tmp/prometheus_metrics
+ENV PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_metrics
+
+ENV FLASK_ENV=production
+
 # Use Gunicorn for production deployment
-CMD ["uv", "run", "--no-dev", "gunicorn", "-w", "4", "-b", "0.0.0.0:8080", "converter.converter:app"]
+CMD ["uv", "run", "--no-dev", "gunicorn", "-c", "gunicorn.conf.py", "-w", "4", "-b", "0.0.0.0:8080", "converter.converter:app"]

converter/README.md

@@ -55,17 +55,22 @@ Note : the tests download files (json samples & schemas) using the Github API. T
 Development mode:
 
 ```bash
-# In converter/, run the command:
+# In converter/, run the commands:
 FLASK_APP=converter.converter \
 FLASK_ENV=development \
 FLASK_DEBUG=1 \
 uv run python -m flask run --port 8080
 ```
 
+*Note :* enable prometheus metrics, add `DEBUG_METRICS=1` in the above command.
+
 Production mode (using Gunicorn):
 
 ```bash
-gunicorn -w 4 -b 0.0.0.0:8080 converter.converter:app
+# In converter/, run the commands:
+mkdir -p ./tmp/prometheus_metrics
+
+PROMETHEUS_MULTIPROC_DIR=./tmp/prometheus_metrics gunicorn -c gunicorn.conf.py -w 4 -b 0.0.0.0:8080 converter.converter:app
 ```
 
 ### Controlling Logging Level

converter/converter/converter.py

@@ -1,7 +1,8 @@
 from flask import Flask, request, jsonify, g
 import logging
-from werkzeug.middleware.dispatcher import DispatcherMiddleware
-from prometheus_client import make_wsgi_app, Histogram
+import os
+from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics
+from prometheus_flask_exporter import PrometheusMetrics
 
 from converter.conversion_strategy.conversion_strategy import conversion_strategy
 from converter.utils import (
@@ -15,26 +16,42 @@
 configure_logging()
 
 app = Flask(__name__)
-logger = logging.getLogger(__name__)
 
-# Add prometheus wsgi middleware to route /metrics requests
-# ignore typing issue with reassigning method
-app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {"/metrics": make_wsgi_app()})  # type: ignore[assignment]
+is_prod = os.getenv("FLASK_ENV") == "production"
 
+if is_prod:
+    metrics = GunicornInternalPrometheusMetrics(app)
+else:
+    metrics = PrometheusMetrics(app)
 
-convertion_timer = Histogram(
-    "conversion_duration_seconds",
-    "The number of seconds it took to the /convert endpoint to answer",
-)
+logger = logging.getLogger(__name__)
 
 
 def raise_error(message, code: int = 400):
     logger.error(message)
     return jsonify({"error": message}), code
 
 
+def extract_message_type_from_payload():
+    try:
+        data = request.get_json(silent=True) or {}
+        edxl_json = data.get("edxl")
+        message_content = extract_message_content(edxl_json)
+        return extract_message_type_from_message_content(message_content)
+    except Exception:
+        return "unknownMessageType"
+
+
 @app.route("/convert", methods=["POST"])
-@convertion_timer.time()
+@metrics.do_not_track()
+@metrics.histogram(
+    "conversion_duration_seconds",
+    "The number of seconds it took to the /convert endpoint to answer",
+    labels={
+        "status": lambda r: r.status_code,
+        "message_type": extract_message_type_from_payload,
+    },
+)
 def convert():
     if not request.is_json:
         return raise_error("Content-Type must be application/json")

converter/gunicorn.conf.py

@@ -0,0 +1,5 @@
+from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics
+
+
+def child_exit(_, worker):
+    GunicornInternalPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid)

converter/pyproject.toml

@@ -8,7 +8,7 @@ dependencies = [
     "flask>=3.1.2",
     "gunicorn>=23.0.0",
     "jsonpath-ng>=1.7.0",
-    "prometheus-client>=0.23.1",
+    "prometheus-flask-exporter>=0.23.2",
     "pydantic>=2.11.9",
     "python-json-logger>=4.0.0",
     "pyyaml>=6.0.2",
@@ -59,5 +59,5 @@ dev = [
 ]
 
 [[tool.mypy.overrides]]
-module = ["snapshottest.*", "jsonpath_ng.*"]
+module = ["snapshottest.*", "jsonpath_ng.*", "prometheus_flask_exporter.*"]
 ignore_missing_imports = true