From 6d228b8815a9c99cd16ee5d60b01d93de9e40566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Di=20Domenico?= Date: Mon, 12 Jan 2026 16:55:52 +0100 Subject: [PATCH 1/7] chore(converter): remove prometheus client & add flask prometheus exporter --- converter/pyproject.toml | 2 +- converter/uv.lock | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/converter/pyproject.toml b/converter/pyproject.toml index a02002a49..51404fba4 100644 --- a/converter/pyproject.toml +++ b/converter/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "flask>=3.1.2", "gunicorn>=23.0.0", "jsonpath-ng>=1.7.0", - "prometheus-client>=0.23.1", + "prometheus-flask-exporter>=0.23.2", "pydantic>=2.11.9", "python-json-logger>=4.0.0", "pyyaml>=6.0.2", diff --git a/converter/uv.lock b/converter/uv.lock index 67f47b3bf..665ee41fa 100644 --- a/converter/uv.lock +++ b/converter/uv.lock @@ -178,7 +178,7 @@ dependencies = [ { name = "flask" }, { name = "gunicorn" }, { name = "jsonpath-ng" }, - { name = "prometheus-client" }, + { name = "prometheus-flask-exporter" }, { name = "pydantic" }, { name = "python-json-logger" }, { name = "pyyaml" }, @@ -207,7 +207,7 @@ requires-dist = [ { name = "flask", specifier = ">=3.1.2" }, { name = "gunicorn", specifier = ">=23.0.0" }, { name = "jsonpath-ng", specifier = ">=1.7.0" }, - { name = "prometheus-client", specifier = ">=0.23.1" }, + { name = "prometheus-flask-exporter", specifier = ">=0.23.2" }, { name = "pydantic", specifier = ">=2.11.9" }, { name = "python-json-logger", specifier = ">=4.0.0" }, { name = "pyyaml", specifier = ">=6.0.2" }, @@ -690,6 +690,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" }, ] +[[package]] +name = "prometheus-flask-exporter" +version = "0.23.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flask" }, + { name = "prometheus-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/40/736bd2cb19b065cea395deb57b5b520a2df105dab92af3fb200ee560ad92/prometheus_flask_exporter-0.23.2.tar.gz", hash = "sha256:41fc9bbd7d48cc958ed8384aacf60c3621d9e903768be61c4e7f0c63872eaf1a", size = 31801, upload-time = "2025-03-11T23:05:30.409Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/04/486040e241708a723ee7673d98733c35f2cf081f63ced0091124b8572177/prometheus_flask_exporter-0.23.2-py3-none-any.whl", hash = "sha256:94922a636d4c1d8b68e1ee605c30a23e9bbb0b21756df8222aa919634871784c", size = 19002, upload-time = "2025-03-11T23:05:29.176Z" }, +] + [[package]] name = "pydantic" version = "2.11.9" From b4e9c781a904c8bf974f529cbc19a6be0c785eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Di=20Domenico?= Date: Mon, 12 Jan 2026 16:56:25 +0100 Subject: [PATCH 2/7] chore(converter): setup flask prometheus exporter --- converter/.gitignore | 5 ++++- converter/Dockerfile | 8 +++++++- converter/README.md | 7 +++++-- converter/converter/converter.py | 26 +++++++++++++++----------- converter/gunicorn.conf.py | 5 +++++ 5 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 converter/gunicorn.conf.py diff --git a/converter/.gitignore b/converter/.gitignore index ecad9c389..f62be6497 100644 --- a/converter/.gitignore +++ b/converter/.gitignore @@ -171,4 +171,7 @@ cython_debug/ .pypirc # Request cache -*_cache.sqlite \ No newline at end of file +*_cache.sqlite + +# prometheus metrics +tmp/prometheus_metrics diff --git a/converter/Dockerfile b/converter/Dockerfile index 0098adc2d..f9231eb20 100644 --- a/converter/Dockerfile +++ b/converter/Dockerfile @@ -51,5 +51,11 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ ENV CONVERTER_VERSION=${CONVERTER_VERSION} +# Flask Prometheus Exporter setup +RUN mkdir /tmp/prometheus_metrics +ENV PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_metrics + +ENV FLASK_ENV=production + # Use Gunicorn for production deployment -CMD ["uv", "run", "--no-dev", "gunicorn", "-w", "4", "-b", "0.0.0.0:8080", "converter.converter:app"] +CMD ["uv", "run", "--no-dev", "gunicorn", "-c", "gunicorn.conf.py", "-w", "4", "-b", "0.0.0.0:8080", "converter.converter:app"] diff --git a/converter/README.md b/converter/README.md index 58d7740a0..49f16ec95 100644 --- a/converter/README.md +++ b/converter/README.md @@ -55,7 +55,7 @@ Note : the tests download files (json samples & schemas) using the Github API. T Development mode: ```bash -# In converter/, run the command: +# In converter/, run the commands: FLASK_APP=converter.converter \ FLASK_ENV=development \ FLASK_DEBUG=1 \ @@ -65,7 +65,10 @@ uv run python -m flask run --port 8080 Production mode (using Gunicorn): ```bash -gunicorn -w 4 -b 0.0.0.0:8080 converter.converter:app +# In converter/, run the commands: +mkdir -p ./tmp/prometheus_metrics + +PROMETHEUS_MULTIPROC_DIR=./tmp/prometheus_metrics gunicorn -c gunicorn.conf.py -w 4 -b 0.0.0.0:8080 converter.converter:app ``` ### Controlling Logging Level diff --git a/converter/converter/converter.py b/converter/converter/converter.py index 545ace2e3..9087853fc 100644 --- a/converter/converter/converter.py +++ b/converter/converter/converter.py @@ -1,7 +1,8 @@ from flask import Flask, request, jsonify, g import logging -from werkzeug.middleware.dispatcher import DispatcherMiddleware -from prometheus_client import make_wsgi_app, Histogram +import os +from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics +from prometheus_flask_exporter import PrometheusMetrics from converter.conversion_strategy.conversion_strategy import conversion_strategy from converter.utils import ( @@ -15,17 +16,15 @@ configure_logging() app = Flask(__name__) -logger = logging.getLogger(__name__) -# Add prometheus wsgi middleware to route /metrics requests -# ignore typing issue with reassigning method -app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {"/metrics": make_wsgi_app()}) # type: ignore[assignment] +is_prod = os.getenv("FLASK_ENV") == "production" +if is_prod: + metrics = GunicornInternalPrometheusMetrics(app) +else: + metrics = PrometheusMetrics(app) -convertion_timer = Histogram( - "conversion_duration_seconds", - "The number of seconds it took to the /convert endpoint to answer", -) +logger = logging.getLogger(__name__) def raise_error(message, code: int = 400): @@ -34,7 +33,12 @@ def raise_error(message, code: int = 400): @app.route("/convert", methods=["POST"]) -@convertion_timer.time() +@metrics.do_not_track() +@metrics.histogram( + "conversion_duration_seconds", + "The number of seconds it took to the /convert endpoint to answer", + labels={"status": lambda r: r.status_code}, +) def convert(): if not request.is_json: return raise_error("Content-Type must be application/json") diff --git a/converter/gunicorn.conf.py b/converter/gunicorn.conf.py new file mode 100644 index 000000000..a2bb52e9c --- /dev/null +++ b/converter/gunicorn.conf.py @@ -0,0 +1,5 @@ +from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics + + +def child_exit(_, worker): + GunicornInternalPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid) From 9893ef09b7ea462c06e2df8ca739045569abfe91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Di=20Domenico?= Date: Mon, 12 Jan 2026 18:20:14 +0100 Subject: [PATCH 3/7] doc(converter): add instruction to debug metrics --- converter/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/converter/README.md b/converter/README.md index 49f16ec95..d54b10eb5 100644 --- a/converter/README.md +++ b/converter/README.md @@ -62,6 +62,8 @@ FLASK_DEBUG=1 \ uv run python -m flask run --port 8080 ``` +*Note :* enable prometheus metrics, add `DEBUG_METRICS=1` in the above command. + Production mode (using Gunicorn): ```bash From 8cadecf0c1f6f0e736b4befde3e1740a847f8499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Di=20Domenico?= Date: Tue, 13 Jan 2026 10:16:31 +0100 Subject: [PATCH 4/7] fix(converter): ignore flask-prometheus-exporter missing typing --- converter/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/converter/pyproject.toml b/converter/pyproject.toml index 51404fba4..06e128b0a 100644 --- a/converter/pyproject.toml +++ b/converter/pyproject.toml @@ -59,5 +59,5 @@ dev = [ ] [[tool.mypy.overrides]] -module = ["snapshottest.*", "jsonpath_ng.*"] +module = ["snapshottest.*", "jsonpath_ng.*", "prometheus_flask_exporter.*"] ignore_missing_imports = true From c05907f76af06a0ebe6e7385e2d41f3de480fa72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Di=20Domenico?= Date: Tue, 13 Jan 2026 12:00:51 +0100 Subject: [PATCH 5/7] chore(converter): use message type as label for conversin duration metric --- converter/converter/converter.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/converter/converter/converter.py b/converter/converter/converter.py index 9087853fc..067a90e9b 100644 --- a/converter/converter/converter.py +++ b/converter/converter/converter.py @@ -32,12 +32,22 @@ def raise_error(message, code: int = 400): return jsonify({"error": message}), code +def extract_message_type_from_paylaod(): + data = request.get_json(silent=True) or {} + edxl_json = data.get("edxl") + message_content = extract_message_content(edxl_json) + return extract_message_type_from_message_content(message_content) + + @app.route("/convert", methods=["POST"]) @metrics.do_not_track() @metrics.histogram( "conversion_duration_seconds", "The number of seconds it took to the /convert endpoint to answer", - labels={"status": lambda r: r.status_code}, + labels={ + "status": lambda r: r.status_code, + "message_type": extract_message_type_from_paylaod, + }, ) def convert(): if not request.is_json: From 5573046aab457f4bc55af0f9efb8e229c4a43990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Di=20Domenico?= Date: Tue, 13 Jan 2026 15:39:23 +0100 Subject: [PATCH 6/7] fix(converter): handle error in lebel extraction method --- converter/converter/converter.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/converter/converter/converter.py b/converter/converter/converter.py index 067a90e9b..5c8a277a1 100644 --- a/converter/converter/converter.py +++ b/converter/converter/converter.py @@ -33,10 +33,13 @@ def raise_error(message, code: int = 400): def extract_message_type_from_paylaod(): - data = request.get_json(silent=True) or {} - edxl_json = data.get("edxl") - message_content = extract_message_content(edxl_json) - return extract_message_type_from_message_content(message_content) + try: + data = request.get_json(silent=True) or {} + edxl_json = data.get("edxl") + message_content = extract_message_content(edxl_json) + return extract_message_type_from_message_content(message_content) + except Exception: + return "unknownMessageType" @app.route("/convert", methods=["POST"]) From 157cf84405b37167dc7aa03c15fb977a19315bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Di=20Domenico?= Date: Fri, 16 Jan 2026 09:16:02 +0100 Subject: [PATCH 7/7] fix(converter): typo in extract method name --- converter/converter/converter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/converter/converter/converter.py b/converter/converter/converter.py index 5c8a277a1..57393246a 100644 --- a/converter/converter/converter.py +++ b/converter/converter/converter.py @@ -32,7 +32,7 @@ def raise_error(message, code: int = 400): return jsonify({"error": message}), code -def extract_message_type_from_paylaod(): +def extract_message_type_from_payload(): try: data = request.get_json(silent=True) or {} edxl_json = data.get("edxl") @@ -49,7 +49,7 @@ def extract_message_type_from_paylaod(): "The number of seconds it took to the /convert endpoint to answer", labels={ "status": lambda r: r.status_code, - "message_type": extract_message_type_from_paylaod, + "message_type": extract_message_type_from_payload, }, ) def convert():