diff --git a/converter/.gitignore b/converter/.gitignore index ecad9c389..f62be6497 100644 --- a/converter/.gitignore +++ b/converter/.gitignore @@ -171,4 +171,7 @@ cython_debug/ .pypirc # Request cache -*_cache.sqlite \ No newline at end of file +*_cache.sqlite + +# prometheus metrics +tmp/prometheus_metrics diff --git a/converter/Dockerfile b/converter/Dockerfile index 0098adc2d..f9231eb20 100644 --- a/converter/Dockerfile +++ b/converter/Dockerfile @@ -51,5 +51,11 @@ HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ ENV CONVERTER_VERSION=${CONVERTER_VERSION} +# Flask Prometheus Exporter setup +RUN mkdir /tmp/prometheus_metrics +ENV PROMETHEUS_MULTIPROC_DIR=/tmp/prometheus_metrics + +ENV FLASK_ENV=production + # Use Gunicorn for production deployment -CMD ["uv", "run", "--no-dev", "gunicorn", "-w", "4", "-b", "0.0.0.0:8080", "converter.converter:app"] +CMD ["uv", "run", "--no-dev", "gunicorn", "-c", "gunicorn.conf.py", "-w", "4", "-b", "0.0.0.0:8080", "converter.converter:app"] diff --git a/converter/README.md b/converter/README.md index 58d7740a0..d54b10eb5 100644 --- a/converter/README.md +++ b/converter/README.md @@ -55,17 +55,22 @@ Note : the tests download files (json samples & schemas) using the Github API. T Development mode: ```bash -# In converter/, run the command: +# In converter/, run the commands: FLASK_APP=converter.converter \ FLASK_ENV=development \ FLASK_DEBUG=1 \ uv run python -m flask run --port 8080 ``` +*Note :* enable prometheus metrics, add `DEBUG_METRICS=1` in the above command. + Production mode (using Gunicorn): ```bash -gunicorn -w 4 -b 0.0.0.0:8080 converter.converter:app +# In converter/, run the commands: +mkdir -p ./tmp/prometheus_metrics + +PROMETHEUS_MULTIPROC_DIR=./tmp/prometheus_metrics gunicorn -c gunicorn.conf.py -w 4 -b 0.0.0.0:8080 converter.converter:app ``` ### Controlling Logging Level diff --git a/converter/converter/converter.py b/converter/converter/converter.py index 545ace2e3..57393246a 100644 --- a/converter/converter/converter.py +++ b/converter/converter/converter.py @@ -1,7 +1,8 @@ from flask import Flask, request, jsonify, g import logging -from werkzeug.middleware.dispatcher import DispatcherMiddleware -from prometheus_client import make_wsgi_app, Histogram +import os +from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics +from prometheus_flask_exporter import PrometheusMetrics from converter.conversion_strategy.conversion_strategy import conversion_strategy from converter.utils import ( @@ -15,17 +16,15 @@ configure_logging() app = Flask(__name__) -logger = logging.getLogger(__name__) -# Add prometheus wsgi middleware to route /metrics requests -# ignore typing issue with reassigning method -app.wsgi_app = DispatcherMiddleware(app.wsgi_app, {"/metrics": make_wsgi_app()}) # type: ignore[assignment] +is_prod = os.getenv("FLASK_ENV") == "production" +if is_prod: + metrics = GunicornInternalPrometheusMetrics(app) +else: + metrics = PrometheusMetrics(app) -convertion_timer = Histogram( - "conversion_duration_seconds", - "The number of seconds it took to the /convert endpoint to answer", -) +logger = logging.getLogger(__name__) def raise_error(message, code: int = 400): @@ -33,8 +32,26 @@ def raise_error(message, code: int = 400): return jsonify({"error": message}), code +def extract_message_type_from_payload(): + try: + data = request.get_json(silent=True) or {} + edxl_json = data.get("edxl") + message_content = extract_message_content(edxl_json) + return extract_message_type_from_message_content(message_content) + except Exception: + return "unknownMessageType" + + @app.route("/convert", methods=["POST"]) -@convertion_timer.time() +@metrics.do_not_track() +@metrics.histogram( + "conversion_duration_seconds", + "The number of seconds it took to the /convert endpoint to answer", + labels={ + "status": lambda r: r.status_code, + "message_type": extract_message_type_from_payload, + }, +) def convert(): if not request.is_json: return raise_error("Content-Type must be application/json") diff --git a/converter/gunicorn.conf.py b/converter/gunicorn.conf.py new file mode 100644 index 000000000..a2bb52e9c --- /dev/null +++ b/converter/gunicorn.conf.py @@ -0,0 +1,5 @@ +from prometheus_flask_exporter.multiprocess import GunicornInternalPrometheusMetrics + + +def child_exit(_, worker): + GunicornInternalPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid) diff --git a/converter/pyproject.toml b/converter/pyproject.toml index a02002a49..06e128b0a 100644 --- a/converter/pyproject.toml +++ b/converter/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "flask>=3.1.2", "gunicorn>=23.0.0", "jsonpath-ng>=1.7.0", - "prometheus-client>=0.23.1", + "prometheus-flask-exporter>=0.23.2", "pydantic>=2.11.9", "python-json-logger>=4.0.0", "pyyaml>=6.0.2", @@ -59,5 +59,5 @@ dev = [ ] [[tool.mypy.overrides]] -module = ["snapshottest.*", "jsonpath_ng.*"] +module = ["snapshottest.*", "jsonpath_ng.*", "prometheus_flask_exporter.*"] ignore_missing_imports = true diff --git a/converter/uv.lock b/converter/uv.lock index 67f47b3bf..665ee41fa 100644 --- a/converter/uv.lock +++ b/converter/uv.lock @@ -178,7 +178,7 @@ dependencies = [ { name = "flask" }, { name = "gunicorn" }, { name = "jsonpath-ng" }, - { name = "prometheus-client" }, + { name = "prometheus-flask-exporter" }, { name = "pydantic" }, { name = "python-json-logger" }, { name = "pyyaml" }, @@ -207,7 +207,7 @@ requires-dist = [ { name = "flask", specifier = ">=3.1.2" }, { name = "gunicorn", specifier = ">=23.0.0" }, { name = "jsonpath-ng", specifier = ">=1.7.0" }, - { name = "prometheus-client", specifier = ">=0.23.1" }, + { name = "prometheus-flask-exporter", specifier = ">=0.23.2" }, { name = "pydantic", specifier = ">=2.11.9" }, { name = "python-json-logger", specifier = ">=4.0.0" }, { name = "pyyaml", specifier = ">=6.0.2" }, @@ -690,6 +690,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b8/db/14bafcb4af2139e046d03fd00dea7873e48eafe18b7d2797e73d6681f210/prometheus_client-0.23.1-py3-none-any.whl", hash = "sha256:dd1913e6e76b59cfe44e7a4b83e01afc9873c1bdfd2ed8739f1e76aeca115f99", size = 61145, upload-time = "2025-09-18T20:47:23.875Z" }, ] +[[package]] +name = "prometheus-flask-exporter" +version = "0.23.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flask" }, + { name = "prometheus-client" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/40/736bd2cb19b065cea395deb57b5b520a2df105dab92af3fb200ee560ad92/prometheus_flask_exporter-0.23.2.tar.gz", hash = "sha256:41fc9bbd7d48cc958ed8384aacf60c3621d9e903768be61c4e7f0c63872eaf1a", size = 31801, upload-time = "2025-03-11T23:05:30.409Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/04/486040e241708a723ee7673d98733c35f2cf081f63ced0091124b8572177/prometheus_flask_exporter-0.23.2-py3-none-any.whl", hash = "sha256:94922a636d4c1d8b68e1ee605c30a23e9bbb0b21756df8222aa919634871784c", size = 19002, upload-time = "2025-03-11T23:05:29.176Z" }, +] + [[package]] name = "pydantic" version = "2.11.9"