feat: add prometheus metrics (#3640)

Author: Panaetius

.dockerignore

@@ -8,4 +8,5 @@
 !.git
 !.gitignore
 !Makefile
+!gunicorn.conf.py
 .git/config

Dockerfile

@@ -15,7 +15,7 @@ RUN apt-get install --no-install-recommends -y build-essential && \
 # time the code changes
 # set the BUILD_CORE_SERVICE to non null to install additional service dependencies
 ARG BUILD_CORE_SERVICE
-COPY pyproject.toml poetry.lock README.rst CHANGES.rst Makefile /code/renku/
+COPY pyproject.toml poetry.lock README.rst CHANGES.rst Makefile gunicorn.conf.py /code/renku/
 COPY .git /code/renku/.git
 COPY renku /code/renku/renku
 WORKDIR /code/renku
@@ -46,6 +46,7 @@ RUN addgroup -gid 1000 shuhitsu && \
     if [ -n "${BUILD_CORE_SERVICE}" ]; then mkdir /svc && chown shuhitsu:shuhitsu /svc ; fi
 
 COPY --from=builder /code/renku /code/renku
+WORKDIR /code/renku
 ENV PATH="${PATH}:/code/renku/.venv/bin"
 
 # shuhitsu (執筆): The "secretary" of the renga, as it were, who is responsible for
@@ -55,5 +56,6 @@ USER shuhitsu
 ENV RENKU_SVC_NUM_WORKERS 4
 ENV RENKU_SVC_NUM_THREADS 8
 ENV RENKU_DISABLE_VERSION_CHECK=1
+ENV PROMETHEUS_MULTIPROC_DIR /tmp
 
 ENTRYPOINT ["tini", "-g", "--", "renku"]

gunicorn.conf.py

@@ -0,0 +1,14 @@
+"""Gunicorn Configuration."""
+import os
+
+from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics
+
+
+def when_ready(server):
+    """Run metrics server on separate port."""
+    GunicornPrometheusMetrics.start_http_server_when_ready(int(os.getenv("METRICS_PORT", "8765")))
+
+
+def child_exit(server, worker):
+    """Properly exit when metrics server stops."""
+    GunicornPrometheusMetrics.mark_process_dead_on_child_exit(worker.pid)

poetry.lock

@@ -114,6 +114,7 @@ redis = { version = ">=3.5.3,<4.6.0,!=4.5.5", optional = true }
 rq = { version = "==1.15.0", optional = true }
 sentry-sdk = { version = ">=1.5.11,<1.26.0", extras = ["flask"],  optional = true }
 walrus = { version = ">=0.8.2,<0.10.0", optional = true }
+prometheus-flask-exporter = "^0.22.4"
 
 [tool.poetry.group.dev.dependencies]
 black = "==23.1.0"
@@ -301,6 +302,7 @@ module = [
     "pexpect",
     "PIL",
     "pluggy",
+    "prometheus_flask_exporter.*",
     "psutil",
     "pyld",
     "pyshacl",

pyproject.toml

@@ -50,6 +50,8 @@ def run_api(addr="0.0.0.0", port=8080, timeout=600):
         "gunicorn",
         "renku.ui.service.entrypoint:app",
         loading_opt,
+        "-c",
+        "gunicorn.conf.py",
         "-b",
         f"{addr}:{port}",
         "--timeout",

renku/ui/cli/service.py

@@ -22,10 +22,12 @@
 import sentry_sdk
 from flask import Flask, Response, jsonify, request, url_for
 from jwt import InvalidTokenError
+from prometheus_flask_exporter.multiprocess import GunicornPrometheusMetrics
 from sentry_sdk.integrations.flask import FlaskIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
 from sentry_sdk.integrations.rq import RqIntegration
 
+from renku.core.util.util import is_test_session_running
 from renku.ui.service.cache import cache
 from renku.ui.service.config import CACHE_DIR, MAX_CONTENT_LENGTH, SENTRY_ENABLED, SENTRY_SAMPLERATE, SERVICE_PREFIX
 from renku.ui.service.errors import (
@@ -74,6 +76,9 @@ def create_app(custom_exceptions=True):
 
     app.config["cache"] = cache
 
+    if not is_test_session_running():
+        GunicornPrometheusMetrics(app)
+
     build_routes(app)
 
     @app.route(SERVICE_PREFIX)