[DOP-19925] Add GET v1/monitoring/metrics endpoint

Ilyas Gasanov · Ilyas Gasanov · commit 366cd6fd3a36 · 2024-11-11T10:18:43.000+03:00
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -36,6 +36,13 @@ services:
       context: .
     ports:
       - 8000:8000
+    # PROMETHEUS_MULTIPROC_DIR is required for multiple workers, see:
+    # https://prometheus.github.io/client_python/multiprocess/
+    environment:
+      PROMETHEUS_MULTIPROC_DIR: /tmp/prometheus-metrics
+    # tmpfs dir is cleaned up each container restart
+    tmpfs:
+      - /tmp/prometheus-metrics
     env_file: .env.docker
     depends_on:
       db:
diff --git a/docs/backend/configuration/index.rst b/docs/backend/configuration/index.rst
@@ -13,6 +13,7 @@ Configuration
     logging
     cors
     debug
+    monitoring
 
 .. autopydantic_settings:: syncmaster.settings.Settings
 .. autopydantic_settings:: syncmaster.settings.server.ServerSettings
diff --git a/docs/backend/configuration/monitoring.rst b/docs/backend/configuration/monitoring.rst
@@ -0,0 +1,16 @@
+.. _configuration-server-monitoring:
+
+Setup monitoring
+================
+
+REST API server provides the following endpoints with Prometheus compatible metrics:
+
+* ``GET /monitoring/metrics`` - server metrics, like number of requests per path and response status, CPU and RAM usage, and so on.
+
+.. dropdown:: Example
+
+    .. literalinclude:: ../../_static/metrics.prom
+
+These endpoints are enabled and configured using settings below:
+
+.. autopydantic_model:: syncmaster.settings.server.monitoring.MonitoringSettings
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -67,13 +67,15 @@ coloredlogs = {version = "*", optional = true}
 python-json-logger = {version = "*", optional = true}
 asyncpg = { version = ">=0.29,<0.31", optional = true }
 apscheduler = { version = "^3.10.4", optional = true }
+starlette-exporter = {version = "^0.23.0", optional = true}
 
 [tool.poetry.extras]
 backend = [
     "pydantic-settings",
     "sqlalchemy",
     "sqlalchemy-utils",
     "fastapi",
+    "starlette-exporter",
     "uvicorn",
     "alembic",
     "python-multipart",
diff --git a/syncmaster/backend/api/monitoring.py b/syncmaster/backend/api/monitoring.py
@@ -3,10 +3,11 @@
 from fastapi import APIRouter
 
 from syncmaster.errors.registration import get_error_responses
+from syncmaster.schemas import PingResponse
 
-router = APIRouter(tags=["monitoring"], prefix="/monitoring", responses=get_error_responses())
+router = APIRouter(tags=["Monitoring"], prefix="/monitoring", responses=get_error_responses())
 
 
 @router.get("/ping")
-async def ping():
-    return {"status": "ok"}
+async def ping() -> PingResponse:
+    return PingResponse()
diff --git a/syncmaster/backend/middlewares/__init__.py b/syncmaster/backend/middlewares/__init__.py
@@ -5,6 +5,9 @@
 
 from syncmaster.backend.middlewares.cors import apply_cors_middleware
 from syncmaster.backend.middlewares.logging import setup_logging
+from syncmaster.backend.middlewares.monitoring.metrics import (
+    apply_monitoring_metrics_middleware,
+)
 from syncmaster.backend.middlewares.request_id import apply_request_id_middleware
 from syncmaster.settings import Settings
 
@@ -19,6 +22,7 @@ def apply_middlewares(
         setup_logging(settings.server.logging.get_log_config_path())
 
     apply_cors_middleware(application, settings.server.cors)
+    apply_monitoring_metrics_middleware(application, settings.server.monitoring)
     apply_request_id_middleware(application, settings.server.request_id)
 
     return application
diff --git a/syncmaster/backend/middlewares/monitoring/__init__.py b/syncmaster/backend/middlewares/monitoring/__init__.py
@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from syncmaster.backend.middlewares.monitoring.metrics import (
+    apply_monitoring_metrics_middleware,
+)
+
+__all__ = ["apply_monitoring_metrics_middleware"]
diff --git a/syncmaster/backend/middlewares/monitoring/metrics.py b/syncmaster/backend/middlewares/monitoring/metrics.py
@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from fastapi import APIRouter, FastAPI
+from starlette.responses import PlainTextResponse
+from starlette_exporter import PrometheusMiddleware, handle_metrics
+
+from syncmaster.backend.utils.slug import slugify
+from syncmaster.settings.server.monitoring import MonitoringSettings
+
+DEFAULT_SKIP_PATHS = {
+    "/monitoring/metrics",
+    "/monitoring/ping",
+    "/static",
+    "/docs",
+    "/redoc",
+    "/openapi.json",
+    "/favicon.ico",
+}
+
+router = APIRouter(tags=["Monitoring"], prefix="/monitoring")
+
+
+router.get(
+    "/metrics",
+    summary="Get server metrics",
+    description="Return server metrics in Prometheus format, using Starlette Exporter",
+    response_class=PlainTextResponse,
+)(handle_metrics)
+
+
+def apply_monitoring_metrics_middleware(app: FastAPI, settings: MonitoringSettings) -> FastAPI:
+    """Add monitoring metrics middleware & endpoint to the application."""
+    if not settings.enabled:
+        return app
+
+    skip_paths = DEFAULT_SKIP_PATHS | settings.skip_paths
+    app.add_middleware(
+        PrometheusMiddleware,
+        app_name=slugify(app.title),
+        skip_paths=skip_paths,
+        **settings.model_dump(exclude={"enabled", "skip_paths"}),
+    )
+    app.include_router(router)
+    return app
diff --git a/syncmaster/backend/utils/__init__.py b/syncmaster/backend/utils/__init__.py
@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
diff --git a/syncmaster/backend/utils/slug.py b/syncmaster/backend/utils/slug.py
@@ -0,0 +1,8 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+def slugify(text: str) -> str:
+    """Convert ``Some value`` to ``some-value``.
+
+    Used to convert ``FastAPI.title`` to short name which can be used as label for Prometheus metrics.
+    """
+    return text.lower().strip().replace(" ", "-")
diff --git a/syncmaster/schemas/__init__.py b/syncmaster/schemas/__init__.py
@@ -1,2 +1,7 @@
 # SPDX-FileCopyrightText: 2023-2024 MTS PJSC
 # SPDX-License-Identifier: Apache-2.0
+from syncmaster.schemas.ping import PingResponse
+
+__all__ = [
+    "PingResponse",
+]
diff --git a/syncmaster/schemas/ping.py b/syncmaster/schemas/ping.py
@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel
+
+
+class PingResponse(BaseModel):
+    """Ping result"""
+
+    status: Literal["ok"] = "ok"
diff --git a/syncmaster/settings/server/__init__.py b/syncmaster/settings/server/__init__.py
@@ -7,6 +7,7 @@
 
 from syncmaster.settings.log import LoggingSettings
 from syncmaster.settings.server.cors import CORSSettings
+from syncmaster.settings.server.monitoring import MonitoringSettings
 from syncmaster.settings.server.request_id import RequestIDSettings
 
 
@@ -42,3 +43,7 @@ class ServerSettings(BaseModel):
         default_factory=CORSSettings,
         description=":ref:`CORS settings <backend-configuration-cors>`",
     )
+    monitoring: MonitoringSettings = Field(
+        default_factory=MonitoringSettings,
+        description=":ref:`Monitoring settings <backend-configuration-monitoring>`",
+    )
diff --git a/syncmaster/settings/server/monitoring.py b/syncmaster/settings/server/monitoring.py
@@ -0,0 +1,65 @@
+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC
+# SPDX-License-Identifier: Apache-2.0
+import textwrap
+
+from pydantic import BaseModel, ConfigDict, Field
+
+
+class MonitoringSettings(BaseModel):
+    """Monitoring Settings.
+
+    See `starlette-exporter <https://github.com/stephenhillier/starlette_exporter#options>`_ documentation.
+
+    .. note::
+
+        You can pass here any extra option supported by ``starlette-exporter``,
+        even if it is not mentioned in documentation.
+
+    Examples
+    --------
+
+    .. code-block:: bash
+
+        SYNCMASTER__SERVER__MONITORING__ENABLED=True
+        SYNCMASTER__SERVER__MONITORING__SKIP_PATHS=["/some/path"]
+        SYNCMASTER__SERVER__MONITORING__SKIP_METHODS=["OPTIONS"]
+    """
+
+    enabled: bool = Field(default=True, description="Set to ``True`` to enable middleware")
+    labels: dict[str, str] = Field(
+        default_factory=dict,
+        description="""Custom labels added to all metrics, e.g. ``{"instance": "production"}``""",
+    )
+    skip_paths: set[str] = Field(
+        default_factory=set,
+        description="Custom paths should be skipped from metrics, like ``/some/endpoint``",
+    )
+    skip_methods: set[str] = Field(
+        default={"OPTIONS"},
+        description="HTTP methods which should be excluded from metrics",
+    )
+    # Starting from starlette-exporter 0.18, group_paths and filter_unhandled_paths are set to True by default:
+    # https://github.com/stephenhillier/starlette_exporter/issues/79
+    group_paths: bool = Field(
+        default=True,
+        description=textwrap.dedent(
+            """
+            If ``True`` (recommended), add request path to metrics literally as described
+            in OpenAPI schema, e.g. ``/namespaces/{id}``, without substitution with path real values.
+
+            If ``False``, all real request paths to metrics, e.g. ``/namespaces/123``.
+            """,
+        ),
+    )
+    filter_unhandled_paths: bool = Field(
+        default=True,
+        description=textwrap.dedent(
+            """
+            If ``True``, add metrics for paths only mentioned in OpenAPI schema.
+
+            If ``False``, add all requested paths to metrics.
+            """,
+        ),
+    )
+
+    model_config = ConfigDict(extra="allow")

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# SPDX-FileCopyrightText: 2023-2024 MTS PJSC`
	`2`	`+# SPDX-License-Identifier: Apache-2.0`