Feature/metrics phase1: per-asset logging + dashboards

.env

@@ -44,3 +44,7 @@ AIOD_NGINX_PORT=80
 #DATA STORAGE
 DATA_PATH=./data
 BACKUP_PATH=./data/backups
+
+#PROMETHEUS and GRAFANA
+AIOD_PROMETHEUS_PORT=9090
+AIOD_GRAFANA_PORT=3000

docker-compose.yaml

@@ -237,6 +237,32 @@ services:
       es_logstash_setup:
         condition: service_completed_successfully
 
+  prometheus:
+    profiles: ["monitoring"]
+    image: prom/prometheus:latest
+    container_name: prometheus
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus_data:/prometheus
+    ports:
+    - "${AIOD_PROMETHEUS_PORT}:9090"
+    restart: unless-stopped
+
+  grafana:
+    profiles: ["monitoring"]
+    image: grafana/grafana:latest
+    container_name: grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin
+    volumes:
+      - grafana_data:/var/lib/grafana
+    depends_on:
+      - prometheus
+    # expose later if you need external access
+    ports:
+    - "${AIOD_GRAFANA_PORT}:3000"
+    restart: unless-stopped
+
   taxonomy:
     profiles: ["taxonomy"]
     container_name: taxonomy
@@ -247,3 +273,7 @@ services:
     depends_on:
       sqlserver:
         condition: service_healthy
+
+volumes:
+  prometheus_data:
+  grafana_data:

docs/metrics.md

@@ -0,0 +1,7 @@
+# Metrics & Monitoring
+
+* **/metrics** – Prometheus exposition created by prometheus_fastapi_instrumentator.
+* **/stats/top/{resource_type}** – JSON list '[asset_id, hits]', success only.
+* **AssetAccessLog** – table schema.
+* Quickstart – 'docker compose --profile monitoring up -d'.
+* Queries – PromQL for per-endpoint, MySQL for per-asset popularity.

prometheus.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 1s
+scrape_configs:
+  - job_name: 'aiod_rest_api'
+    metrics_path: /metrics
+    static_configs:
+      - targets: ['app:8000']  

pyproject.toml

@@ -37,6 +37,7 @@ dependencies = [
     "mysql-connector-python==9.1.0",
     "elasticsearch==8.16.0",
     "jinja2==3.1.4",
+    "prometheus-fastapi-instrumentator==6.1.0",
 ]
 readme = "README.md"
 

src/database/model/access/access_log.py

@@ -0,0 +1,10 @@
+from datetime import datetime
+from sqlmodel import SQLModel, Field
+
+
+class AssetAccessLog(SQLModel, table=True):  # type: ignore[call-arg]
+    id: int | None = Field(default=None, primary_key=True)
+    asset_id: str
+    resource_type: str  # “datasets”, “models”, etc.
+    status: int  # HTTP status code
+    accessed_at: datetime = Field(default_factory=datetime.utcnow, index=True)

src/main.py

@@ -41,6 +41,9 @@
     bookmark_router,
     asset_router,
 )
+from setup_logger import setup_logger
+from prometheus_fastapi_instrumentator import Instrumentator
+from middleware.access_log import AccessLogMiddleware
 from versioning import versions, add_version_to_openapi, add_deprecation_and_sunset_middleware
 
 
@@ -139,23 +142,26 @@ def build_app(*, url_prefix: str = "", version: str = "dev"):
         version="latest",
         **kwargs,
     )
-    add_routes(main_app)
-    main_app.add_exception_handler(HTTPException, http_exception_handler)
-    add_version_to_openapi(main_app, root_path=url_prefix)
-
-    for version, info in versions.items():
-        if info.retired:
-            continue
-        app = FastAPI(
+    versioned_apps = [
+        FastAPI(
             title=f"AIoD Metadata Catalogue {version}",
             version=f"{version}",
             **kwargs,
         )
+        for version, info in versions.items()
+        if not info.retired
+    ]
+    for app in [main_app] + versioned_apps:
         add_routes(app)
         app.add_exception_handler(HTTPException, http_exception_handler)
         add_deprecation_and_sunset_middleware(app)
         add_version_to_openapi(app, root_path=url_prefix)
-        main_app.mount(f"/{version}", app)
+        Instrumentator().instrument(app).expose(app, endpoint="/metrics", include_in_schema=False)
+        app.add_middleware(AccessLogMiddleware)
+
+    for app in versioned_apps:
+        main_app.mount(f"/{app.version}", app)
+
     return main_app
 
 

src/middleware/access_log.py

@@ -0,0 +1,33 @@
+from starlette.middleware.base import BaseHTTPMiddleware
+from starlette.requests import Request
+from starlette.responses import Response
+
+from database.session import DbSession
+from database.model.access.access_log import AssetAccessLog
+
+from middleware.resource_types import all_resource_types
+
+VALID_TYPES = all_resource_types()
+
+
+class AccessLogMiddleware(BaseHTTPMiddleware):
+    """Write one AssetAccessLog row for /datasets/<id> and /models/<id>."""
+
+    async def dispatch(self, request: Request, call_next):
+        response: Response = await call_next(request)
+
+        segments = request.url.path.strip("/").split("/")
+        if len(segments) >= 2 and segments[0] in VALID_TYPES:
+            resource_type = segments[0]
+            asset_id = "/".join(segments[1:])
+
+            entry = AssetAccessLog(
+                asset_id=asset_id,
+                resource_type=resource_type,
+                status=response.status_code,
+            )
+            with DbSession() as sess:
+                sess.add(entry)
+                sess.commit()
+
+        return response

src/middleware/resource_types.py

@@ -0,0 +1,21 @@
+from typing import Set
+from routers import (
+    resource_routers,
+    parent_routers,
+    uploader_routers,
+)
+
+
+def all_resource_types() -> Set[str]:
+    """
+    Gather every plural resource name exposed by *any* router group,
+    e.g. {'datasets', 'ml_models', 'computational_assets', …}.
+    Uses getattr guard so it doesn’t crash when a router lacks the attribute.
+    """
+    router_lists = resource_routers.router_list
+    types: Set[str] = set()
+    for router in router_lists:
+        val = getattr(router, "resource_name_plural", None)
+        if val:
+            types.add(val)
+    return types