Provide seperate documentation pages for each API version (#577)

* Move addition of versioning to submodule

* Make use of the new prefix convention to improve checks

* Allow API versions to be mounted with updated openapi schema

* Move adding of versions up to the main application

* Remove versioning from router move it to higher level

* Remove versioning since it's moved up

* Remove remainder of versioning

* Add support for `root_path`

* Fix the middleware router to not rely on request path

* Add additional valid redirect urls to work with versioned docs

* Allow removing endpoints from a (hard-coded) configuration

* Add minimal UI for switching between docs

* Add favicon to the doc pages

* Add basic styling to buttons - bug with incorrect openapi

* Make sure docs point to right openapi spec

* Update logging to show configuration

* Support for root_path for links to versioned documentation

* Allow servers to be absent, fixes bug for main app without root_path

* Add gaps between buttons

* Do not show 'retired' api docs

* Make versioning metadata configurable

* update redoc openapi reference

* Remove f prefix since there is no interpolation

* Add clarification

* configure oauth in custom swagger pages so you can log in

Author: PGijsbers

authentication/import/aiod-realm.json

@@ -602,7 +602,7 @@
     "enabled" : true,
     "alwaysDisplayInConsole" : false,
     "clientAuthenticatorType" : "client-secret",
-    "redirectUris" : [ "http://localhost/docs/oauth2-redirect" ],
+    "redirectUris" : [ "http://localhost/v1/docs/oauth2-redirect", "http://localhost/v3/docs/oauth2-redirect", "http://localhost/docs/oauth2-redirect", "http://localhost/v2/docs/oauth2-redirect" ],
     "webOrigins" : [ "http://localhost:8000" ],
     "notBefore" : 0,
     "bearerOnly" : false,
@@ -1224,7 +1224,7 @@
       "subType" : "authenticated",
       "subComponents" : { },
       "config" : {
-        "allowed-protocol-mapper-types" : [ "oidc-full-name-mapper", "saml-user-attribute-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-address-mapper", "oidc-usermodel-property-mapper", "oidc-usermodel-attribute-mapper", "saml-role-list-mapper", "saml-user-property-mapper" ]
+        "allowed-protocol-mapper-types" : [ "oidc-full-name-mapper", "saml-user-property-mapper", "saml-user-attribute-mapper", "oidc-address-mapper", "oidc-usermodel-property-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-usermodel-attribute-mapper", "saml-role-list-mapper" ]
       }
     }, {
       "id" : "10f8b9b2-1038-4c98-b7a5-a9ac88fed69e",
@@ -1266,7 +1266,7 @@
       "subType" : "anonymous",
       "subComponents" : { },
       "config" : {
-        "allowed-protocol-mapper-types" : [ "oidc-usermodel-property-mapper", "saml-user-property-mapper", "saml-user-attribute-mapper", "oidc-usermodel-attribute-mapper", "oidc-address-mapper", "saml-role-list-mapper", "oidc-full-name-mapper", "oidc-sha256-pairwise-sub-mapper" ]
+        "allowed-protocol-mapper-types" : [ "oidc-usermodel-attribute-mapper", "oidc-address-mapper", "saml-role-list-mapper", "oidc-sha256-pairwise-sub-mapper", "oidc-usermodel-property-mapper", "saml-user-attribute-mapper", "oidc-full-name-mapper", "saml-user-property-mapper" ]
       }
     }, {
       "id" : "1d21f027-e0ae-4b80-b95e-f21d9426f115",

docker-compose.yaml

@@ -7,6 +7,7 @@ services:
     container_name: apiserver
     volumes:
       - ./src/config.override.toml:/app/config.override.toml:ro
+      - ./src/versions.toml:/app/versions.toml:ro
     environment:
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
       - REVIEWER_ROLE_NAME=$REVIEWER_ROLE_NAME

src/config.default.toml

@@ -3,6 +3,10 @@
 
 domain = "https://api.aiod.eu/"
 
+[configuration]
+# paths to other configuration files, relative to this file
+versions = "versions.toml"
+
 # Information on which database to connect to
 [database]
 host = "sqlserver"

src/config.py

@@ -1,17 +1,33 @@
 import copy
+import logging
 import pathlib
 import tomllib
-from typing import Any
+from collections import deque
+from typing import Any, Sequence
 
-with open(pathlib.Path(__file__).parent / "config.default.toml", "rb") as fh:
+
+# The logger isn't yet configured when we load our configuration,
+# since the configuration includes our log settings. Instead, we
+# keep track of what we want to log so it can be logged later.
+_log_lines: deque[tuple[int, str]] = deque()
+logger = logging.getLogger(__file__)
+
+default_config_path = pathlib.Path(__file__).parent / "config.default.toml"
+with open(default_config_path, "rb") as fh:
     DEFAULT_CONFIG = tomllib.load(fh)
+    _log_lines.append((logging.INFO, f"Loaded default configuration from {default_config_path}"))
+    logger.info("Actually Foo")
 
 OVERRIDE_CONFIG_PATH = pathlib.Path(__file__).parent / "config.override.toml"
 if OVERRIDE_CONFIG_PATH.exists() and OVERRIDE_CONFIG_PATH.is_file():
     with open(OVERRIDE_CONFIG_PATH, "rb") as fh:
         OVERRIDE_CONFIG = tomllib.load(fh)
+        _log_lines.append(
+            (logging.INFO, f"Loaded configuration overrides from {OVERRIDE_CONFIG_PATH}")
+        )
 else:
     OVERRIDE_CONFIG = {}
+    _log_lines.append((logging.INFO, f"No custom overrides detected in {OVERRIDE_CONFIG_PATH}"))
 
 
 def _merge_configurations(
@@ -29,6 +45,28 @@ def _merge_configurations(
     return merged
 
 
+def _mask_configuration(
+    configuration: dict[str, Any], to_mask: Sequence[str] | None = None
+) -> dict[str, Any]:
+    if to_mask is None:
+        to_mask = ["password"]
+    masked = copy.copy(configuration)
+    for key, value in configuration.items():
+        if key in to_mask:
+            masked[key] = "****"
+        elif isinstance(value, dict):
+            masked[key] = _mask_configuration(value, to_mask)
+    return masked
+
+
+def log_configuration():
+    """Logs stacked log lines and then outputs the current configuration."""
+    while _log_lines:
+        level, message = _log_lines.popleft()
+        logger.log(level, message)
+    logger.info(f"Starting with merged configuration: {_mask_configuration(CONFIG)}")
+
+
 CONFIG = _merge_configurations(DEFAULT_CONFIG, OVERRIDE_CONFIG)
 DB_CONFIG = CONFIG.get("database", {})
 KEYCLOAK_CONFIG = CONFIG.get("keycloak", {})

src/main.py

@@ -6,7 +6,6 @@
 """
 
 import argparse
-from datetime import datetime, timezone
 import logging
 from pathlib import Path
 
@@ -15,7 +14,6 @@
 from fastapi import Depends, FastAPI, HTTPException
 from fastapi.responses import HTMLResponse
 from sqlmodel import select, SQLModel
-from starlette.requests import Request
 
 from authentication import get_user_or_raise, KeycloakUser, assert_required_settings_configured
 from config import KEYCLOAK_CONFIG, DB_CONFIG, DEV_CONFIG
@@ -29,10 +27,10 @@
 from database.model.platform.platform_names import PlatformName
 from database.session import EngineSingleton, DbSession
 from database.setup import create_database, database_exists
+from setup_logger import setup_logger
 from taxonomies.synchronize_taxonomy import synchronize_taxonomy_from_file
 from triggers import disable_review_process, enable_review_process
 from error_handling import http_exception_handler
-from database.model.agent.agent import Agent
 from routers import (
     resource_routers,
     parent_routers,
@@ -42,13 +40,13 @@
     user_router,
     bookmark_router,
 )
-from setup_logger import setup_logger
+from versioning import versions, add_version_to_openapi, add_deprecation_and_sunset_middleware
 
 
 def add_routes(app: FastAPI, url_prefix=""):
     """Add routes to the FastAPI application"""
 
-    @app.get(url_prefix + "/", response_class=HTMLResponse)
+    @app.get(url_prefix + "/", include_in_schema=False, response_class=HTMLResponse)
     def home() -> str:
         """Provides a redirect page to the docs."""
         return """
@@ -63,23 +61,21 @@ def home() -> str:
         </html>
         """
 
-    for path in ["/v2/{endpoint}", "/{endpoint}"]:
-
-        @app.get(url_prefix + path.format(endpoint="authorization_test"))
-        def test_authorization(user: KeycloakUser = Depends(get_user_or_raise)) -> KeycloakUser:
-            """
-            Returns the user, if authenticated correctly.
-            """
-            return user
+    @app.get("/authorization_test")
+    def test_authorization(user: KeycloakUser = Depends(get_user_or_raise)) -> KeycloakUser:
+        """
+        Returns the user, if authenticated correctly.
+        """
+        return user
 
-        @app.get(url_prefix + path.format(endpoint="counts"))
-        def counts() -> dict:
-            return {
-                router.resource_name_plural: count
-                for router in resource_routers.router_list
-                if issubclass(router.resource_class, AIoDConcept)
-                and (count := router.get_resource_count_func()(detailed=True))
-            }
+    @app.get("/counts")
+    def counts() -> dict:
+        return {
+            router.resource_name_plural: count
+            for router in resource_routers.router_list
+            if issubclass(router.resource_class, AIoDConcept)
+            and (count := router.get_resource_count_func()(detailed=True))
+        }
 
     for router in (
         resource_routers.router_list
@@ -119,17 +115,15 @@ def create_app() -> FastAPI:
 
 
 def build_app(*, url_prefix: str = "", version: str = "dev"):
-    app = FastAPI(
-        openapi_url=f"{url_prefix}/openapi.json",
-        docs_url=f"{url_prefix}/docs",
-        title="AIoD Metadata Catalogue",
+    kwargs = dict(
+        docs_url=None,  # We override the default pages with custom html
+        redoc_url=None,
         description="This is the REST API documentation of the AIoD Metadata Catalogue. "
         "See also our general "
         '<a href="https://aiondemand.github.io/AIOD-rest-api/">metadata catalogue documentation</a>, '
         "and our "
         '<a href="https://github.com/aiondemand/AIOD-rest-api/releases">changelog</a>.',
-        version=version,
-        swagger_ui_oauth2_redirect_url=f"{url_prefix}/docs/oauth2-redirect",
+        swagger_ui_oauth2_redirect_url=f"/docs/oauth2-redirect",
         swagger_ui_init_oauth={
             "clientId": KEYCLOAK_CONFIG.get("client_id_swagger"),
             "realm": KEYCLOAK_CONFIG.get("realm"),
@@ -138,42 +132,30 @@ def build_app(*, url_prefix: str = "", version: str = "dev"):
             "scopes": KEYCLOAK_CONFIG.get("scopes"),
         },
     )
-    add_routes(app, url_prefix=url_prefix)
-    app.add_exception_handler(HTTPException, http_exception_handler)
-
-    @app.middleware("http")
-    async def add_deprecation_header(request: Request, call_next):
-        """Adds a deprecation header: https://datatracker.ietf.org/doc/html/rfc9745"""
-        response = await call_next(request)
-        if "v1" in request.scope["path"]:
-            deprecation_date = datetime(year=2025, month=5, day=30, tzinfo=timezone.utc)
-            response.headers["Deprecation"] = f"@{int(deprecation_date.timestamp())}"
-            deprecation_link = '<https://aiondemand.github.io/AIOD-rest-api/using/migration-v1-v2>; rel="deprecation"; type="text/html"'
-            if links := response.headers.get("Link"):
-                response.headers["Link"] = ", ".join([links, deprecation_link])
-            else:
-                response.headers["Link"] = deprecation_link
-        return response
-
-    @app.middleware("http")
-    async def add_sunset_header(request: Request, call_next):
-        """Adds a sunset header: https://datatracker.ietf.org/doc/html/rfc8594"""
-        response = await call_next(request)
-        if "v1" in request.scope["path"]:
-            sunset_date = datetime(year=2025, month=6, day=11, tzinfo=timezone.utc)
-            response.headers["Sunset"] = sunset_date.strftime("%a, %d %b %Y %H:%M:%S %Z")
-            sunset_link = '<https://aiondemand.github.io/AIOD-rest-api/using/migration-v1-v2>; rel="sunset"; type="text/html"'
-            if links := response.headers.get("Link"):
-                response.headers["Link"] = ", ".join([links, sunset_link])
-            else:
-                response.headers["Link"] = sunset_link
-        return response
-
-    # Adds a visual deprecation style to the generated docs:
-    for route in app.routes:
-        if "v1" in route.path:
-            route.deprecated = True
-    return app
+    main_app = FastAPI(
+        root_path=url_prefix,
+        title="AI-on-Demand Metadata Catalogue REST API",
+        version="latest",
+        **kwargs,
+    )
+    add_routes(main_app)
+    main_app.add_exception_handler(HTTPException, http_exception_handler)
+    add_version_to_openapi(main_app, root_path=url_prefix)
+
+    for version, info in versions.items():
+        if info.retired:
+            continue
+        app = FastAPI(
+            title=f"AIoD Metadata Catalogue {version}",
+            version=f"{version}",
+            **kwargs,
+        )
+        add_routes(app)
+        app.add_exception_handler(HTTPException, http_exception_handler)
+        add_deprecation_and_sunset_middleware(app)
+        add_version_to_openapi(app, root_path=url_prefix)
+        main_app.mount(f"/{version}", app)
+    return main_app
 
 
 def build_database(drop_database: bool = False):