refactor: add validation to storage, repo and session blueprints

Author: Panaetius

components/renku_data_services/repositories/blueprints.py

@@ -3,14 +3,15 @@
 from dataclasses import dataclass
 from urllib.parse import unquote
 
-from sanic import HTTPResponse, Request, json
+from sanic import HTTPResponse, Request
 from sanic.response import JSONResponse
 
 import renku_data_services.base_models as base_models
 from renku_data_services import errors
 from renku_data_services.base_api.auth import authenticate
 from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint
 from renku_data_services.base_api.etag import extract_if_none_match
+from renku_data_services.base_models.validation import validated_json
 from renku_data_services.repositories import apispec
 from renku_data_services.repositories.apispec_base import RepositoryParams
 from renku_data_services.repositories.db import GitRepositoriesRepository
@@ -53,10 +54,7 @@ async def get_internal_gitlab_user() -> base_models.APIUser:
                 if result.repository_metadata and result.repository_metadata.etag is not None
                 else None
             )
-            return json(
-                apispec.RepositoryProviderMatch.model_validate(result).model_dump(exclude_none=True, mode="json"),
-                headers=headers,
-            )
+            return validated_json(apispec.RepositoryProviderMatch, result, headers=headers)
 
         return "/repositories/<repository_url>", ["GET"], _get_one_repository
 

components/renku_data_services/session/apispec_base.py

@@ -14,12 +14,26 @@ class Config:
 
         from_attributes = True
 
+    @field_validator("id", mode="before", check_fields=False)
+    @classmethod
+    def serialize_id(cls, id: str | ULID) -> str:
+        """Custom serializer that can handle ULIDs."""
+        return str(id)
+
     @field_validator("project_id", mode="before", check_fields=False)
     @classmethod
-    def serialize_id(cls, project_id: str | ULID) -> str:
+    def serialize_project_id(cls, project_id: str | ULID) -> str:
         """Custom serializer that can handle ULIDs."""
         return str(project_id)
 
+    @field_validator("environment_id", mode="before", check_fields=False)
+    @classmethod
+    def serialize_environment_id(cls, environment_id: str | ULID | None) -> str | None:
+        """Custom serializer that can handle ULIDs."""
+        if environment_id is None:
+            return None
+        return str(environment_id)
+
     @field_validator("environment_kind", mode="before", check_fields=False)
     @classmethod
     def serialize_environment_kind(cls, environment_kind: models.EnvironmentKind | str) -> str:

components/renku_data_services/session/blueprints.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass
 from datetime import UTC, datetime
 
-from sanic import HTTPResponse, Request, json
+from sanic import HTTPResponse, Request
 from sanic.response import JSONResponse
 from sanic_ext import validate
 from ulid import ULID
@@ -12,6 +12,7 @@
 from renku_data_services import errors
 from renku_data_services.base_api.auth import authenticate, only_authenticated
 from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint
+from renku_data_services.base_models.validation import validated_json
 from renku_data_services.session import apispec, models
 from renku_data_services.session.db import SessionRepository
 
@@ -28,9 +29,7 @@ def get_all(self) -> BlueprintFactoryResponse:
 
         async def _get_all(_: Request) -> JSONResponse:
             environments = await self.session_repo.get_environments()
-            return json(
-                [apispec.Environment.model_validate(e).model_dump(exclude_none=True, mode="json") for e in environments]
-            )
+            return validated_json(apispec.EnvironmentList, environments)
 
         return "/environments", ["GET"], _get_all
 
@@ -39,7 +38,7 @@ def get_one(self) -> BlueprintFactoryResponse:
 
         async def _get_one(_: Request, environment_id: ULID) -> JSONResponse:
             environment = await self.session_repo.get_environment(environment_id=environment_id)
-            return json(apispec.Environment.model_validate(environment).model_dump(exclude_none=True, mode="json"))
+            return validated_json(apispec.Environment, environment)
 
         return "/environments/<environment_id:ulid>", ["GET"], _get_one
 
@@ -59,7 +58,7 @@ async def _post(_: Request, user: base_models.APIUser, body: apispec.Environment
                 creation_date=datetime.now(UTC).replace(microsecond=0),
             )
             environment = await self.session_repo.insert_environment(user=user, new_environment=environment_model)
-            return json(apispec.Environment.model_validate(environment).model_dump(exclude_none=True, mode="json"), 201)
+            return validated_json(apispec.Environment, environment, 201)
 
         return "/environments", ["POST"], _post
 
@@ -75,7 +74,7 @@ async def _patch(
             environment = await self.session_repo.update_environment(
                 user=user, environment_id=environment_id, **body_dict
             )
-            return json(apispec.Environment.model_validate(environment).model_dump(exclude_none=True, mode="json"))
+            return validated_json(apispec.Environment, environment)
 
         return "/environments/<environment_id:ulid>", ["PATCH"], _patch
 
@@ -103,12 +102,7 @@ def get_all(self) -> BlueprintFactoryResponse:
         @authenticate(self.authenticator)
         async def _get_all(_: Request, user: base_models.APIUser) -> JSONResponse:
             launchers = await self.session_repo.get_launchers(user=user)
-            return json(
-                [
-                    apispec.SessionLauncher.model_validate(item).model_dump(exclude_none=True, mode="json")
-                    for item in launchers
-                ]
-            )
+            return validated_json(apispec.SessionLaunchersList, launchers)
 
         return "/session_launchers", ["GET"], _get_all
 
@@ -118,7 +112,7 @@ def get_one(self) -> BlueprintFactoryResponse:
         @authenticate(self.authenticator)
         async def _get_one(_: Request, user: base_models.APIUser, launcher_id: ULID) -> JSONResponse:
             launcher = await self.session_repo.get_launcher(user=user, launcher_id=launcher_id)
-            return json(apispec.SessionLauncher.model_validate(launcher).model_dump(exclude_none=True, mode="json"))
+            return validated_json(apispec.SessionLauncher, launcher)
 
         return "/session_launchers/<launcher_id:ulid>", ["GET"], _get_one
 
@@ -150,9 +144,7 @@ async def _post(_: Request, user: base_models.APIUser, body: apispec.SessionLaun
                 creation_date=datetime.now(UTC).replace(microsecond=0),
             )
             launcher = await self.session_repo.insert_launcher(user=user, new_launcher=launcher_model)
-            return json(
-                apispec.SessionLauncher.model_validate(launcher).model_dump(exclude_none=True, mode="json"), 201
-            )
+            return validated_json(apispec.SessionLauncher, launcher, 201)
 
         return "/session_launchers", ["POST"], _post
 
@@ -166,7 +158,7 @@ async def _patch(
         ) -> JSONResponse:
             body_dict = body.model_dump(exclude_none=True)
             launcher = await self.session_repo.update_launcher(user=user, launcher_id=launcher_id, **body_dict)
-            return json(apispec.SessionLauncher.model_validate(launcher).model_dump(exclude_none=True, mode="json"))
+            return validated_json(apispec.SessionLauncher, launcher)
 
         return "/session_launchers/<launcher_id:ulid>", ["PATCH"], _patch
 
@@ -186,11 +178,6 @@ def get_project_launchers(self) -> BlueprintFactoryResponse:
         @authenticate(self.authenticator)
         async def _get_launcher(_: Request, user: base_models.APIUser, project_id: ULID) -> JSONResponse:
             launchers = await self.session_repo.get_project_launchers(user=user, project_id=project_id)
-            return json(
-                [
-                    apispec.SessionLauncher.model_validate(item).model_dump(exclude_none=True, mode="json")
-                    for item in launchers
-                ]
-            )
+            return validated_json(apispec.SessionLaunchersList, launchers)
 
         return "/projects/<project_id:ulid>/session_launchers", ["GET"], _get_launcher

components/renku_data_services/session/models.py

@@ -41,7 +41,7 @@ class UnsavedEnvironment(BaseModel):
 class Environment(UnsavedEnvironment):  # type: ignore[misc]
     """Session environment model."""
 
-    id: str
+    id: ULID
 
 
 @dataclass(frozen=True, eq=True, kw_only=True)

components/renku_data_services/session/orm.py

@@ -10,6 +10,7 @@
 from renku_data_services.crc.orm import ResourceClassORM
 from renku_data_services.project.orm import ProjectORM
 from renku_data_services.session import models
+from renku_data_services.utils.sqlalchemy import ULIDType
 
 metadata_obj = MetaData(schema="sessions")  # Has to match alembic ini section name
 
@@ -25,7 +26,7 @@ class EnvironmentORM(BaseORM):
 
     __tablename__ = "environments"
 
-    id: Mapped[str] = mapped_column("id", String(26), primary_key=True, default_factory=lambda: str(ULID()), init=False)
+    id: Mapped[ULID] = mapped_column("id", ULIDType, primary_key=True, default_factory=lambda: str(ULID()), init=False)
     """Id of this session environment object."""
 
     name: Mapped[str] = mapped_column("name", String(99))

components/renku_data_services/storage/api.spec.yaml

@@ -262,7 +262,7 @@ paths:
         content:
           application/json:
             schema:
-              $ref: "#/components/schemas/RCloneConfig"
+              $ref: "#/components/schemas/RCloneConfigValidate"
       responses:
         "204":
           description: The configuration is valid
@@ -346,6 +346,16 @@ components:
           nullable: true
         - type: boolean
         - type: object
+    RCloneConfigValidate: #this is the same as RCloneConfig but duplicated so a class gets generated
+      type: object
+      description: Dictionary of rclone key:value pairs (based on schema from '/storage_schema')
+      additionalProperties:
+        oneOf:
+        - type: integer
+        - type: string
+          nullable: true
+        - type: boolean
+        - type: object
     CloudStorageUrl:
       allOf:
         - $ref: "#/components/schemas/GitRequest"

components/renku_data_services/storage/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-08-06T05:55:29+00:00
+#   timestamp: 2024-08-09T12:39:58+00:00
 
 from __future__ import annotations
 
@@ -11,6 +11,12 @@
 from renku_data_services.storage.apispec_base import BaseAPISpec
 
 
+class RCloneConfigValidate(
+    RootModel[Optional[Dict[str, Union[int, Optional[str], bool, Dict[str, Any]]]]]
+):
+    root: Optional[Dict[str, Union[int, Optional[str], bool, Dict[str, Any]]]] = None
+
+
 class Example(BaseAPISpec):
     value: Optional[str] = Field(
         None, description="a potential value for the option (think enum)"