DB models

Author: pcrespov

packages/models-library/src/models_library/services_metadata_editable.py

@@ -28,13 +28,15 @@ class ServiceMetaDataEditable(ServiceBaseDisplay):
     version_display: str | None = None
 
     # Below fields only in the database ----
-    deprecated: datetime | None = Field(
-        default=None,
-        description="Owner can set the date to retire the service. Three possibilities:"
-        "If None, the service is marked as `published`;"
-        "If now<deprecated the service is marked as deprecated;"
-        "If now>=deprecated, the service is retired",
-    )
+    deprecated: Annotated[
+        datetime | None,
+        Field(
+            description="Owner can set the date to retire the service. Three possibilities:"
+            "If None, the service is marked as `published`;"
+            "If now<deprecated the service is marked as deprecated;"
+            "If now>=deprecated, the service is retired",
+        ),
+    ] = None
     classifiers: list[str] | None
     quality: Annotated[
         dict[str, Any], Field(default_factory=dict, json_schema_extra={"default": {}})

services/catalog/src/simcore_service_catalog/api/rest/_services.py

@@ -22,7 +22,7 @@
 )
 from ...db.repositories.groups import GroupsRepository
 from ...db.repositories.services import ServicesRepository
-from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataAtDB
+from ...models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBGet
 from ...services.director import DirectorApi
 from ..dependencies.database import get_repository
 from ..dependencies.director import get_director_api
@@ -35,7 +35,7 @@
 
 def _compose_service_details(
     service_in_registry: dict[str, Any],  # published part
-    service_in_db: ServiceMetaDataAtDB,  # editable part
+    service_in_db: ServiceMetaDataDBGet,  # editable part
     service_access_rights_in_db: list[ServiceAccessRightsAtDB],
     service_owner: str | None,
 ) -> ServiceGet | None:

services/catalog/src/simcore_service_catalog/core/background_tasks.py

@@ -28,7 +28,7 @@
 from ..db.repositories.groups import GroupsRepository
 from ..db.repositories.projects import ProjectsRepository
 from ..db.repositories.services import ServicesRepository
-from ..models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataAtDB
+from ..models.services_db import ServiceAccessRightsAtDB, ServiceMetaDataDBGet
 from ..services import access_rights
 
 _logger = logging.getLogger(__name__)
@@ -89,7 +89,7 @@ def _by_version(t: tuple[ServiceKey, ServiceVersion]) -> Version:
 
             # set the service in the DB
             await services_repo.create_or_update_service(
-                ServiceMetaDataAtDB(**service_metadata.model_dump(), owner=owner_gid),
+                ServiceMetaDataDBGet(**service_metadata.model_dump(), owner=owner_gid),
                 service_access_rights,
             )
 

services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py

@@ -4,11 +4,13 @@
 from models_library.products import ProductName
 from models_library.services_types import ServiceKey, ServiceVersion
 from models_library.users import UserID
+from simcore_postgres_database.utils_repos import get_columns_from_db_model
 from sqlalchemy.dialects.postgresql import ARRAY, INTEGER, array_agg
 from sqlalchemy.sql import and_, or_
 from sqlalchemy.sql.expression import func
 from sqlalchemy.sql.selectable import Select
 
+from ...models.services_db import ServiceMetaDataDBGet
 from ..tables import (
     services_access_rights,
     services_compatibility,
@@ -17,6 +19,10 @@
     users,
 )
 
+SERVICES_META_DATA_COLS = get_columns_from_db_model(
+    services_meta_data, ServiceMetaDataDBGet
+)
+
 
 def list_services_stmt(
     *,
@@ -26,7 +32,7 @@ def list_services_stmt(
     combine_access_with_and: bool | None = True,
     product_name: str | None = None,
 ) -> Select:
-    stmt = sa.select(services_meta_data)
+    stmt = sa.select(SERVICES_META_DATA_COLS)
     if gids or execute_access or write_access:
         conditions: list[Any] = []
 
@@ -50,13 +56,9 @@ def list_services_stmt(
         if product_name:
             conditions.append(services_access_rights.c.product_name == product_name)
 
-        stmt = (
-            sa.select(
-                services_meta_data,
-            )
-            .distinct(services_meta_data.c.key, services_meta_data.c.version)
-            .select_from(services_meta_data.join(services_access_rights))
-        )
+        stmt = stmt.distinct(
+            services_meta_data.c.key, services_meta_data.c.version
+        ).select_from(services_meta_data.join(services_access_rights))
         if conditions:
             stmt = stmt.where(and_(*conditions))
         stmt = stmt.order_by(services_meta_data.c.key, services_meta_data.c.version)

services/catalog/src/simcore_service_catalog/db/repositories/services.py

@@ -17,21 +17,23 @@
 from psycopg2.errors import ForeignKeyViolation
 from pydantic import PositiveInt, TypeAdapter, ValidationError
 from simcore_postgres_database.utils_services import create_select_latest_services_query
-from sqlalchemy import literal_column
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.sql import and_, or_
 from sqlalchemy.sql.expression import tuple_
 
 from ...models.services_db import (
     ReleaseFromDB,
     ServiceAccessRightsAtDB,
-    ServiceMetaDataAtDB,
+    ServiceMetaDataDBCreate,
+    ServiceMetaDataDBGet,
+    ServiceMetaDataDBPatch,
     ServiceWithHistoryFromDB,
 )
 from ...models.services_specifications import ServiceSpecificationsAtDB
 from ..tables import services_access_rights, services_meta_data, services_specifications
 from ._base import BaseRepository
 from ._services_sql import (
+    SERVICES_META_DATA_COLS,
     AccessRightsClauses,
     can_get_service_stmt,
     get_service_history_stmt,
@@ -79,11 +81,11 @@ async def list_services(
         write_access: bool | None = None,
         combine_access_with_and: bool | None = True,
         product_name: str | None = None,
-    ) -> list[ServiceMetaDataAtDB]:
+    ) -> list[ServiceMetaDataDBGet]:
 
         async with self.db_engine.connect() as conn:
             return [
-                ServiceMetaDataAtDB.model_validate(row)
+                ServiceMetaDataDBGet.model_validate(row)
                 async for row in await conn.stream(
                     list_services_stmt(
                         gids=gids,
@@ -102,7 +104,7 @@ async def list_service_releases(
         major: int | None = None,
         minor: int | None = None,
         limit_count: int | None = None,
-    ) -> list[ServiceMetaDataAtDB]:
+    ) -> list[ServiceMetaDataDBGet]:
         """Lists LAST n releases of a given service, sorted from latest first
 
         major, minor is used to filter as major.minor.* or major.*
@@ -124,7 +126,7 @@ async def list_service_releases(
                 search_condition &= services_meta_data.c.version.like(f"{major}.%")
 
         query = (
-            sa.select(services_meta_data)
+            sa.select(SERVICES_META_DATA_COLS)
             .where(search_condition)
             .order_by(sa.desc(services_meta_data.c.version))
         )
@@ -134,22 +136,22 @@ async def list_service_releases(
 
         async with self.db_engine.connect() as conn:
             releases = [
-                ServiceMetaDataAtDB.model_validate(row)
+                ServiceMetaDataDBGet.model_validate(row)
                 async for row in await conn.stream(query)
             ]
 
         # Now sort naturally from latest first: (This is lame, the sorting should be done in the db)
-        def _by_version(x: ServiceMetaDataAtDB) -> packaging.version.Version:
+        def _by_version(x: ServiceMetaDataDBGet) -> packaging.version.Version:
             return packaging.version.parse(x.version)
 
         return sorted(releases, key=_by_version, reverse=True)
 
-    async def get_latest_release(self, key: str) -> ServiceMetaDataAtDB | None:
+    async def get_latest_release(self, key: str) -> ServiceMetaDataDBGet | None:
         """Returns last release or None if service was never released"""
         services_latest = create_select_latest_services_query().alias("services_latest")
 
         query = (
-            sa.select(services_meta_data)
+            sa.select(SERVICES_META_DATA_COLS)
             .select_from(
                 services_latest.join(
                     services_meta_data,
@@ -163,7 +165,7 @@ async def get_latest_release(self, key: str) -> ServiceMetaDataAtDB | None:
             result = await conn.execute(query)
             row = result.first()
         if row:
-            return ServiceMetaDataAtDB.model_validate(row)
+            return ServiceMetaDataDBGet.model_validate(row)
         return None  # mypy
 
     async def get_service(
@@ -175,18 +177,11 @@ async def get_service(
         execute_access: bool | None = None,
         write_access: bool | None = None,
         product_name: str | None = None,
-    ) -> ServiceMetaDataAtDB | None:
+    ) -> ServiceMetaDataDBGet | None:
 
-        query = sa.select(services_meta_data).where(
-            (services_meta_data.c.key == key)
-            & (services_meta_data.c.version == version)
-        )
-        if gids or execute_access or write_access:
-
-            query = sa.select(services_meta_data).select_from(
-                services_meta_data.join(services_access_rights)
-            )
+        query = sa.select(SERVICES_META_DATA_COLS)
 
+        if gids or execute_access or write_access:
             conditions = [
                 services_meta_data.c.key == key,
                 services_meta_data.c.version == version,
@@ -202,20 +197,27 @@ async def get_service(
             if product_name:
                 conditions.append(services_access_rights.c.product_name == product_name)
 
-            query = query.where(and_(*conditions))
+            query = query.select_from(
+                services_meta_data.join(services_access_rights)
+            ).where(and_(*conditions))
+        else:
+            query = query.where(
+                (services_meta_data.c.key == key)
+                & (services_meta_data.c.version == version)
+            )
 
         async with self.db_engine.connect() as conn:
             result = await conn.execute(query)
             row = result.first()
         if row:
-            return ServiceMetaDataAtDB.model_validate(row)
+            return ServiceMetaDataDBGet.model_validate(row)
         return None  # mypy
 
     async def create_or_update_service(
         self,
-        new_service: ServiceMetaDataAtDB,
+        new_service: ServiceMetaDataDBCreate,
         new_service_access_rights: list[ServiceAccessRightsAtDB],
-    ) -> ServiceMetaDataAtDB:
+    ) -> ServiceMetaDataDBGet:
         for access_rights in new_service_access_rights:
             if (
                 access_rights.key != new_service.key
@@ -229,12 +231,12 @@ async def create_or_update_service(
             result = await conn.execute(
                 # pylint: disable=no-value-for-parameter
                 services_meta_data.insert()
-                .values(**new_service.model_dump(by_alias=True))
-                .returning(literal_column("*"))
+                .values(**new_service.model_dump(by_alias=True, exclude_unset=True))
+                .returning(*SERVICES_META_DATA_COLS)
             )
             row = result.first()
             assert row  # nosec
-            created_service = ServiceMetaDataAtDB.model_validate(row)
+            created_service = ServiceMetaDataDBGet.model_validate(row)
 
             for access_rights in new_service_access_rights:
                 insert_stmt = pg_insert(services_access_rights).values(
@@ -243,13 +245,18 @@ async def create_or_update_service(
                 await conn.execute(insert_stmt)
         return created_service
 
-    async def update_service(self, patched_service: ServiceMetaDataAtDB) -> None:
+    async def update_service(
+        self,
+        service_key: ServiceKey,
+        service_version: ServiceVersion,
+        patched_service: ServiceMetaDataDBPatch,
+    ) -> None:
 
         stmt_update = (
             services_meta_data.update()
             .where(
-                (services_meta_data.c.key == patched_service.key)
-                & (services_meta_data.c.version == patched_service.version)
+                (services_meta_data.c.key == service_key)
+                & (services_meta_data.c.version == service_version)
             )
             .values(
                 **patched_service.model_dump(