draft tests

Author: pcrespov

services/catalog/src/simcore_service_catalog/db/repositories/_services_sql.py

@@ -66,7 +66,7 @@ def list_services_stmt(
     return stmt
 
 
-def _version(column_or_value):
+def by_version(column_or_value):
     # converts version value string to array[integer] that can be compared
     # i.e. '1.2.3' -> [1, 2, 3]
     return sa.func.string_to_array(column_or_value, ".").cast(ARRAY(INTEGER))
@@ -165,7 +165,7 @@ def list_latest_services_stmt(
         .where(access_rights)
         .order_by(
             services_meta_data.c.key,
-            sa.desc(_version(services_meta_data.c.version)),  # latest first
+            sa.desc(by_version(services_meta_data.c.version)),  # latest first
         )
         .distinct(services_meta_data.c.key)  # get only first
         .limit(limit)
@@ -312,12 +312,7 @@ def get_service_history_stmt(
     user_id: UserID,
     access_rights: sa.sql.ClauseElement,
     service_key: ServiceKey,
-    limit: int | None,
-    offset: int | None,
 ):
-    assert offset is None, "UNDER DEV"
-    assert limit is None, "UNDER DEV"
-
     _sq = (
         sa.select(
             services_meta_data.c.key,
@@ -356,7 +351,7 @@ def get_service_history_stmt(
     history_subquery = (
         sa.select(_sq)
         .order_by(
-            sa.desc(_version(_sq.c.version)),  # latest version first
+            sa.desc(by_version(_sq.c.version)),  # latest version first
         )
         .alias("history_subquery")
     )

services/catalog/src/simcore_service_catalog/db/repositories/services.py

@@ -16,7 +16,9 @@
 from models_library.users import UserID
 from psycopg2.errors import ForeignKeyViolation
 from pydantic import PositiveInt, TypeAdapter, ValidationError
+from simcore_postgres_database.utils_repos import pass_or_acquire_connection
 from simcore_postgres_database.utils_services import create_select_latest_services_query
+from sqlalchemy import sql
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.sql import and_, or_
 from sqlalchemy.sql.expression import tuple_
@@ -30,11 +32,18 @@
     ServiceWithHistoryDBGet,
 )
 from ...models.services_specifications import ServiceSpecificationsAtDB
-from ..tables import services_access_rights, services_meta_data, services_specifications
+from ..tables import (
+    services_access_rights,
+    services_compatibility,
+    services_meta_data,
+    services_specifications,
+    user_to_groups,
+)
 from ._base import BaseRepository
 from ._services_sql import (
     SERVICES_META_DATA_COLS,
     AccessRightsClauses,
+    by_version,
     can_get_service_stmt,
     get_service_history_stmt,
     get_service_stmt,
@@ -340,8 +349,6 @@ async def get_service_with_history(
                 user_id=user_id,
                 access_rights=AccessRightsClauses.can_read,
                 service_key=key,
-                limit=None,
-                offset=None,
             )
             async with self.db_engine.begin() as conn:
                 result = await conn.execute(stmt_history)
@@ -440,30 +447,95 @@ async def get_service_history(
         user_id: UserID,
         # get args
         key: ServiceKey,
-        # list args: pagination
-        limit: int | None = None,
-        offset: int | None = None,
-    ) -> tuple[PositiveInt, list[ReleaseDBGet]]:
-
-        stmt_total, stmt_history = get_service_history_stmt(
+    ) -> list[ReleaseDBGet]:
+        """
+        DEPRECATED: use get_service_history_page instead!
+        """
+        stmt_history = get_service_history_stmt(
             product_name=product_name,
             user_id=user_id,
             access_rights=AccessRightsClauses.can_read,
             service_key=key,
-            offset=offset,
-            limit=limit,
         )
         async with self.db_engine.connect() as conn:
-            result = await conn.execute(stmt_total)
-            total_count = result.scalar() or 0
-
             result = await conn.execute(stmt_history)
             row = result.one_or_none()
 
-        items = (
+        return (
             TypeAdapter(list[ReleaseDBGet]).validate_python(row.history) if row else []
         )
 
+    async def get_service_history_page(
+        self,
+        # access-rights
+        product_name: ProductName,
+        user_id: UserID,
+        # get args
+        key: ServiceKey,
+        # list args: pagination
+        limit: int | None = None,
+        offset: int | None = None,
+    ) -> tuple[PositiveInt, list[ReleaseDBGet]]:
+
+        base_query = (
+            sql.select(
+                services_meta_data.c.key,
+                services_meta_data.c.version,
+                services_meta_data.c.version_display,
+                services_meta_data.c.deprecated,
+                services_meta_data.c.created,
+                services_compatibility.c.custom_policy.label(
+                    "compatibility_policy"
+                ),  # CompatiblePolicyDict | None
+            )
+            .select_from(
+                # joins because access-rights might change per version
+                services_meta_data.join(
+                    services_access_rights,
+                    (services_meta_data.c.key == services_access_rights.c.key)
+                    & (
+                        services_meta_data.c.version == services_access_rights.c.version
+                    ),
+                )
+                .join(
+                    user_to_groups,
+                    (user_to_groups.c.gid == services_access_rights.c.gid),
+                )
+                .outerjoin(
+                    services_compatibility,
+                    (services_meta_data.c.key == services_compatibility.c.key)
+                    & (
+                        services_meta_data.c.version == services_compatibility.c.version
+                    ),
+                )
+            )
+            .where(
+                (services_meta_data.c.key == key)
+                & (services_access_rights.c.product_name == product_name)
+                & (user_to_groups.c.uid == user_id)
+                & AccessRightsClauses.can_read
+            )
+            .distinct()
+        )
+
+        subquery = base_query.subquery()
+        count_query = sql.select(sql.func.count()).select_from(subquery)
+
+        page_query = (
+            base_query.order_by(sql.desc(by_version(base_query.c.version)))
+            .offset(offset)
+            .limit(limit)
+        )
+
+        async with pass_or_acquire_connection(self.db_engine) as conn:
+            total_count: PositiveInt = await conn.scalar(count_query) or 0
+
+            result = await conn.stream(page_query)
+            items: list[ReleaseDBGet] = [
+                ReleaseDBGet.model_validate(row, from_attributes=True)
+                async for row in result
+            ]
+
         return total_count, items
 
     # Service Access Rights ----

services/catalog/src/simcore_service_catalog/services/services_api.py

@@ -476,15 +476,15 @@ async def get_service_history(
     offset: NonNegativeInt | None = None,
     include_compatibility: bool = False,
 ) -> tuple[PageTotalCount, list[ServiceRelease]]:
-    total_count, history = await repo.get_service_history(
+    total_count, history = await repo.get_service_history_page(
         # NOTE: that the service history might be different for each user
         # since access-rights are defined on a k:v basis
         product_name=product_name,
         user_id=user_id,
         key=service_key,
         limit=limit,
         offset=offset,
-    ) or (0, [])
+    )
 
     compatibility_map = {}
     if include_compatibility:

services/catalog/tests/unit/with_dbs/test_db_repositories.py

@@ -476,3 +476,69 @@ async def test_can_get_service(
         key=service_key,
         version=service_version,
     )
+
+
+async def test_get_service_history_page(
+    target_product: ProductName,
+    create_fake_service_data: Callable,
+    services_db_tables_injector: Callable,
+    services_repo: ServicesRepository,
+    user_id: UserID,
+):
+    # inject services with multiple versions
+    service_key = "simcore/services/dynamic/test-service"
+    num_versions = 10
+    await services_db_tables_injector(
+        [
+            create_fake_service_data(
+                service_key,
+                f"{v}.0.0",
+                team_access=None,
+                everyone_access=None,
+                product=target_product,
+            )
+            for v in range(num_versions)
+        ]
+    )
+
+    # fetch full history using get_service_history_page
+    total_count, history = await services_repo.get_service_history_page(
+        product_name=target_product,
+        user_id=user_id,
+        key=service_key,
+    )
+    assert total_count == num_versions
+    assert len(history) == num_versions
+    assert [release.version for release in history] == [
+        f"{v}.0.0" for v in reversed(range(num_versions))
+    ]
+
+    # fetch full history using deprecated get_service_history
+    deprecated_history = await services_repo.get_service_history(
+        product_name=target_product,
+        user_id=user_id,
+        key=service_key,
+    )
+    assert len(deprecated_history) == len(history)
+    assert [release.version for release in deprecated_history] == [
+        release.version for release in history
+    ]
+
+    # fetch paginated history
+    limit = 3
+    offset = 2
+    total_count, paginated_history = await services_repo.get_service_history_page(
+        product_name=target_product,
+        user_id=user_id,
+        key=service_key,
+        limit=limit,
+        offset=offset,
+    )
+    assert total_count == num_versions
+    assert len(paginated_history) == limit
+    assert [release.version for release in paginated_history] == [
+        f"{v}.0.0" for v in reversed(range(offset, offset + limit))
+    ]
+
+    # compare paginated results with the corresponding slice of the full history
+    assert paginated_history == history[offset : offset + limit]