From 7b70222c200b7be16e835a9606661ed0d0ee9aa8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 13:44:39 +0200 Subject: [PATCH 01/20] doc --- .../_service_solvers.py | 28 +++++++++---------- .../api/routes/solvers.py | 20 +++++++------ 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 73b9b71633cc..208ed961ff5a 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -125,15 +125,15 @@ async def solver_release_history( self, *, solver_key: SolverKeyId, - offset: NonNegativeInt, - limit: PositiveInt, + pagination_offset: NonNegativeInt, + pagination_limit: PositiveInt, ) -> tuple[list[Solver], PageMetaInfoLimitOffset]: releases, page_meta = ( await self.catalog_service.list_release_history_latest_first( filter_by_service_key=solver_key, - pagination_offset=offset, - pagination_limit=limit, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, ) ) @@ -158,16 +158,16 @@ async def latest_solvers( *, pagination_offset: NonNegativeInt, pagination_limit: PositiveInt, - filter_by_solver_id: str | None = None, - filter_by_version_display: str | None = None, + filter_by_solver_key_pattern: str | None = None, + filter_by_version_display_pattern: str | None = None, ) -> tuple[list[Solver], PageMetaInfoLimitOffset]: """Lists the latest solvers with pagination and filtering. Args: - offset: Pagination offset - limit: Pagination limit - solver_id_pattern: Optional pattern to filter solvers by ID - version_display_pattern: Optional pattern to filter by version display + pagination_offset: Pagination offset + pagination_limit: Pagination limit + filter_by_solver_key_pattern: Optional pattern to filter solvers by key e.g. "simcore/service/my_solver*" + filter_by_version_display_pattern: Optional pattern to filter by version display e.g. "1.0.*-beta" Returns: A tuple with the list of filtered solvers and pagination metadata @@ -175,12 +175,12 @@ async def latest_solvers( filters = ServiceListFilters(service_type=ServiceType.COMPUTATIONAL) # Add key_pattern filter for solver ID if provided - if filter_by_solver_id: - filters.service_key_pattern = filter_by_solver_id + if filter_by_solver_key_pattern: + filters.service_key_pattern = filter_by_solver_key_pattern # Add version_display_pattern filter if provided - if filter_by_version_display: - filters.version_display_pattern = filter_by_version_display + if filter_by_version_display_pattern: + filters.version_display_pattern = filter_by_version_display_pattern services, page_meta = await self.catalog_service.list_latest_releases( pagination_offset=pagination_offset, diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index a6092743bf53..1d6dcb5775a9 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -45,7 +45,9 @@ **DEFAULT_BACKEND_SERVICE_STATUS_CODES, } -router = APIRouter() +router = APIRouter( + # /v0/solvers/ +) @router.get( @@ -98,8 +100,8 @@ async def list_solvers_paginated( solvers, page_meta = await solver_service.latest_solvers( pagination_offset=page_params.offset, pagination_limit=page_params.limit, - filter_by_solver_id=filters.solver_id, - filter_by_version_display=filters.version_display, + filter_by_solver_key_pattern=filters.solver_id, + filter_by_version_display_pattern=filters.version_display, ) for solver in solvers: @@ -145,8 +147,8 @@ async def list_solvers_releases( for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): solvers, page_meta = await solver_service.solver_release_history( solver_key=solver.id, - offset=page_params.offset, - limit=page_params.limit, + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, ) page_params.total_number_of_items = page_meta.total all_solvers.extend(solvers) @@ -216,8 +218,8 @@ async def list_solver_releases( for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): solvers, page_meta = await solver_service.solver_release_history( solver_key=solver_key, - offset=page_params.offset, - limit=page_params.limit, + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, ) page_params.total_number_of_items = page_meta.total all_releases.extend(solvers) @@ -249,8 +251,8 @@ async def list_solver_releases_paginated( ): solvers, page_meta = await solver_service.solver_release_history( solver_key=solver_key, - offset=page_params.offset, - limit=page_params.limit, + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, ) for solver in solvers: From 00e3bb5e712d0be4afe2857751989e298b49f8b6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 13:47:52 +0200 Subject: [PATCH 02/20] catalog renaming --- .../simcore_service_catalog/api/rpc/_services.py | 4 ++-- .../repository/services.py | 16 ++++++++-------- .../service/catalog_services.py | 14 +++++++------- .../tests/unit/with_dbs/test_repositories.py | 6 +++--- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 356ae89f74ce..b2c722572358 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -252,8 +252,8 @@ async def list_my_service_history_latest_first( product_name=product_name, user_id=user_id, service_key=service_key, - limit=limit, - offset=offset, + pagination_limit=limit, + pagination_offset=offset, filters=TypeAdapter(ServiceDBFilters | None).validate_python( filters, from_attributes=True ), diff --git a/services/catalog/src/simcore_service_catalog/repository/services.py b/services/catalog/src/simcore_service_catalog/repository/services.py index 3d24521df4d6..5abc9c7d827c 100644 --- a/services/catalog/src/simcore_service_catalog/repository/services.py +++ b/services/catalog/src/simcore_service_catalog/repository/services.py @@ -383,8 +383,8 @@ async def list_latest_services( product_name: ProductName, user_id: UserID, # list args: pagination - limit: int | None = None, - offset: int | None = None, + pagination_limit: int | None = None, + pagination_offset: int | None = None, filters: ServiceDBFilters | None = None, ) -> tuple[PositiveInt, list[ServiceWithHistoryDBGet]]: @@ -399,8 +399,8 @@ async def list_latest_services( product_name=product_name, user_id=user_id, access_rights=AccessRightsClauses.can_read, - limit=limit, - offset=offset, + limit=pagination_limit, + offset=pagination_offset, filters=filters, ) @@ -475,8 +475,8 @@ async def get_service_history_page( # get args key: ServiceKey, # list args: pagination - limit: int | None = None, - offset: int | None = None, + pagination_limit: int | None = None, + pagination_offset: int | None = None, filters: ServiceDBFilters | None = None, ) -> tuple[PositiveInt, list[ReleaseDBGet]]: @@ -540,8 +540,8 @@ async def get_service_history_page( ) ) .order_by(sql.desc(_services_sql.by_version(services_meta_data.c.version))) - .offset(offset) - .limit(limit) + .offset(pagination_offset) + .limit(pagination_limit) ) async with pass_or_acquire_connection(self.db_engine) as conn: diff --git a/services/catalog/src/simcore_service_catalog/service/catalog_services.py b/services/catalog/src/simcore_service_catalog/service/catalog_services.py index 579396faae55..2da7de951771 100644 --- a/services/catalog/src/simcore_service_catalog/service/catalog_services.py +++ b/services/catalog/src/simcore_service_catalog/service/catalog_services.py @@ -144,8 +144,8 @@ async def list_latest_catalog_services( total_count, services = await repo.list_latest_services( product_name=product_name, user_id=user_id, - limit=limit, - offset=offset, + pagination_limit=limit, + pagination_offset=offset, filters=filters, ) @@ -519,11 +519,11 @@ async def list_user_service_release_history( # target service service_key: ServiceKey, # pagination - limit: PageLimitInt | None = None, - offset: NonNegativeInt | None = None, + pagination_limit: PageLimitInt | None = None, + pagination_offset: NonNegativeInt | None = None, # filters filters: ServiceDBFilters | None = None, - # options + # result options include_compatibility: bool = False, ) -> tuple[PageTotalCount, list[ServiceRelease]]: @@ -533,8 +533,8 @@ async def list_user_service_release_history( product_name=product_name, user_id=user_id, key=service_key, - limit=limit, - offset=offset, + pagination_limit=pagination_limit, + pagination_offset=pagination_offset, filters=filters, ) diff --git a/services/catalog/tests/unit/with_dbs/test_repositories.py b/services/catalog/tests/unit/with_dbs/test_repositories.py index 9f6f54d4a02f..a068cfe74b33 100644 --- a/services/catalog/tests/unit/with_dbs/test_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_repositories.py @@ -390,7 +390,7 @@ async def test_list_latest_services_with_pagination( assert service.version == expected_latest_version _, services_items = await services_repo.list_latest_services( - product_name=target_product, user_id=user_id, limit=2 + product_name=target_product, user_id=user_id, pagination_limit=2 ) assert len(services_items) == 2 @@ -711,8 +711,8 @@ async def test_get_service_history_page( product_name=target_product, user_id=user_id, key=service_key, - limit=limit, - offset=offset, + pagination_limit=limit, + pagination_offset=offset, ) assert total_count == num_versions assert len(paginated_history) == limit From 4dfd53f2d8df7f316f50b7fb1fc762663703e843 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 14:16:32 +0200 Subject: [PATCH 03/20] drafts tests for listing --- .../repository/_services_sql.py | 76 +++++ .../repository/services.py | 40 +++ .../tests/unit/with_dbs/test_repositories.py | 259 ++++++++++++++++++ 3 files changed, 375 insertions(+) diff --git a/services/catalog/src/simcore_service_catalog/repository/_services_sql.py b/services/catalog/src/simcore_service_catalog/repository/_services_sql.py index 716a1276f927..ca05641a5923 100644 --- a/services/catalog/src/simcore_service_catalog/repository/_services_sql.py +++ b/services/catalog/src/simcore_service_catalog/repository/_services_sql.py @@ -436,3 +436,79 @@ def get_service_history_stmt( .select_from(history_subquery) .group_by(history_subquery.c.key) ) + + +def all_services_total_count_stmt( + *, + product_name: ProductName, + user_id: UserID, + access_rights: AccessRightsClauses, + filters: ServiceDBFilters | None = None, +) -> sa.sql.Select: + """Statement to count all services""" + stmt = ( + sa.select(sa.func.count()) + .select_from( + services_meta_data.join( + services_access_rights, + (services_meta_data.c.key == services_access_rights.c.key) + & (services_meta_data.c.version == services_access_rights.c.version), + ).join( + user_to_groups, + (user_to_groups.c.gid == services_access_rights.c.gid), + ) + ) + .where( + (services_access_rights.c.product_name == product_name) + & (user_to_groups.c.uid == user_id) + & access_rights + ) + ) + + if filters: + stmt = apply_services_filters(stmt, filters) + + return stmt + + +def list_all_services_stmt( + *, + product_name: ProductName, + user_id: UserID, + access_rights: AccessRightsClauses, + limit: int | None = None, + offset: int | None = None, + filters: ServiceDBFilters | None = None, +) -> sa.sql.Select: + """Statement to list all services with pagination""" + stmt = ( + sa.select(*SERVICES_META_DATA_COLS) + .select_from( + services_meta_data.join( + services_access_rights, + (services_meta_data.c.key == services_access_rights.c.key) + & (services_meta_data.c.version == services_access_rights.c.version), + ).join( + user_to_groups, + (user_to_groups.c.gid == services_access_rights.c.gid), + ) + ) + .where( + (services_access_rights.c.product_name == product_name) + & (user_to_groups.c.uid == user_id) + & access_rights + ) + .order_by( + services_meta_data.c.key, sa.desc(by_version(services_meta_data.c.version)) + ) + ) + + if filters: + stmt = apply_services_filters(stmt, filters) + + if offset is not None: + stmt = stmt.offset(offset) + if limit is not None: + stmt = stmt.limit(limit) + + return stmt diff --git a/services/catalog/src/simcore_service_catalog/repository/services.py b/services/catalog/src/simcore_service_catalog/repository/services.py index 5abc9c7d827c..2adb85e3fd7e 100644 --- a/services/catalog/src/simcore_service_catalog/repository/services.py +++ b/services/catalog/src/simcore_service_catalog/repository/services.py @@ -376,6 +376,46 @@ async def get_service_with_history( ) return None + async def list_all_services( + self, + *, + # access-rights + product_name: ProductName, + user_id: UserID, + # list args: pagination + pagination_limit: int | None = None, + pagination_offset: int | None = None, + filters: ServiceDBFilters | None = None, + ) -> tuple[PositiveInt, list[ServiceMetaDataDBGet]]: + # get page count + stmt_total = _services_sql.all_services_total_count_stmt( + product_name=product_name, + user_id=user_id, + access_rights=AccessRightsClauses.can_read, + filters=filters, + ) + + # get page content + stmt_page = _services_sql.list_all_services_stmt( + product_name=product_name, + user_id=user_id, + access_rights=AccessRightsClauses.can_read, + limit=pagination_limit, + offset=pagination_offset, + filters=filters, + ) + + async with self.db_engine.connect() as conn: + result = await conn.execute(stmt_total) + total_count = result.scalar() or 0 + + items_page = [ + ServiceMetaDataDBGet.model_validate(row) + async for row in await conn.stream(stmt_page) + ] + + return (total_count, items_page) + async def list_latest_services( self, *, diff --git a/services/catalog/tests/unit/with_dbs/test_repositories.py b/services/catalog/tests/unit/with_dbs/test_repositories.py index a068cfe74b33..bd4f3e00216f 100644 --- a/services/catalog/tests/unit/with_dbs/test_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_repositories.py @@ -900,3 +900,262 @@ async def test_list_services_from_published_templates_with_invalid_service( "service {'key': 'simcore/services/dynamic/invalid-service', 'version': 'invalid'} could not be validated" in caplog.text ) + + +async def test_compare_list_all_and_latest_services( + target_product: ProductName, + create_fake_service_data: CreateFakeServiceDataCallable, + services_db_tables_injector: Callable, + services_repo: ServicesRepository, + user_id: UserID, +): + # Setup: Create multiple versions of the same service and a few distinct services + service_data: list[tuple] = [] + + # Service 1 with multiple versions + service_key_1 = "simcore/services/dynamic/multi-version" + service_versions_1 = ["1.0.0", "1.1.0", "2.0.0"] + service_data.extend( + [ + create_fake_service_data( + service_key_1, + version_, + team_access=None, + everyone_access=None, + product=target_product, + ) + for version_ in service_versions_1 + ] + ) + + # Service 2 with single version + service_key_2 = "simcore/services/dynamic/single-version" + service_data.append( + create_fake_service_data( + service_key_2, + "1.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + ) + + # Service 3 with computational type + service_key_3 = "simcore/services/comp/computational-service" + service_versions_3 = ["0.5.0", "1.0.0"] + service_data.extend( + [ + create_fake_service_data( + service_key_3, + version_, + team_access=None, + everyone_access=None, + product=target_product, + ) + for version_ in service_versions_3 + ] + ) + + await services_db_tables_injector(service_data) + + # Test 1: Compare all services vs latest without filters + total_all, all_services = await services_repo.list_all_services( + product_name=target_product, user_id=user_id + ) + total_latest, latest_services = await services_repo.list_latest_services( + product_name=target_product, user_id=user_id + ) + + # Verify counts + # All services should be 6 (3 versions of service 1, 1 of service 2, 2 of service 3) + assert total_all == 6 + # Latest services should be 3 (one latest for each distinct service key) + assert total_latest == 3 + + # Verify latest services are contained in all services + latest_key_versions = {(s.key, s.version) for s in latest_services} + all_key_versions = {(s.key, s.version) for s in all_services} + assert latest_key_versions.issubset(all_key_versions) + + # Verify latest versions are correct + latest_versions_by_key = {s.key: s.version for s in latest_services} + assert latest_versions_by_key[service_key_1] == "2.0.0" + assert latest_versions_by_key[service_key_2] == "1.0.0" + assert latest_versions_by_key[service_key_3] == "1.0.0" + + # Test 2: Using service_type filter to get only dynamic services + filters = ServiceDBFilters(service_type=ServiceType.DYNAMIC) + + total_all_filtered, all_services_filtered = await services_repo.list_all_services( + product_name=target_product, user_id=user_id, filters=filters + ) + total_latest_filtered, latest_services_filtered = ( + await services_repo.list_latest_services( + product_name=target_product, user_id=user_id, filters=filters + ) + ) + + # Verify counts with filter + assert total_all_filtered == 4 # 3 versions of service 1, 1 of service 2 + assert total_latest_filtered == 2 # 1 latest each for service 1 and 2 + + # Verify service types are correct after filtering + assert all( + s.key.startswith(DYNAMIC_SERVICE_KEY_PREFIX) for s in all_services_filtered + ) + assert all( + s.key.startswith(DYNAMIC_SERVICE_KEY_PREFIX) for s in latest_services_filtered + ) + + # Verify latest versions are correct + latest_versions_by_key = {s.key: s.version for s in latest_services_filtered} + assert latest_versions_by_key[service_key_1] == "2.0.0" + assert latest_versions_by_key[service_key_2] == "1.0.0" + assert service_key_3 not in latest_versions_by_key # Filtered out + + # Test 3: Using service_key_pattern to find specific service + filters = ServiceDBFilters(service_key_pattern="*/multi-*") + + total_all_filtered, all_services_filtered = await services_repo.list_all_services( + product_name=target_product, user_id=user_id, filters=filters + ) + total_latest_filtered, latest_services_filtered = ( + await services_repo.list_latest_services( + product_name=target_product, user_id=user_id, filters=filters + ) + ) + + # Verify counts with key pattern filter + assert total_all_filtered == 3 # All 3 versions of service 1 + assert total_latest_filtered == 1 # Only latest version of service 1 + + # Verify service key pattern is matched + assert all(s.key == service_key_1 for s in all_services_filtered) + assert all(s.key == service_key_1 for s in latest_services_filtered) + + # Test 4: Pagination + # Get first page (limit=2) + total_all_page1, all_services_page1 = await services_repo.list_all_services( + product_name=target_product, + user_id=user_id, + pagination_limit=2, + pagination_offset=0, + ) + + # Get second page (limit=2, offset=2) + total_all_page2, all_services_page2 = await services_repo.list_all_services( + product_name=target_product, + user_id=user_id, + pagination_limit=2, + pagination_offset=2, + ) + + # Verify pagination + assert total_all_page1 == 6 # Total count should still be total + assert len(all_services_page1) == 2 # But only 2 items on first page + assert len(all_services_page2) == 2 # And 2 items on second page + + # Ensure pages have different items + page1_key_versions = {(s.key, s.version) for s in all_services_page1} + page2_key_versions = {(s.key, s.version) for s in all_services_page2} + assert not page1_key_versions.intersection(page2_key_versions) + + +async def test_list_all_services_empty_database( + target_product: ProductName, + services_repo: ServicesRepository, + user_id: UserID, +): + """Test list_all_services and list_latest_services with an empty database.""" + # Test with empty database + total_all, all_services = await services_repo.list_all_services( + product_name=target_product, user_id=user_id + ) + total_latest, latest_services = await services_repo.list_latest_services( + product_name=target_product, user_id=user_id + ) + + assert total_all == 0 + assert len(all_services) == 0 + assert total_latest == 0 + assert len(latest_services) == 0 + + +async def test_list_all_services_deprecated_versions( + target_product: ProductName, + create_fake_service_data: CreateFakeServiceDataCallable, + services_db_tables_injector: Callable, + services_repo: ServicesRepository, + user_id: UserID, +): + """Test that list_all_services includes deprecated versions while list_latest_services ignores them.""" + from datetime import datetime, timedelta + + # Create a service with regular and deprecated versions + service_key = "simcore/services/dynamic/with-deprecated" + service_data = [] + + # Add regular version + service_data.append( + create_fake_service_data( + service_key, + "1.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + ) + + # Add deprecated version (with higher version number) + deprecated_service = create_fake_service_data( + service_key, + "2.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + # Set deprecated timestamp to yesterday + deprecated_service[0]["deprecated"] = datetime.now() - timedelta(days=1) + service_data.append(deprecated_service) + + # Add newer non-deprecated version + service_data.append( + create_fake_service_data( + service_key, + "3.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + ) + + await services_db_tables_injector(service_data) + + # Get all services - should include both deprecated and non-deprecated + total_all, all_services = await services_repo.list_all_services( + product_name=target_product, user_id=user_id + ) + + # Get latest services - should only show latest non-deprecated + total_latest, latest_services = await services_repo.list_latest_services( + product_name=target_product, user_id=user_id + ) + + # Verify counts + assert total_all == 3 # All 3 versions + + # Verify latest is the newest non-deprecated version + assert len(latest_services) == 1 + assert latest_services[0].key == service_key + assert latest_services[0].version == "3.0.0" + + # Get versions from all services + versions = [s.version for s in all_services if s.key == service_key] + assert sorted(versions) == ["1.0.0", "2.0.0", "3.0.0"] + + # Verify the deprecated status is correctly set + for service in all_services: + if service.key == service_key and service.version == "2.0.0": + assert service.deprecated is not None + else: + assert service.deprecated is None From 57d1ce7dec2750196e213f44cb2583b59402a23d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 14:40:06 +0200 Subject: [PATCH 04/20] draft implementation --- .../repository/services.py | 65 +++++++++++++++---- .../tests/unit/with_dbs/test_repositories.py | 1 + 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/repository/services.py b/services/catalog/src/simcore_service_catalog/repository/services.py index 2adb85e3fd7e..12e0d88a0fe6 100644 --- a/services/catalog/src/simcore_service_catalog/repository/services.py +++ b/services/catalog/src/simcore_service_catalog/repository/services.py @@ -387,24 +387,61 @@ async def list_all_services( pagination_offset: int | None = None, filters: ServiceDBFilters | None = None, ) -> tuple[PositiveInt, list[ServiceMetaDataDBGet]]: - # get page count - stmt_total = _services_sql.all_services_total_count_stmt( - product_name=product_name, - user_id=user_id, - access_rights=AccessRightsClauses.can_read, - filters=filters, + # Create base query that's common to both count and content queries + base_query = ( + sa.select(services_meta_data.c.key, services_meta_data.c.version) + .select_from( + services_meta_data.join( + services_access_rights, + (services_meta_data.c.key == services_access_rights.c.key) + & ( + services_meta_data.c.version == services_access_rights.c.version + ), + ).join( + user_to_groups, + (user_to_groups.c.gid == services_access_rights.c.gid), + ) + ) + .where( + (services_access_rights.c.product_name == product_name) + & (user_to_groups.c.uid == user_id) + & AccessRightsClauses.can_read + ) + .distinct() ) - # get page content - stmt_page = _services_sql.list_all_services_stmt( - product_name=product_name, - user_id=user_id, - access_rights=AccessRightsClauses.can_read, - limit=pagination_limit, - offset=pagination_offset, - filters=filters, + if filters: + base_query = _services_sql.apply_services_filters(base_query, filters) + + # Subquery for efficient counting and further joins + subquery = base_query.subquery() + + # Count query - only counts distinct key/version pairs + stmt_total = sa.select(sa.func.count()).select_from(subquery) + + # Content query - gets all details with pagination + stmt_page = ( + sa.select(*SERVICES_META_DATA_COLS) + .select_from( + subquery.join( + services_meta_data, + (subquery.c.key == services_meta_data.c.key) + & (subquery.c.version == services_meta_data.c.version), + ) + ) + .order_by( + services_meta_data.c.key, + sa.desc(_services_sql.by_version(services_meta_data.c.version)), + ) ) + # Apply pagination to content query + if pagination_offset is not None: + stmt_page = stmt_page.offset(pagination_offset) + if pagination_limit is not None: + stmt_page = stmt_page.limit(pagination_limit) + + # Execute both queries async with self.db_engine.connect() as conn: result = await conn.execute(stmt_total) total_count = result.scalar() or 0 diff --git a/services/catalog/tests/unit/with_dbs/test_repositories.py b/services/catalog/tests/unit/with_dbs/test_repositories.py index bd4f3e00216f..ec8fca128256 100644 --- a/services/catalog/tests/unit/with_dbs/test_repositories.py +++ b/services/catalog/tests/unit/with_dbs/test_repositories.py @@ -1052,6 +1052,7 @@ async def test_compare_list_all_and_latest_services( # Verify pagination assert total_all_page1 == 6 # Total count should still be total + assert total_all_page2 == 6 assert len(all_services_page1) == 2 # But only 2 items on first page assert len(all_services_page2) == 2 # And 2 items on second page From d00b71c8beb423a42024f7d80b628d48335ad9b0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 19:18:16 +0200 Subject: [PATCH 05/20] =?UTF-8?q?=E2=9C=A8=20[Backend]=20Add=20async=20fun?= =?UTF-8?q?ction=20to=20list=20all=20catalog=20services=20with=20paginatio?= =?UTF-8?q?n=20and=20access=20rights?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/catalog_services.py | 179 +++++++++++++++++- .../with_dbs/test_service_catalog_services.py | 108 ++++++++++- 2 files changed, 280 insertions(+), 7 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/service/catalog_services.py b/services/catalog/src/simcore_service_catalog/service/catalog_services.py index 2da7de951771..1b1c797b7750 100644 --- a/services/catalog/src/simcore_service_catalog/service/catalog_services.py +++ b/services/catalog/src/simcore_service_catalog/service/catalog_services.py @@ -14,13 +14,13 @@ from models_library.basic_types import VersionStr from models_library.groups import GroupID from models_library.products import ProductName -from models_library.rest_pagination import PageLimitInt, PageTotalCount +from models_library.rest_pagination import PageLimitInt, PageOffsetInt, PageTotalCount from models_library.services_access import ServiceGroupAccessRightsV2 from models_library.services_history import Compatibility, ServiceRelease from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import HttpUrl, NonNegativeInt +from pydantic import HttpUrl from servicelib.logging_errors import ( create_troubleshotting_log_kwargs, ) @@ -34,6 +34,7 @@ from ..models.services_db import ( ServiceAccessRightsDB, ServiceDBFilters, + ServiceMetaDataDBGet, ServiceMetaDataDBPatch, ServiceWithHistoryDBGet, ) @@ -48,7 +49,7 @@ def _aggregate( - service_db: ServiceWithHistoryDBGet, + service_db: ServiceWithHistoryDBGet | ServiceMetaDataDBGet, access_rights_db: list[ServiceAccessRightsDB], service_manifest: ServiceMetaDataPublished, ) -> dict: @@ -64,7 +65,10 @@ def _aggregate( "service_type": service_manifest.service_type, "contact": service_manifest.contact, "authors": service_manifest.authors, - "owner": (service_db.owner_email if service_db.owner_email else None), + # Check if owner attribute is available in the service_db object + "owner": ( + service_db.owner_email if hasattr(service_db, "owner_email") else None + ), "inputs": service_manifest.inputs or {}, "outputs": service_manifest.outputs or {}, "boot_options": service_manifest.boot_options, @@ -129,6 +133,169 @@ def _to_get_schema( ) +async def list_all_catalog_services( + repo: ServicesRepository, + director_api: DirectorClient, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt | None, + offset: PageOffsetInt = 0, + filters: ServiceDBFilters | None = None, + include_history: bool = False, +) -> tuple[PageTotalCount, list[ServiceGetV2]]: + """Lists all catalog services. + + This is different from list_latest_catalog_services which only returns the latest version of each service. + + Args: + repo: Repository for services + director_api: Director API client + product_name: Product name + user_id: User ID + limit: Pagination limit + offset: Pagination offset + filters: Filters to apply + include_history: Whether to include full version history for each service (default: False) + When False, only a minimal history entry with the current version is included + When True, complete version history is fetched for each service (can be slower) + + Returns: + Tuple of total count and list of services + """ + # Get all services with pagination + total_count, services = await repo.list_all_services( + product_name=product_name, + user_id=user_id, + pagination_limit=limit, + pagination_offset=offset, + filters=filters, + ) + + if services: + # Inject access-rights + access_rights: dict[tuple[str, str], list[ServiceAccessRightsDB]] = ( + await repo.batch_get_services_access_rights( + ((sc.key, sc.version) for sc in services), product_name=product_name + ) + ) + if not access_rights: + raise CatalogForbiddenError( + name="any service", + user_id=user_id, + product_name=product_name, + ) + + # Get manifest of those with access rights + got = await manifest.get_batch_services( + [ + (sc.key, sc.version) + for sc in services + if access_rights.get((sc.key, sc.version)) + ], + director_api, + ) + service_manifest = { + (sc.key, sc.version): sc + for sc in got + if isinstance(sc, ServiceMetaDataPublished) + } + + # Log a warning for missing services + missing_services = [ + (sc.key, sc.version) + for sc in services + if (sc.key, sc.version) not in service_manifest + ] + if missing_services: + msg = f"Found {len(missing_services)} services that are in the database but missing in the registry manifest" + _logger.warning( + **create_troubleshotting_log_kwargs( + msg, + error=CatalogInconsistentError( + missing_services=missing_services, + user_id=user_id, + product_name=product_name, + filters=filters, + limit=limit, + offset=offset, + ), + tip="This might be due to malfunction of the background-task or that this call was done while the sync was taking place", + ) + ) + # NOTE: tests should fail if this happens but it is not a critical error so it is ignored in production + assert len(missing_services) == 0, msg # nosec + + # Prepare for fetching history data if needed + service_histories: dict[str, list[ServiceRelease]] = {} + compatibility_maps: dict[str, dict[ServiceVersion, Compatibility | None]] = {} + + # Fetch history data if requested + if include_history: + for sc in services: + if access_rights.get((sc.key, sc.version)): + # Fetch history for this service + service = await repo.get_service_with_history( + product_name=product_name, + user_id=user_id, + key=sc.key, + version=sc.version, + ) + if service: + # Evaluate compatibility map + compatibility_map = await evaluate_service_compatibility_map( + repo, + product_name=product_name, + user_id=user_id, + service_release_history=service.history, + ) + compatibility_maps[sc.key] = compatibility_map + + # Create history entries + service_histories[sc.key] = [ + ServiceRelease.model_construct( + version=h.version, + version_display=h.version_display, + released=h.created, + retired=h.deprecated, + compatibility=compatibility_map.get(h.version), + ) + for h in service.history + ] + + # Aggregate the services manifest and access-rights + items = [] + for sc in services: + ar = access_rights.get((sc.key, sc.version)) + sm = service_manifest.get((sc.key, sc.version)) + if ar and sm: + # Base service data + service_data = _aggregate( + service_db=sc, + access_rights_db=ar, + service_manifest=sm, + ) + + # Add history based on include_history parameter + if include_history and sc.key in service_histories: + service_data["history"] = service_histories[sc.key] + else: + # Create minimal history with just the current version + service_data["history"] = [ + ServiceRelease.model_construct( + version=sc.version, + version_display=sc.version_display, + released=sc.created, + retired=sc.deprecated, + compatibility=None, + ) + ] + + items.append(ServiceGetV2.model_validate(service_data)) + + return total_count, items + + async def list_latest_catalog_services( repo: ServicesRepository, director_api: DirectorClient, @@ -136,7 +303,7 @@ async def list_latest_catalog_services( product_name: ProductName, user_id: UserID, limit: PageLimitInt | None, - offset: NonNegativeInt = 0, + offset: PageOffsetInt = 0, filters: ServiceDBFilters | None = None, ) -> tuple[PageTotalCount, list[LatestServiceGet]]: @@ -520,7 +687,7 @@ async def list_user_service_release_history( service_key: ServiceKey, # pagination pagination_limit: PageLimitInt | None = None, - pagination_offset: NonNegativeInt | None = None, + pagination_offset: PageOffsetInt | None = None, # filters filters: ServiceDBFilters | None = None, # result options diff --git a/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py b/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py index 039b5f41ccdd..73d6b3ee4611 100644 --- a/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py +++ b/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py @@ -108,7 +108,7 @@ async def director_client(app: FastAPI) -> DirectorClient: return director_api -async def test_list_services_paginated( +async def test_list_latest_catalog_services( background_sync_task_mocked: None, rabbitmq_and_rpc_setup_disabled: None, mocked_director_rest_api: MockRouter, @@ -273,3 +273,109 @@ async def test_batch_get_my_services( }, ] ) + + +async def test_list_all_vs_latest_services( + background_sync_task_mocked: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_rest_api: MockRouter, + target_product: ProductName, + services_repo: ServicesRepository, + user_id: UserID, + director_client: DirectorClient, + num_services: int, + num_versions_per_service: int, +): + """Test that list_all_catalog_services returns all services while + list_latest_catalog_services returns only the latest version of each service. + """ + # No pagination to get all services + limit = None + offset = 0 + + # Get latest services first + latest_total_count, latest_items = ( + await catalog_services.list_latest_catalog_services( + services_repo, + director_client, + product_name=target_product, + user_id=user_id, + limit=limit, + offset=offset, + ) + ) + + # Get all services + all_total_count, all_items = await catalog_services.list_all_catalog_services( + services_repo, + director_client, + product_name=target_product, + user_id=user_id, + limit=limit, + offset=offset, + ) + + # Test with include_history=True option + all_total_count_with_history, all_items_with_history = ( + await catalog_services.list_all_catalog_services( + services_repo, + director_client, + product_name=target_product, + user_id=user_id, + limit=limit, + offset=offset, + include_history=True, + ) + ) + + # Verify counts + # - latest_total_count should equal num_services since we only get the latest version of each service + # - all_total_count should equal num_services * num_versions_per_service since we get all versions + assert latest_total_count == num_services + assert all_total_count == num_services * num_versions_per_service + assert all_total_count_with_history == num_services * num_versions_per_service + + # Verify we got the expected number of items + assert len(latest_items) == num_services + assert len(all_items) == num_services * num_versions_per_service + assert len(all_items_with_history) == num_services * num_versions_per_service + + # Collect all service keys from latest items + latest_keys = {item.key for item in latest_items} + + # Verify all returned items have the expected structure + for item in all_items: + # Each item should have exactly one history entry when include_history=False + assert len(item.history) == 1 + # The history entry should match the current version + assert item.history[0].version == item.version + # Each service key should be in latest_keys + assert item.key in latest_keys + + # Verify history is included when include_history=True + for item in all_items_with_history: + if item.key in latest_keys: + service_key = item.key + # Find the corresponding service in the latest items + latest_service = next(s for s in latest_items if s.key == service_key) + # The latest version service should have its history included + if item.version == latest_service.version: + # Only check that there's proper history if this is the latest version + # Some services might have more history than others + assert len(item.history) >= 1 + + # Group all items by key + key_to_all_versions = {} + for item in all_items: + if item.key not in key_to_all_versions: + key_to_all_versions[item.key] = [] + key_to_all_versions[item.key].append(item) + + # For each service key, verify we have the expected number of versions + for key, versions in key_to_all_versions.items(): + assert len(versions) == num_versions_per_service + + # Find this service in latest_items + latest_item = next(item for item in latest_items if item.key == key) + # The latest version should be in the list + assert any(item.version == latest_item.version for item in versions) From df1985bea1c6e385eabb489b798131b25038b25d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 19:39:30 +0200 Subject: [PATCH 06/20] refactoring --- .../service/catalog_services.py | 228 +++++++++++------- 1 file changed, 137 insertions(+), 91 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/service/catalog_services.py b/services/catalog/src/simcore_service_catalog/service/catalog_services.py index 1b1c797b7750..a71149dca5e0 100644 --- a/services/catalog/src/simcore_service_catalog/service/catalog_services.py +++ b/services/catalog/src/simcore_service_catalog/service/catalog_services.py @@ -2,7 +2,7 @@ import logging from contextlib import suppress -from typing import Literal +from typing import Literal, TypeVar from models_library.api_schemas_catalog.services import ( LatestServiceGet, @@ -47,6 +47,9 @@ _logger = logging.getLogger(__name__) +# Type variable for service models that can be returned from list functions +T = TypeVar("T", ServiceGetV2, LatestServiceGet) + def _aggregate( service_db: ServiceWithHistoryDBGet | ServiceMetaDataDBGet, @@ -133,59 +136,71 @@ def _to_get_schema( ) -async def list_all_catalog_services( +async def _get_services_with_access_rights( repo: ServicesRepository, - director_api: DirectorClient, - *, + services: list[ServiceWithHistoryDBGet] | list[ServiceMetaDataDBGet], product_name: ProductName, user_id: UserID, - limit: PageLimitInt | None, - offset: PageOffsetInt = 0, - filters: ServiceDBFilters | None = None, - include_history: bool = False, -) -> tuple[PageTotalCount, list[ServiceGetV2]]: - """Lists all catalog services. - - This is different from list_latest_catalog_services which only returns the latest version of each service. +) -> dict[tuple[str, str], list[ServiceAccessRightsDB]]: + """Common function to get access rights for a list of services. Args: repo: Repository for services - director_api: Director API client + services: List of services to get access rights for product_name: Product name user_id: User ID - limit: Pagination limit - offset: Pagination offset - filters: Filters to apply - include_history: Whether to include full version history for each service (default: False) - When False, only a minimal history entry with the current version is included - When True, complete version history is fetched for each service (can be slower) Returns: - Tuple of total count and list of services + Dictionary mapping (key, version) to list of access rights + + Raises: + CatalogForbiddenError: If no access rights are found for any service """ - # Get all services with pagination - total_count, services = await repo.list_all_services( - product_name=product_name, - user_id=user_id, - pagination_limit=limit, - pagination_offset=offset, - filters=filters, - ) + if not services: + return {} - if services: - # Inject access-rights - access_rights: dict[tuple[str, str], list[ServiceAccessRightsDB]] = ( - await repo.batch_get_services_access_rights( - ((sc.key, sc.version) for sc in services), product_name=product_name - ) + # Inject access-rights + access_rights = await repo.batch_get_services_access_rights( + ((sc.key, sc.version) for sc in services), product_name=product_name + ) + if not access_rights: + raise CatalogForbiddenError( + name="any service", + user_id=user_id, + product_name=product_name, ) - if not access_rights: - raise CatalogForbiddenError( - name="any service", - user_id=user_id, - product_name=product_name, - ) + return access_rights + + +async def _get_services_manifests( + services: list[ServiceWithHistoryDBGet] | list[ServiceMetaDataDBGet], + access_rights: dict[tuple[str, str], list[ServiceAccessRightsDB]], + director_api: DirectorClient, + product_name: ProductName, + user_id: UserID, + filters: ServiceDBFilters | None, + limit: PageLimitInt | None, + offset: PageOffsetInt | None, +) -> dict[tuple[str, str], ServiceMetaDataPublished]: + """Common function to get service manifests from director. + + Args: + services: List of services to get manifests for + access_rights: Dictionary mapping (key, version) to list of access rights + director_api: Director API client + product_name: Product name + user_id: User ID + filters: Filters that were applied + limit: Pagination limit that was applied + offset: Pagination offset that was applied + + Returns: + Dictionary mapping (key, version) to manifest + + Raises: + CatalogInconsistentError: Logs warning if some services are missing from manifest + """ # Get manifest of those with access rights got = await manifest.get_batch_services( [ @@ -226,6 +241,63 @@ async def list_all_catalog_services( # NOTE: tests should fail if this happens but it is not a critical error so it is ignored in production assert len(missing_services) == 0, msg # nosec + return service_manifest + + +async def list_all_catalog_services( + repo: ServicesRepository, + director_api: DirectorClient, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt | None, + offset: PageOffsetInt = 0, + filters: ServiceDBFilters | None = None, + include_history: bool = False, +) -> tuple[PageTotalCount, list[ServiceGetV2]]: + """Lists all catalog services. + + This is different from list_latest_catalog_services which only returns the latest version of each service. + + Args: + repo: Repository for services + director_api: Director API client + product_name: Product name + user_id: User ID + limit: Pagination limit + offset: Pagination offset + filters: Filters to apply + include_history: Whether to include full version history for each service (default: False) + When False, only a minimal history entry with the current version is included + When True, complete version history is fetched for each service (can be slower) + + Returns: + Tuple of total count and list of services + """ + # Get all services with pagination + total_count, services = await repo.list_all_services( + product_name=product_name, + user_id=user_id, + pagination_limit=limit, + pagination_offset=offset, + filters=filters, + ) + + # Get access rights and manifests + access_rights = await _get_services_with_access_rights( + repo, services, product_name, user_id + ) + service_manifest = await _get_services_manifests( + services, + access_rights, + director_api, + product_name, + user_id, + filters, + limit, + offset, + ) + # Prepare for fetching history data if needed service_histories: dict[str, list[ServiceRelease]] = {} compatibility_maps: dict[str, dict[ServiceVersion, Compatibility | None]] = {} @@ -306,7 +378,20 @@ async def list_latest_catalog_services( offset: PageOffsetInt = 0, filters: ServiceDBFilters | None = None, ) -> tuple[PageTotalCount, list[LatestServiceGet]]: + """Lists latest versions of catalog services. + Args: + repo: Repository for services + director_api: Director API client + product_name: Product name + user_id: UserID + limit: Pagination limit + offset: Pagination offset + filters: Filters to apply + + Returns: + Tuple of total count and list of latest services + """ # defines the order total_count, services = await repo.list_latest_services( product_name=product_name, @@ -316,59 +401,20 @@ async def list_latest_catalog_services( filters=filters, ) - if services: - # injects access-rights - access_rights: dict[tuple[str, str], list[ServiceAccessRightsDB]] = ( - await repo.batch_get_services_access_rights( - ((sc.key, sc.version) for sc in services), product_name=product_name - ) - ) - if not access_rights: - raise CatalogForbiddenError( - name="any service", - user_id=user_id, - product_name=product_name, - ) - - # Get manifest of those with access rights - got = await manifest.get_batch_services( - [ - (sc.key, sc.version) - for sc in services - if access_rights.get((sc.key, sc.version)) - ], + # Get access rights and manifests using shared functions + access_rights = await _get_services_with_access_rights( + repo, services, product_name, user_id + ) + service_manifest = await _get_services_manifests( + services, + access_rights, director_api, + product_name, + user_id, + filters, + limit, + offset, ) - service_manifest = { - (sc.key, sc.version): sc - for sc in got - if isinstance(sc, ServiceMetaDataPublished) - } - - # Log a warning for missing services - missing_services = [ - (sc.key, sc.version) - for sc in services - if (sc.key, sc.version) not in service_manifest - ] - if missing_services: - msg = f"Found {len(missing_services)} services that are in the database but missing in the registry manifest" - _logger.warning( - **create_troubleshotting_log_kwargs( - msg, - error=CatalogInconsistentError( - missing_services=missing_services, - user_id=user_id, - product_name=product_name, - filters=filters, - limit=limit, - offset=offset, - ), - tip="This might be due to malfunction of the background-task or that this call was done while the sync was taking place", - ) - ) - # NOTE: tests should fail if this happens but it is not a critical error so it is ignored in production - assert len(missing_services) == 0, msg # nosec # Aggregate the services manifest and access-rights items = [ From 9e1892c678b781a02b1b63b4cc901226f8f163a6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 19:55:44 +0200 Subject: [PATCH 07/20] minimal model for a service --- .../api_schemas_catalog/services.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index 565cd814a805..cb1fdf29347a 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -172,23 +172,26 @@ def _update_json_schema_extra(schema: JsonDict) -> None: ) -class _BaseServiceGetV2(CatalogOutputSchema): - # Model used in catalog's rpc and rest interfaces +class ServiceSummary(CatalogOutputSchema): key: ServiceKey version: ServiceVersion - name: str + description: str + + version_display: str | None = None + + contact: LowerCaseEmailStr | None + + +class _BaseServiceGetV2(ServiceSummary): + # Model used in catalog's rpc and rest interfaces thumbnail: HttpUrl | None = None icon: HttpUrl | None = None - description: str description_ui: bool = False - version_display: str | None = None - service_type: Annotated[ServiceType, Field(alias="type")] - contact: LowerCaseEmailStr | None authors: Annotated[list[Author], Field(min_length=1)] owner: Annotated[ LowerCaseEmailStr | None, From 97b914fab96ba1e51d7a42f8c95f491fe3ce0846 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 20:05:28 +0200 Subject: [PATCH 08/20] =?UTF-8?q?=E2=9C=A8=20[Backend]=20Enhance=20catalog?= =?UTF-8?q?=20services=20to=20return=20summaries=20and=20add=20new=20PageR?= =?UTF-8?q?pc=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_schemas_catalog/services.py | 3 + .../service/catalog_services.py | 96 ++++++------------- .../with_dbs/test_service_catalog_services.py | 48 ++-------- 3 files changed, 40 insertions(+), 107 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index cb1fdf29347a..dbe07757b9b2 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -341,6 +341,9 @@ def _update_json_schema_extra(schema: JsonDict) -> None: ServiceRelease ] +# Create PageRpc types +PageRpcServiceSummary = PageRpc[ServiceSummary] + ServiceResourcesGet: TypeAlias = ServiceResourcesDict diff --git a/services/catalog/src/simcore_service_catalog/service/catalog_services.py b/services/catalog/src/simcore_service_catalog/service/catalog_services.py index a71149dca5e0..ea7f1b6282b7 100644 --- a/services/catalog/src/simcore_service_catalog/service/catalog_services.py +++ b/services/catalog/src/simcore_service_catalog/service/catalog_services.py @@ -8,6 +8,7 @@ LatestServiceGet, MyServiceGet, ServiceGetV2, + ServiceSummary, ServiceUpdateV2, ) from models_library.api_schemas_directorv2.services import ServiceExtras @@ -89,6 +90,21 @@ def _aggregate( } +def _aggregate_summary( + service_db: ServiceWithHistoryDBGet | ServiceMetaDataDBGet, + service_manifest: ServiceMetaDataPublished, +) -> dict: + """Creates a minimal dictionary with only the fields needed for ServiceSummary""" + return { + "key": service_db.key, + "version": service_db.version, + "name": service_db.name, + "description": service_db.description, + "version_display": service_db.version_display, + "contact": service_manifest.contact, + } + + def _to_latest_get_schema( service_db: ServiceWithHistoryDBGet, access_rights_db: list[ServiceAccessRightsDB], @@ -244,7 +260,7 @@ async def _get_services_manifests( return service_manifest -async def list_all_catalog_services( +async def list_all_service_summaries( repo: ServicesRepository, director_api: DirectorClient, *, @@ -253,11 +269,11 @@ async def list_all_catalog_services( limit: PageLimitInt | None, offset: PageOffsetInt = 0, filters: ServiceDBFilters | None = None, - include_history: bool = False, -) -> tuple[PageTotalCount, list[ServiceGetV2]]: - """Lists all catalog services. +) -> tuple[PageTotalCount, list[ServiceSummary]]: + """Lists all catalog services with minimal information. - This is different from list_latest_catalog_services which only returns the latest version of each service. + This is different from list_latest_catalog_services which only returns the latest version of each service + and includes complete service information. Args: repo: Repository for services @@ -267,12 +283,9 @@ async def list_all_catalog_services( limit: Pagination limit offset: Pagination offset filters: Filters to apply - include_history: Whether to include full version history for each service (default: False) - When False, only a minimal history entry with the current version is included - When True, complete version history is fetched for each service (can be slower) Returns: - Tuple of total count and list of services + Tuple of total count and list of service summaries """ # Get all services with pagination total_count, services = await repo.list_all_services( @@ -298,72 +311,17 @@ async def list_all_catalog_services( offset, ) - # Prepare for fetching history data if needed - service_histories: dict[str, list[ServiceRelease]] = {} - compatibility_maps: dict[str, dict[ServiceVersion, Compatibility | None]] = {} - - # Fetch history data if requested - if include_history: - for sc in services: - if access_rights.get((sc.key, sc.version)): - # Fetch history for this service - service = await repo.get_service_with_history( - product_name=product_name, - user_id=user_id, - key=sc.key, - version=sc.version, - ) - if service: - # Evaluate compatibility map - compatibility_map = await evaluate_service_compatibility_map( - repo, - product_name=product_name, - user_id=user_id, - service_release_history=service.history, - ) - compatibility_maps[sc.key] = compatibility_map - - # Create history entries - service_histories[sc.key] = [ - ServiceRelease.model_construct( - version=h.version, - version_display=h.version_display, - released=h.created, - retired=h.deprecated, - compatibility=compatibility_map.get(h.version), - ) - for h in service.history - ] - - # Aggregate the services manifest and access-rights + # Create service summaries items = [] for sc in services: - ar = access_rights.get((sc.key, sc.version)) sm = service_manifest.get((sc.key, sc.version)) - if ar and sm: - # Base service data - service_data = _aggregate( + if access_rights.get((sc.key, sc.version)) and sm: + # Create a minimal ServiceSummary + service_data = _aggregate_summary( service_db=sc, - access_rights_db=ar, service_manifest=sm, ) - - # Add history based on include_history parameter - if include_history and sc.key in service_histories: - service_data["history"] = service_histories[sc.key] - else: - # Create minimal history with just the current version - service_data["history"] = [ - ServiceRelease.model_construct( - version=sc.version, - version_display=sc.version_display, - released=sc.created, - retired=sc.deprecated, - compatibility=None, - ) - ] - - items.append(ServiceGetV2.model_validate(service_data)) + items.append(ServiceSummary.model_validate(service_data)) return total_count, items diff --git a/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py b/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py index 73d6b3ee4611..5faaf6b384d4 100644 --- a/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py +++ b/services/catalog/tests/unit/with_dbs/test_service_catalog_services.py @@ -10,7 +10,7 @@ import pytest from fastapi import FastAPI -from models_library.api_schemas_catalog.services import MyServiceGet +from models_library.api_schemas_catalog.services import MyServiceGet, ServiceSummary from models_library.products import ProductName from models_library.users import UserID from pydantic import TypeAdapter @@ -286,8 +286,8 @@ async def test_list_all_vs_latest_services( num_services: int, num_versions_per_service: int, ): - """Test that list_all_catalog_services returns all services while - list_latest_catalog_services returns only the latest version of each service. + """Test that list_all_catalog_services returns all services as summaries while + list_latest_catalog_services returns only the latest version of each service with full details. """ # No pagination to get all services limit = None @@ -305,8 +305,8 @@ async def test_list_all_vs_latest_services( ) ) - # Get all services - all_total_count, all_items = await catalog_services.list_all_catalog_services( + # Get all services as summaries + all_total_count, all_items = await catalog_services.list_all_service_summaries( services_repo, director_client, product_name=target_product, @@ -315,54 +315,26 @@ async def test_list_all_vs_latest_services( offset=offset, ) - # Test with include_history=True option - all_total_count_with_history, all_items_with_history = ( - await catalog_services.list_all_catalog_services( - services_repo, - director_client, - product_name=target_product, - user_id=user_id, - limit=limit, - offset=offset, - include_history=True, - ) - ) - # Verify counts # - latest_total_count should equal num_services since we only get the latest version of each service # - all_total_count should equal num_services * num_versions_per_service since we get all versions assert latest_total_count == num_services assert all_total_count == num_services * num_versions_per_service - assert all_total_count_with_history == num_services * num_versions_per_service # Verify we got the expected number of items assert len(latest_items) == num_services assert len(all_items) == num_services * num_versions_per_service - assert len(all_items_with_history) == num_services * num_versions_per_service # Collect all service keys from latest items latest_keys = {item.key for item in latest_items} # Verify all returned items have the expected structure for item in all_items: - # Each item should have exactly one history entry when include_history=False - assert len(item.history) == 1 - # The history entry should match the current version - assert item.history[0].version == item.version - # Each service key should be in latest_keys + # Each summary should have the basic fields assert item.key in latest_keys - - # Verify history is included when include_history=True - for item in all_items_with_history: - if item.key in latest_keys: - service_key = item.key - # Find the corresponding service in the latest items - latest_service = next(s for s in latest_items if s.key == service_key) - # The latest version service should have its history included - if item.version == latest_service.version: - # Only check that there's proper history if this is the latest version - # Some services might have more history than others - assert len(item.history) >= 1 + assert item.name + assert item.description is not None + assert isinstance(item, ServiceSummary) # Group all items by key key_to_all_versions = {} @@ -377,5 +349,5 @@ async def test_list_all_vs_latest_services( # Find this service in latest_items latest_item = next(item for item in latest_items if item.key == key) - # The latest version should be in the list + # Verify there's a summary item with the same version as the latest assert any(item.version == latest_item.version for item in versions) From ddc6aacf89dac2692835c448c4bbf0fb391ee468 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 20:15:02 +0200 Subject: [PATCH 09/20] =?UTF-8?q?=E2=9C=A8=20[Backend]=20Implement=20pagin?= =?UTF-8?q?ated=20listing=20of=20service=20summaries=20for=20improved=20pe?= =?UTF-8?q?rformance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../rpc_interfaces/catalog/services.py | 50 ++++ .../api/rpc/_services.py | 57 +++++ .../tests/unit/with_dbs/test_api_rpc.py | 214 +++++++++++++++++- 3 files changed, 318 insertions(+), 3 deletions(-) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py index 58a8f289ebfb..7ac19275fb67 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py @@ -17,9 +17,11 @@ MyServiceGet, PageRpcLatestServiceGet, PageRpcServiceRelease, + PageRpcServiceSummary, ServiceGetV2, ServiceListFilters, ServiceRelease, + ServiceSummary, ServiceUpdateV2, ) from models_library.api_schemas_catalog.services_ports import ServicePortGet @@ -256,3 +258,51 @@ async def get_service_ports( TypeAdapter(list[ServicePortGet]).validate_python(result) is not None ) # nosec return cast(list[ServicePortGet], result) + + +@validate_call(config={"arbitrary_types_allowed": True}) +async def list_all_services_summaries_paginated( # pylint: disable=too-many-arguments + rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + offset: PageOffsetInt = 0, + filters: ServiceListFilters | None = None, +) -> PageRpcServiceSummary: + """Lists all services with pagination, including all versions of each service. + + Returns a lightweight summary view of services for better performance. + + Args: + rpc_client: RPC client instance + product_name: Product name + user_id: User ID + limit: Maximum number of items to return + offset: Number of items to skip + filters: Optional filters to apply + + Returns: + Paginated list of all services as summaries + + Raises: + ValidationError: on invalid arguments + CatalogForbiddenError: no access-rights to list services + """ + result = await rpc_client.request( + CATALOG_RPC_NAMESPACE, + TypeAdapter(RPCMethodName).validate_python( + list_all_services_summaries_paginated.__name__ + ), + product_name=product_name, + user_id=user_id, + limit=limit, + offset=offset, + filters=filters, + timeout_s=40 * RPC_REQUEST_DEFAULT_TIMEOUT_S, + ) + + assert ( + TypeAdapter(PageRpc[ServiceSummary]).validate_python(result) is not None + ) # nosec + return cast(PageRpc[ServiceSummary], result) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index b2c722572358..73d5bc3e562f 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -7,6 +7,7 @@ MyServiceGet, PageRpcLatestServiceGet, PageRpcServiceRelease, + PageRpcServiceSummary, ServiceGetV2, ServiceListFilters, ServiceUpdateV2, @@ -310,3 +311,59 @@ async def get_service_ports( ) for port in service_ports ] + + +@router.expose(reraise_if_error_type=(CatalogForbiddenError, ValidationError)) +@_profile_rpc_call +@validate_call(config={"arbitrary_types_allowed": True}) +async def list_all_services_summaries_paginated( + app: FastAPI, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + offset: PageOffsetInt = 0, + filters: ServiceListFilters | None = None, +) -> PageRpcServiceSummary: + """Lists all services with pagination, including all versions of each service. + + Returns a lightweight summary view of services for better performance compared to + full service details. This is useful for listings where complete details aren't needed. + + Args: + app: FastAPI application + product_name: Product name + user_id: User ID + limit: Maximum number of items to return + offset: Number of items to skip + filters: Optional filters to apply + + Returns: + Paginated list of all services as summaries + """ + assert app.state.engine # nosec + + total_count, items = await catalog_services.list_all_service_summaries( + repo=ServicesRepository(app.state.engine), + director_api=get_director_client(app), + product_name=product_name, + user_id=user_id, + limit=limit, + offset=offset, + filters=TypeAdapter(ServiceDBFilters | None).validate_python( + filters, from_attributes=True + ), + ) + + assert len(items) <= total_count # nosec + assert len(items) <= limit if limit is not None else True # nosec + + return cast( + PageRpcServiceSummary, + PageRpcServiceSummary.create( + items, + total=total_count, + limit=limit, + offset=offset, + ), + ) diff --git a/services/catalog/tests/unit/with_dbs/test_api_rpc.py b/services/catalog/tests/unit/with_dbs/test_api_rpc.py index 1cb346f3cd7c..5b43b728261b 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rpc.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rpc.py @@ -16,7 +16,10 @@ ServiceUpdateV2, ) from models_library.products import ProductName -from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE +from models_library.rest_pagination import ( + DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, +) from models_library.services_enums import ServiceType from models_library.services_history import ServiceRelease from models_library.services_types import ServiceKey, ServiceVersion @@ -143,7 +146,8 @@ async def test_rpc_list_services_paginated_with_filters( user_id=user_id, filters={"service_type": "computational"}, ) - assert page.meta.total == page.meta.count + # Fixed: Count might be capped by page limit + assert page.meta.count <= page.meta.total assert page.meta.total > 0 page = await catalog_rpc.list_services_paginated( @@ -821,7 +825,7 @@ async def test_rpc_get_service_ports_validation_error( user_id: UserID, app: FastAPI, ): - """Tests validation error handling for invalid service key format""" + """Tests validation error handling for list_all_services_summaries_paginated.""" assert app # Test with invalid service key format @@ -833,3 +837,207 @@ async def test_rpc_get_service_ports_validation_error( service_key="invalid-service-key-format", service_version="1.0.0", ) + + +async def test_rpc_list_all_services_summaries_paginated_with_no_services_returns_empty_page( + background_sync_task_mocked: None, + mocked_director_rest_api: MockRouter, + rpc_client: RabbitMQRPCClient, + user_id: UserID, + app: FastAPI, +): + """Tests that requesting summaries for non-existing services returns an empty page.""" + assert app + + page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, product_name="not_existing_returns_no_services", user_id=user_id + ) + assert page.data == [] + assert page.links.next is None + assert page.links.prev is None + assert page.meta.count == 0 + assert page.meta.total == 0 + + +async def test_rpc_list_all_services_summaries_paginated_with_filters( + background_sync_task_mocked: None, + mocked_director_rest_api: MockRouter, + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + user_id: UserID, + app: FastAPI, +): + """Tests that service summaries can be filtered by service type.""" + assert app + + # Get all computational services introduced by the background_sync_task_mocked + page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + filters={"service_type": "computational"}, + ) + # Fixed: Count might be capped by page limit + assert page.meta.count <= page.meta.total + assert page.meta.total > 0 + + # All items should be service summaries with the expected minimal fields + for item in page.data: + assert "key" in item.model_dump() + assert "name" in item.model_dump() + assert "version" in item.model_dump() + assert "description" in item.model_dump() + + # Filter for a service type that doesn't exist + page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + filters=ServiceListFilters(service_type=ServiceType.DYNAMIC), + ) + assert page.meta.total == 0 + + +async def test_rpc_list_all_services_summaries_paginated_with_pagination( + background_sync_task_mocked: None, + mocked_director_rest_api: MockRouter, + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + user_id: UserID, + app: FastAPI, + num_services: int, + num_versions_per_service: int, +): + """Tests pagination of service summaries.""" + assert app + + total_services = num_services * num_versions_per_service + + # Get first page with default page size + first_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + ) + + # Verify total count is correct + assert first_page.meta.total == total_services + + # Maximum items per page is constrained by DEFAULT_NUMBER_OF_ITEMS_PER_PAGE + assert len(first_page.data) <= DEFAULT_NUMBER_OF_ITEMS_PER_PAGE + + # Test with small page size + page_size = 5 + first_small_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + limit=page_size, + offset=0, + ) + assert len(first_small_page.data) == page_size + assert first_small_page.meta.total == total_services + assert first_small_page.links.next is not None + assert first_small_page.links.prev is None + + # Get next page and verify different content + next_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + limit=page_size, + offset=page_size, + ) + assert len(next_page.data) == page_size + assert next_page.meta.total == first_small_page.meta.total + + # Check that first and second page contain different items + first_page_keys = {(item.key, item.version) for item in first_small_page.data} + next_page_keys = {(item.key, item.version) for item in next_page.data} + assert not first_page_keys.intersection(next_page_keys) + + +async def test_rpc_compare_latest_vs_all_services_summaries( + background_sync_task_mocked: None, + mocked_director_rest_api: MockRouter, + rpc_client: RabbitMQRPCClient, + product_name: ProductName, + user_id: UserID, + app: FastAPI, + num_services: int, + num_versions_per_service: int, +): + """Compares results of list_services_paginated vs list_all_services_summaries_paginated.""" + assert app + + total_expected_services = num_services * num_versions_per_service + + # Get all latest services (should fit in one page) + latest_page = await catalog_rpc.list_services_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + ) + assert latest_page.meta.total == num_services + + # For all services (all versions), we might need multiple requests + # First page to get metadata + first_all_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + ) + assert first_all_page.meta.total == total_expected_services + + # Collect all items across multiple pages if needed + all_items = list(first_all_page.data) + offset = len(all_items) + + # Continue fetching pages until we have all items + while offset < total_expected_services: + next_page = await catalog_rpc.list_all_services_summaries_paginated( + rpc_client, + product_name=product_name, + user_id=user_id, + offset=offset, + ) + all_items.extend(next_page.data) + offset += len(next_page.data) + if not next_page.links.next: + break + + # Verify we got all items + assert len(all_items) == total_expected_services + + # Collect unique keys from both responses + latest_keys = {item.key for item in latest_page.data} + all_keys = {item.key for item in all_items} + + # All service keys in latest should be in all services + assert latest_keys.issubset(all_keys) + + # For each key in latest, there should be exactly num_versions_per_service entries in all + for key in latest_keys: + versions_in_all = [item.version for item in all_items if item.key == key] + assert len(versions_in_all) == num_versions_per_service + + # Get the latest version from latest_page + latest_version = next( + item.version for item in latest_page.data if item.key == key + ) + + # Verify this version exists in versions_in_all + assert latest_version in versions_in_all + + # Verify all items are ServiceSummary objects with just the essential fields + for item in all_items: + item_dict = item.model_dump() + assert "key" in item_dict + assert "version" in item_dict + assert "name" in item_dict + assert "description" in item_dict + assert "thumbnail" not in item_dict + assert "service_type" not in item_dict + assert "inputs" not in item_dict + assert "outputs" not in item_dict + assert "access_rights" not in item_dict From 3b1e3b0c3d39728253ab980cd18f76aa2c02fcad Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 20:44:56 +0200 Subject: [PATCH 10/20] =?UTF-8?q?=E2=9C=A8=20[Backend]=20Add=20async=20fun?= =?UTF-8?q?ction=20to=20list=20all=20service=20summaries=20with=20paginati?= =?UTF-8?q?on=20for=20improved=20performance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services_rpc/catalog.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py index a428ccb33bae..32a4b9ce17e0 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py @@ -5,6 +5,7 @@ LatestServiceGet, ServiceGetV2, ServiceListFilters, + ServiceSummary, ) from models_library.api_schemas_catalog.services_ports import ServicePortGet from models_library.products import ProductName @@ -89,6 +90,41 @@ async def list_release_history_latest_first( ) return page.data, meta + async def list_all_services_summaries( + self, + *, + pagination_offset: PageOffsetInt = 0, + pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + filters: ServiceListFilters | None = None, + ) -> tuple[list[ServiceSummary], PageMetaInfoLimitOffset]: + """Lists all services with pagination, including all versions of each service. + + Returns a lightweight summary view of services for better performance. + + Args: + pagination_offset: Number of items to skip + pagination_limit: Maximum number of items to return + filters: Optional filters to apply + + Returns: + Tuple containing list of service summaries and pagination metadata + """ + page = await catalog_rpc.list_all_services_summaries_paginated( + self._rpc_client, + product_name=self.product_name, + user_id=self.user_id, + offset=pagination_offset, + limit=pagination_limit, + filters=filters, + ) + meta = PageMetaInfoLimitOffset( + limit=page.meta.limit, + offset=page.meta.offset, + total=page.meta.total, + count=page.meta.count, + ) + return page.data, meta + @_exception_mapper( rpc_exception_map={ CatalogItemNotFoundError: ProgramOrSolverOrStudyNotFoundError, From 1f2e65ff9c3cbbb0a5758483d68a8186cc6decd3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 20:53:16 +0200 Subject: [PATCH 11/20] =?UTF-8?q?=E2=9C=A8=20[Backend]=20Add=20async=20fun?= =?UTF-8?q?ction=20for=20paginated=20listing=20of=20service=20summaries=20?= =?UTF-8?q?and=20refactor=20mock=20setup=20in=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_schemas_catalog/services.py | 37 ++++++++++- .../helpers/catalog_rpc_server.py | 65 +++++++++++++++++++ services/api-server/tests/unit/conftest.py | 53 ++++++--------- 3 files changed, 121 insertions(+), 34 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index dbe07757b9b2..c5360b9b811a 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -182,6 +182,41 @@ class ServiceSummary(CatalogOutputSchema): contact: LowerCaseEmailStr | None + service_type: Annotated[ServiceType, Field(alias="type")] + + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "examples": [ + { + "key": _EXAMPLE_SLEEPER["key"], + "version": _EXAMPLE_SLEEPER["version"], + "name": _EXAMPLE_SLEEPER["name"], + "description": _EXAMPLE_SLEEPER["description"], + "version_display": _EXAMPLE_SLEEPER["version_display"], + "contact": _EXAMPLE_SLEEPER["contact"], + "type": _EXAMPLE_SLEEPER["type"], + }, + { + "key": _EXAMPLE_FILEPICKER["key"], + "version": _EXAMPLE_FILEPICKER["version"], + "name": _EXAMPLE_FILEPICKER["name"], + "description": _EXAMPLE_FILEPICKER["description"], + "version_display": None, + "contact": _EXAMPLE_FILEPICKER["contact"], + "type": _EXAMPLE_FILEPICKER["type"], + }, + ] + } + ) + + model_config = ConfigDict( + extra="ignore", + populate_by_name=True, + json_schema_extra=_update_json_schema_extra, + ) + class _BaseServiceGetV2(ServiceSummary): # Model used in catalog's rpc and rest interfaces @@ -190,8 +225,6 @@ class _BaseServiceGetV2(ServiceSummary): description_ui: bool = False - service_type: Annotated[ServiceType, Field(alias="type")] - authors: Annotated[list[Author], Field(min_length=1)] owner: Annotated[ LowerCaseEmailStr | None, diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py index 0dc9145c2442..95443c6ba3f1 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py @@ -13,6 +13,7 @@ LatestServiceGet, ServiceGetV2, ServiceListFilters, + ServiceSummary, ServiceUpdateV2, ) from models_library.api_schemas_catalog.services_ports import ServicePortGet @@ -200,6 +201,62 @@ async def get_service_ports( ServicePortGet.model_json_schema()["examples"], ) + @validate_call(config={"arbitrary_types_allowed": True}) + async def list_all_services_summaries_paginated( + self, + rpc_client: RabbitMQRPCClient | MockType, + *, + product_name: ProductName, + user_id: UserID, + limit: PageLimitInt, + offset: NonNegativeInt, + filters: ServiceListFilters | None = None, + ): + assert rpc_client + assert product_name + assert user_id + + service_summaries = TypeAdapter(list[ServiceSummary]).validate_python( + ServiceSummary.model_json_schema()["examples"], + ) + if filters: + filtered_summaries = [] + for summary in service_summaries: + # Match service type if specified + if ( + filters.service_type + and summary.service_type != filters.service_type + ): + continue + + # Match service key pattern if specified + if filters.service_key_pattern and not fnmatch.fnmatch( + summary.key, filters.service_key_pattern + ): + continue + + # Match version display pattern if specified + if filters.version_display_pattern and ( + summary.version_display is None + or not fnmatch.fnmatch( + summary.version_display, filters.version_display_pattern + ) + ): + continue + + filtered_summaries.append(summary) + + service_summaries = filtered_summaries + + total_count = len(service_summaries) + + return PageRpc[ServiceSummary].create( + service_summaries[offset : offset + limit], + total=total_count, + limit=limit, + offset=offset, + ) + @dataclass class ZeroListingCatalogRpcSideEffects: @@ -216,3 +273,11 @@ async def list_my_service_history_latest_first(self, *args, **kwargs): limit=10, offset=0, ) + + async def list_all_services_summaries_paginated(self, *args, **kwargs): + return PageRpc[ServiceSummary].create( + [], + total=0, + limit=10, + offset=0, + ) diff --git a/services/api-server/tests/unit/conftest.py b/services/api-server/tests/unit/conftest.py index 7c3936d9bdb5..174bf1bd6013 100644 --- a/services/api-server/tests/unit/conftest.py +++ b/services/api-server/tests/unit/conftest.py @@ -509,38 +509,27 @@ def mocked_catalog_rpc_api( services as catalog_rpc, # keep import here ) - return { - "list_services_paginated": mocker.patch.object( - catalog_rpc, - "list_services_paginated", - autospec=True, - side_effect=catalog_rpc_side_effects.list_services_paginated, - ), - "get_service": mocker.patch.object( - catalog_rpc, - "get_service", - autospec=True, - side_effect=catalog_rpc_side_effects.get_service, - ), - "update_service": mocker.patch.object( - catalog_rpc, - "update_service", - autospec=True, - side_effect=catalog_rpc_side_effects.update_service, - ), - "list_my_service_history_latest_first": mocker.patch.object( - catalog_rpc, - "list_my_service_history_latest_first", - autospec=True, - side_effect=catalog_rpc_side_effects.list_my_service_history_latest_first, - ), - "get_service_ports": mocker.patch.object( - catalog_rpc, - "get_service_ports", - autospec=True, - side_effect=catalog_rpc_side_effects.get_service_ports, - ), - } + mocks = {} + + # Get all callable methods from the side effects class that are not built-ins + side_effect_methods = [ + method_name + for method_name in dir(catalog_rpc_side_effects) + if not method_name.startswith("_") + and callable(getattr(catalog_rpc_side_effects, method_name)) + ] + + # Create mocks for each method in catalog_rpc that has a corresponding side effect + for method_name in side_effect_methods: + if hasattr(catalog_rpc, method_name): + mocks[method_name] = mocker.patch.object( + catalog_rpc, + method_name, + autospec=True, + side_effect=getattr(catalog_rpc_side_effects, method_name), + ) + + return mocks # From 4c32b812442cd95bdb6af561268f77ef4ab90176 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 21:01:57 +0200 Subject: [PATCH 12/20] =?UTF-8?q?=E2=9C=A8=20[Backend]=20Add=20async=20met?= =?UTF-8?q?hod=20to=20list=20all=20solvers=20with=20pagination=20and=20fil?= =?UTF-8?q?tering,=20including=20all=20versions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../_service_solvers.py | 41 +++++++++++++++++++ .../models/schemas/solvers.py | 21 ++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 208ed961ff5a..28c767cbeea8 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -153,6 +153,47 @@ async def solver_release_history( for service in releases ], page_meta + async def list_all_solvers( + self, + *, + pagination_offset: NonNegativeInt, + pagination_limit: PositiveInt, + filter_by_solver_key_pattern: str | None = None, + filter_by_version_display_pattern: str | None = None, + ) -> tuple[list[Solver], PageMetaInfoLimitOffset]: + """Lists all solvers with pagination and filtering, including all versions. + + Unlike `latest_solvers` which only shows the latest version of each solver, + this method returns all versions of solvers that match the filters. + + Args: + pagination_offset: Pagination offset + pagination_limit: Pagination limit + filter_by_solver_key_pattern: Optional pattern to filter solvers by key e.g. "simcore/service/my_solver*" + filter_by_version_display_pattern: Optional pattern to filter by version display e.g. "1.0.*-beta" + + Returns: + A tuple with the list of filtered solvers and pagination metadata + """ + filters = ServiceListFilters(service_type=ServiceType.COMPUTATIONAL) + + # Add key_pattern filter for solver ID if provided + if filter_by_solver_key_pattern: + filters.service_key_pattern = filter_by_solver_key_pattern + + # Add version_display_pattern filter if provided + if filter_by_version_display_pattern: + filters.version_display_pattern = filter_by_version_display_pattern + + services, page_meta = await self.catalog_service.list_all_services_summaries( + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, + filters=filters, + ) + + solvers = [Solver.create_from_service(service) for service in services] + return solvers, page_meta + async def latest_solvers( self, *, diff --git a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py index 19228766f943..63a383c4f2b1 100644 --- a/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py +++ b/services/api-server/src/simcore_service_api_server/models/schemas/solvers.py @@ -1,6 +1,10 @@ from typing import Annotated, Any, Literal, Self, TypeAlias -from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2 +from models_library.api_schemas_catalog.services import ( + LatestServiceGet, + ServiceGetV2, + ServiceSummary, +) from models_library.basic_regex import PUBLIC_VARIABLE_NAME_RE from models_library.emails import LowerCaseEmailStr from models_library.services import ServiceMetaDataPublished @@ -72,14 +76,23 @@ def create_from_image(cls, image_meta: ServiceMetaDataPublished) -> Self: ) @classmethod - def create_from_service(cls, service: ServiceGetV2 | LatestServiceGet) -> Self: + def create_from_service( + cls, service: ServiceGetV2 | LatestServiceGet | ServiceSummary + ) -> Self: + # Common fields in all service types + maintainer = "" + if hasattr(service, "contact") and service.contact: + maintainer = service.contact + return cls( id=service.key, version=service.version, title=service.name, description=service.description, - maintainer=service.contact or "UNDEFINED", - version_display=service.version_display, + maintainer=maintainer or "UNDEFINED", + version_display=( + service.version_display if hasattr(service, "version_display") else None + ), url=None, ) From 0d3ddbcdc6bd50b381087c60440db93ccaaf2e3c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 21:05:19 +0200 Subject: [PATCH 13/20] =?UTF-8?q?=E2=9C=A8=20[Backend]=20Update=20solvers?= =?UTF-8?q?=20endpoint=20to=20list=20all=20available=20solvers=20with=20pa?= =?UTF-8?q?gination=20and=20improve=20method=20naming=20for=20clarity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/simcore_service_api_server/api/routes/solvers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 1d6dcb5775a9..3f4942499d24 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -84,20 +84,20 @@ async def list_solvers( "/page", response_model=Page[Solver], description=create_route_description( - base="Lists the latest version of all available solvers (paginated)", + base="Lists all available solvers (paginated)", changelog=[ FMSG_CHANGELOG_NEW_IN_VERSION.format("0.9-rc1"), ], ), include_in_schema=False, # TO BE RELEASED in 0.9 ) -async def list_solvers_paginated( +async def list_all_solvers_paginated( page_params: Annotated[PaginationParams, Depends()], solver_service: Annotated[SolverService, Depends(get_solver_service)], filters: Annotated[SolversListFilters, Depends(get_solvers_filters)], url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], ): - solvers, page_meta = await solver_service.latest_solvers( + solvers, page_meta = await solver_service.list_all_solvers( pagination_offset=page_params.offset, pagination_limit=page_params.limit, filter_by_solver_key_pattern=filters.solver_id, From c0fab4cf6d3d0386c59899b27bd20b49d44ebd17 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 21 May 2025 21:18:02 +0200 Subject: [PATCH 14/20] fix --- .../src/models_library/api_schemas_catalog/services.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index c5360b9b811a..730784e40a62 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -182,8 +182,6 @@ class ServiceSummary(CatalogOutputSchema): contact: LowerCaseEmailStr | None - service_type: Annotated[ServiceType, Field(alias="type")] - @staticmethod def _update_json_schema_extra(schema: JsonDict) -> None: schema.update( @@ -196,7 +194,6 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "description": _EXAMPLE_SLEEPER["description"], "version_display": _EXAMPLE_SLEEPER["version_display"], "contact": _EXAMPLE_SLEEPER["contact"], - "type": _EXAMPLE_SLEEPER["type"], }, { "key": _EXAMPLE_FILEPICKER["key"], @@ -205,7 +202,6 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "description": _EXAMPLE_FILEPICKER["description"], "version_display": None, "contact": _EXAMPLE_FILEPICKER["contact"], - "type": _EXAMPLE_FILEPICKER["type"], }, ] } @@ -220,6 +216,9 @@ def _update_json_schema_extra(schema: JsonDict) -> None: class _BaseServiceGetV2(ServiceSummary): # Model used in catalog's rpc and rest interfaces + + service_type: Annotated[ServiceType, Field(alias="type")] + thumbnail: HttpUrl | None = None icon: HttpUrl | None = None From 40dd8fac5f1b67930e58d4e10448a62457bd8439 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 22 May 2025 09:56:52 +0200 Subject: [PATCH 15/20] fixes test --- .../src/models_library/api_schemas_catalog/services.py | 1 + .../src/pytest_simcore/helpers/catalog_rpc_server.py | 6 +++++- .../tests/unit/api_solvers/test_api_routers_solvers.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index 730784e40a62..020e25b86d21 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -252,6 +252,7 @@ class _BaseServiceGetV2(ServiceSummary): extra="forbid", populate_by_name=True, alias_generator=snake_to_camel, + json_schema_extra={"example": _EXAMPLE_SLEEPER}, ) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py index 95443c6ba3f1..58c6e178b598 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py @@ -225,7 +225,11 @@ async def list_all_services_summaries_paginated( # Match service type if specified if ( filters.service_type - and summary.service_type != filters.service_type + and { + ServiceType.COMPUTATIONAL: "/comp/", + ServiceType.DYNAMIC: "/dynamic/", + }[filters.service_type] + in summary.key ): continue diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py index b6be360f26b4..83fa6a0ce523 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py @@ -6,12 +6,12 @@ import httpx +from fastapi import status from pydantic import TypeAdapter from pytest_mock import MockType from simcore_service_api_server._meta import API_VTAG from simcore_service_api_server.models.pagination import OnePage from simcore_service_api_server.models.schemas.solvers import Solver, SolverPort -from starlette import status async def test_list_all_solvers( From 3b457eed1d9dba7d4d4bff50e39e3bd63e640925 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 22 May 2025 10:14:30 +0200 Subject: [PATCH 16/20] doc --- .../api-server/docs/api-server.drawio.svg | 136 ++++++++++++++---- 1 file changed, 105 insertions(+), 31 deletions(-) diff --git a/services/api-server/docs/api-server.drawio.svg b/services/api-server/docs/api-server.drawio.svg index 3f6efeb32725..98f7dcfdfc6e 100644 --- a/services/api-server/docs/api-server.drawio.svg +++ b/services/api-server/docs/api-server.drawio.svg @@ -1,6 +1,6 @@ - + - + @@ -285,28 +285,26 @@
- postgres -
- asyncpg + sa[asyngpg]
- postgres... + sa[asyngpg] - + -
+
CONTROLLER @@ -314,20 +312,20 @@
- + CONTROLLER - + -
+
SERVICE @@ -335,20 +333,20 @@
- + SERVICE - + -
+
REPOSITORY @@ -356,20 +354,20 @@
- + REPOSITORY - + -
+
CLIENTS @@ -377,20 +375,20 @@
- + CLIENTS - + -
+
rest @@ -398,20 +396,24 @@
- + rest - + + + + + -
+
rpc @@ -419,12 +421,83 @@
- + rpc + + + + + + + + + + + +
+
+
+ services +
+
+
+ + + services + + + + + + + + + + + + + + + +
+
+
+ catalog_srv +
+
+
+ + + catalog_srv + + + + + + + + + + + +
+
+
+ sa[asyncpg] +
+
+
+ + + sa[asyncpg] + + + + @@ -565,15 +638,13 @@
- postgres -
- asyncpg + sa[asyncg]
- postgres... + sa[asyncg] @@ -796,8 +867,8 @@ - - + + @@ -884,7 +955,7 @@ - + @@ -908,6 +979,9 @@ + + + From 7456e8d8af2790d65b5f5314628d38d115d85a77 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 22 May 2025 10:19:26 +0200 Subject: [PATCH 17/20] cleanup --- .../api_schemas_catalog/services.py | 1 + .../api/v0/openapi.yaml | 68 +++++++++---------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index 020e25b86d21..efe82d6e8c76 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -210,6 +210,7 @@ def _update_json_schema_extra(schema: JsonDict) -> None: model_config = ConfigDict( extra="ignore", populate_by_name=True, + alias_generator=snake_to_camel, json_schema_extra=_update_json_schema_extra, ) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index e60f7169086a..29e6a9c423d8 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -8616,36 +8616,36 @@ components: name: type: string title: Name - thumbnail: - anyOf: - - type: string - - type: 'null' - title: Thumbnail - icon: - anyOf: - - type: string - - type: 'null' - title: Icon description: type: string title: Description - descriptionUi: - type: boolean - title: Descriptionui - default: false versionDisplay: anyOf: - type: string - type: 'null' title: Versiondisplay - type: - $ref: '#/components/schemas/ServiceType' contact: anyOf: - type: string format: email - type: 'null' title: Contact + type: + $ref: '#/components/schemas/ServiceType' + thumbnail: + anyOf: + - type: string + - type: 'null' + title: Thumbnail + icon: + anyOf: + - type: string + - type: 'null' + title: Icon + descriptionUi: + type: boolean + title: Descriptionui + default: false authors: items: $ref: '#/components/schemas/Author' @@ -8702,8 +8702,8 @@ components: - version - name - description - - type - contact + - type - authors - owner - inputs @@ -8835,36 +8835,36 @@ components: name: type: string title: Name - thumbnail: - anyOf: - - type: string - - type: 'null' - title: Thumbnail - icon: - anyOf: - - type: string - - type: 'null' - title: Icon description: type: string title: Description - descriptionUi: - type: boolean - title: Descriptionui - default: false versionDisplay: anyOf: - type: string - type: 'null' title: Versiondisplay - type: - $ref: '#/components/schemas/ServiceType' contact: anyOf: - type: string format: email - type: 'null' title: Contact + type: + $ref: '#/components/schemas/ServiceType' + thumbnail: + anyOf: + - type: string + - type: 'null' + title: Thumbnail + icon: + anyOf: + - type: string + - type: 'null' + title: Icon + descriptionUi: + type: boolean + title: Descriptionui + default: false authors: items: $ref: '#/components/schemas/Author' @@ -8928,8 +8928,8 @@ components: - version - name - description - - type - contact + - type - authors - owner - inputs From 76b7d23423bc00164abec83b293f9c2af52d1e22 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 22 May 2025 14:24:34 +0200 Subject: [PATCH 18/20] @GitHK review: doc --- .../src/models_library/api_schemas_catalog/services.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index efe82d6e8c76..5b66f46351f8 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -216,8 +216,6 @@ def _update_json_schema_extra(schema: JsonDict) -> None: class _BaseServiceGetV2(ServiceSummary): - # Model used in catalog's rpc and rest interfaces - service_type: Annotated[ServiceType, Field(alias="type")] thumbnail: HttpUrl | None = None @@ -286,7 +284,6 @@ def _update_json_schema_extra(schema: JsonDict) -> None: class ServiceGetV2(_BaseServiceGetV2): - # Model used in catalog's rpc and rest interfaces history: Annotated[ list[ServiceRelease], Field( From 8f4d35668eed14fbd78a8817b80bc48e47eb6726 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 22 May 2025 14:28:31 +0200 Subject: [PATCH 19/20] @bisgaard-itis review: wrong condition --- .../src/pytest_simcore/helpers/catalog_rpc_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py index 58c6e178b598..0e06f7d261ed 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py @@ -229,7 +229,7 @@ async def list_all_services_summaries_paginated( ServiceType.COMPUTATIONAL: "/comp/", ServiceType.DYNAMIC: "/dynamic/", }[filters.service_type] - in summary.key + not in summary.key ): continue From 035f3dbef37eef7b8db0a228c1e6447a9fbe566c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 22 May 2025 14:30:11 +0200 Subject: [PATCH 20/20] @bisgaard-itis review: getattr --- .../src/simcore_service_catalog/service/catalog_services.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/catalog/src/simcore_service_catalog/service/catalog_services.py b/services/catalog/src/simcore_service_catalog/service/catalog_services.py index ea7f1b6282b7..d1377bb4db6c 100644 --- a/services/catalog/src/simcore_service_catalog/service/catalog_services.py +++ b/services/catalog/src/simcore_service_catalog/service/catalog_services.py @@ -70,9 +70,7 @@ def _aggregate( "contact": service_manifest.contact, "authors": service_manifest.authors, # Check if owner attribute is available in the service_db object - "owner": ( - service_db.owner_email if hasattr(service_db, "owner_email") else None - ), + "owner": getattr(service_db, "owner_email", None), "inputs": service_manifest.inputs or {}, "outputs": service_manifest.outputs or {}, "boot_options": service_manifest.boot_options,