fix: remove workbench references from catalog

giancarloromeo · giancarloromeo · commit 20cef279fcfe · 2025-07-25T21:28:00.000+02:00
diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py b/packages/pytest-simcore/src/pytest_simcore/helpers/faker_factories.py
@@ -25,6 +25,15 @@
 DEFAULT_FAKER: Final = Faker()
 
 
+def random_service_key(fake: Faker = DEFAULT_FAKER, *, name: str | None = None) -> str:
+    """Generates a random service key"""
+    return f"simcore/services/{fake.random_element(['dynamic', 'computational'])}/{name or fake.name()}"
+
+
+def random_service_version(fake: Faker = DEFAULT_FAKER) -> str:
+    return ".".join([str(fake.pyint()) for _ in range(3)])
+
+
 def random_icon_url(fake: Faker):
     return fake.image_url(width=16, height=16)
 
@@ -186,6 +195,26 @@ def random_project(fake: Faker = DEFAULT_FAKER, **overrides) -> dict[str, Any]:
     return data
 
 
+def random_project_node(fake: Faker = DEFAULT_FAKER, **overrides) -> dict[str, Any]:
+    """Generates random fake data project nodes DATABASE table"""
+    from simcore_postgres_database.models.projects_nodes import projects_nodes
+
+    _name = fake.name()
+
+    data = {
+        "node_id": fake.uuid4(),
+        "project_uuid": fake.uuid4(),
+        "key": random_service_key(fake, name=_name),
+        "version": random_service_version(fake),
+        "label": _name,
+    }
+
+    assert set(data.keys()).issubset({c.name for c in projects_nodes.columns})
+
+    data.update(overrides)
+    return data
+
+
 def random_group(fake: Faker = DEFAULT_FAKER, **overrides) -> dict[str, Any]:
     from simcore_postgres_database.models.groups import groups
     from simcore_postgres_database.webserver_models import GroupType
@@ -481,7 +510,7 @@ def random_service_meta_data(
 ) -> dict[str, Any]:
     from simcore_postgres_database.models.services import services_meta_data
 
-    _version = ".".join([str(fake.pyint()) for _ in range(3)])
+    _version = random_service_version(fake)
     _name = fake.name()
 
     data: dict[str, Any] = {
diff --git a/services/catalog/src/simcore_service_catalog/repository/projects.py b/services/catalog/src/simcore_service_catalog/repository/projects.py
@@ -4,37 +4,53 @@
 from models_library.services import ServiceKeyVersion
 from pydantic import ValidationError
 from simcore_postgres_database.models.projects import ProjectType, projects
+from simcore_postgres_database.models.projects_nodes import projects_nodes
 
 from ._base import BaseRepository
 
 _logger = logging.getLogger(__name__)
 
 
+_IGNORED_SERVICE_KEYS: set[str] = {
+    # NOTE: frontend only nodes
+    "simcore/services/frontend/file-picker",
+    "simcore/services/frontend/nodes-group",
+}
+
+
 class ProjectsRepository(BaseRepository):
     async def list_services_from_published_templates(self) -> list[ServiceKeyVersion]:
-        list_of_published_services: list[ServiceKeyVersion] = []
         async with self.db_engine.connect() as conn:
-            async for row in await conn.stream(
-                sa.select(projects).where(
-                    (projects.c.type == ProjectType.TEMPLATE)
-                    & (projects.c.published.is_(True))
+            query = (
+                sa.select(projects_nodes.c.key, projects_nodes.c.version)
+                .distinct()
+                .select_from(
+                    projects_nodes.join(
+                        projects, projects_nodes.c.project_uuid == projects.c.uuid
+                    )
+                )
+                .where(
+                    sa.and_(
+                        projects.c.type == ProjectType.TEMPLATE,
+                        projects.c.published.is_(True),
+                        projects_nodes.c.key.notin_(_IGNORED_SERVICE_KEYS),
+                    )
                 )
-            ):
-                project_workbench = row.workbench
-                for node in project_workbench:
-                    service = project_workbench[node]
-                    try:
-                        if (
-                            "file-picker" in service["key"]
-                            or "nodes-group" in service["key"]
-                        ):
-                            # these 2 are not going to pass the validation tests, they are frontend only nodes.
-                            continue
-                        list_of_published_services.append(ServiceKeyVersion(**service))
-                    except ValidationError:
-                        _logger.warning(
-                            "service %s could not be validated", service, exc_info=True
-                        )
-                        continue
-
-        return list_of_published_services
+            )
+
+            services = []
+            async for row in await conn.stream(query):
+                try:
+                    service = ServiceKeyVersion.model_validate(
+                        row, from_attributes=True
+                    )
+                    services.append(service)
+                except ValidationError:
+                    _logger.warning(
+                        "service with key=%s and version=%s could not be validated",
+                        row.key,
+                        row.version,
+                        exc_info=True,
+                    )
+
+            return services
diff --git a/services/catalog/tests/unit/with_dbs/test_repositories.py b/services/catalog/tests/unit/with_dbs/test_repositories.py
@@ -23,9 +23,10 @@
 from packaging import version
 from pydantic import EmailStr, HttpUrl, TypeAdapter
 from pytest_simcore.helpers.catalog_services import CreateFakeServiceDataCallable
-from pytest_simcore.helpers.faker_factories import random_project
+from pytest_simcore.helpers.faker_factories import random_project, random_project_node
 from pytest_simcore.helpers.postgres_tools import insert_and_get_row_lifespan
 from simcore_postgres_database.models.projects import ProjectType, projects
+from simcore_postgres_database.models.projects_nodes import projects_nodes
 from simcore_service_catalog.models.services_db import (
     ServiceAccessRightsDB,
     ServiceDBFilters,
@@ -812,21 +813,41 @@ async def test_list_services_from_published_templates(
                     type=ProjectType.TEMPLATE,
                     published=True,
                     prj_owner=user["id"],
-                    workbench={
-                        "node-1": {
-                            "key": "simcore/services/dynamic/jupyterlab",
-                            "version": "1.0.0",
-                        },
-                        "node-2": {
-                            "key": "simcore/services/frontend/file-picker",
-                            "version": "1.0.0",
-                        },
-                    },
                 ),
                 pk_col=projects.c.uuid,
                 pk_value="template-1",
             )
         )
+        await stack.enter_async_context(
+            insert_and_get_row_lifespan(
+                sqlalchemy_async_engine,
+                table=projects_nodes,
+                values=random_project_node(
+                    node_id="node-1.1",
+                    project_uuid="template-1",
+                    key="simcore/services/dynamic/jupyterlab",
+                    version="1.0.0",
+                    label="jupyterlab",
+                ),
+                pk_col=projects_nodes.c.node_id,
+                pk_value="node-1.1",
+            )
+        )
+        await stack.enter_async_context(
+            insert_and_get_row_lifespan(
+                sqlalchemy_async_engine,
+                table=projects_nodes,
+                values=random_project_node(
+                    node_id="node-1.2",
+                    project_uuid="template-1",
+                    key="simcore/services/frontend/file-picker",
+                    version="1.0.0",
+                    label="file-picker",
+                ),
+                pk_col=projects_nodes.c.node_id,
+                pk_value="node-1.2",
+            )
+        )
         await stack.enter_async_context(
             insert_and_get_row_lifespan(
                 sqlalchemy_async_engine,
@@ -836,17 +857,26 @@ async def test_list_services_from_published_templates(
                     type=ProjectType.TEMPLATE,
                     published=False,
                     prj_owner=user["id"],
-                    workbench={
-                        "node-1": {
-                            "key": "simcore/services/dynamic/some-service",
-                            "version": "2.0.0",
-                        },
-                    },
                 ),
                 pk_col=projects.c.uuid,
                 pk_value="template-2",
             )
         )
+        await stack.enter_async_context(
+            insert_and_get_row_lifespan(
+                sqlalchemy_async_engine,
+                table=projects_nodes,
+                values=random_project_node(
+                    node_id="node-2.1",
+                    project_uuid="template-2",
+                    key="simcore/services/dynamic/some-service",
+                    version="2.0.0",
+                    label="some-service",
+                ),
+                pk_col=projects_nodes.c.node_id,
+                pk_value="node-2.1",
+            )
+        )
 
         # Act: Call the method
         services = await projects_repo.list_services_from_published_templates()
@@ -874,21 +904,41 @@ async def test_list_services_from_published_templates_with_invalid_service(
                     type=ProjectType.TEMPLATE,
                     published=True,
                     prj_owner=user["id"],
-                    workbench={
-                        "node-1": {
-                            "key": "simcore/services/frontend/file-picker",
-                            "version": "1.0.0",
-                        },
-                        "node-2": {
-                            "key": "simcore/services/dynamic/invalid-service",
-                            "version": "invalid",
-                        },
-                    },
                 ),
                 pk_col=projects.c.uuid,
                 pk_value="template-1",
             )
         )
+        await stack.enter_async_context(
+            insert_and_get_row_lifespan(
+                sqlalchemy_async_engine,
+                table=projects_nodes,
+                values=random_project_node(
+                    node_id="node-1.1",
+                    project_uuid="template-1",
+                    key="simcore/services/frontend/file-picker",
+                    version="1.0.0",
+                    label="file-picker",
+                ),
+                pk_col=projects_nodes.c.node_id,
+                pk_value="node-1.1",
+            )
+        )
+        await stack.enter_async_context(
+            insert_and_get_row_lifespan(
+                sqlalchemy_async_engine,
+                table=projects_nodes,
+                values=random_project_node(
+                    node_id="node-1.2",
+                    project_uuid="template-1",
+                    key="simcore/services/dynamic/invalid-service",
+                    version="invalid",  # NOTE: invalid version
+                    label="invalid-service",
+                ),
+                pk_col=projects_nodes.c.node_id,
+                pk_value="node-1.2",
+            )
+        )
 
         # Act: Call the method and capture logs
         with caplog.at_level(logging.WARNING):
@@ -897,7 +947,7 @@ async def test_list_services_from_published_templates_with_invalid_service(
         # Assert: Validate the results
         assert len(services) == 0  # No valid services should be returned
         assert (
-            "service {'key': 'simcore/services/dynamic/invalid-service', 'version': 'invalid'} could not be validated"
+            "service with key=simcore/services/dynamic/invalid-service and version=invalid could not be validated"
             in caplog.text
         )
 
