🎨 Improve listing task manager (add filter + and project name, project custom metadata fields) (#7661)

matusdrobuliak66 · web-flow · commit db7161627a50 · 2025-05-13T13:57:19.000+02:00
diff --git a/packages/models-library/src/models_library/api_schemas_webserver/computations.py b/packages/models-library/src/models_library/api_schemas_webserver/computations.py
@@ -81,7 +81,11 @@ class ComputationStarted(OutputSchemaWithoutCamelCase):
 class ComputationRunListQueryParams(
     PageQueryParameters,
     ComputationRunListOrderParams,  # type: ignore[misc, valid-type]
-): ...
+):
+    filter_only_running: bool = Field(
+        default=False,
+        description="If true, only running computations are returned",
+    )
 
 
 class ComputationRunRestGet(OutputSchema):
@@ -92,6 +96,8 @@ class ComputationRunRestGet(OutputSchema):
     submitted_at: datetime
     started_at: datetime | None
     ended_at: datetime | None
+    root_project_name: str
+    project_custom_metadata: dict[str, Any]
 
 
 ### Computation Task
diff --git a/packages/models-library/src/models_library/computations.py b/packages/models-library/src/models_library/computations.py
@@ -22,3 +22,17 @@ class ComputationTaskWithAttributes(BaseModel):
     # Attributes added by the webserver
     node_name: str
     osparc_credits: Decimal | None
+
+
+class ComputationRunWithAttributes(BaseModel):
+    project_uuid: ProjectID
+    iteration: int
+    state: RunningState
+    info: dict[str, Any]
+    submitted_at: datetime
+    started_at: datetime | None
+    ended_at: datetime | None
+
+    # Attributes added by the webserver
+    root_project_name: str
+    project_custom_metadata: dict[str, Any]
diff --git a/packages/models-library/src/models_library/projects_state.py b/packages/models-library/src/models_library/projects_state.py
@@ -35,14 +35,18 @@ class RunningState(str, Enum):
     ABORTED = "ABORTED"
     WAITING_FOR_CLUSTER = "WAITING_FOR_CLUSTER"
 
-    def is_running(self) -> bool:
-        return self in (
+    @staticmethod
+    def list_running_states() -> list["RunningState"]:
+        return [
             RunningState.PUBLISHED,
             RunningState.PENDING,
             RunningState.WAITING_FOR_RESOURCES,
             RunningState.STARTED,
             RunningState.WAITING_FOR_CLUSTER,
-        )
+        ]
+
+    def is_running(self) -> bool:
+        return self in self.list_running_states()
 
 
 @unique
diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py
@@ -33,6 +33,8 @@ async def list_computations_latest_iteration_page(
     *,
     product_name: ProductName,
     user_id: UserID,
+    # filters
+    filter_only_running: bool = False,
     # pagination
     offset: int = 0,
     limit: int = 20,
@@ -46,6 +48,7 @@ async def list_computations_latest_iteration_page(
         ),
         product_name=product_name,
         user_id=user_id,
+        filter_only_running=filter_only_running,
         offset=offset,
         limit=limit,
         order_by=order_by,
diff --git a/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py b/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py
@@ -28,6 +28,8 @@ async def list_computations_latest_iteration_page(
     *,
     product_name: ProductName,
     user_id: UserID,
+    # filters
+    filter_only_running: bool = False,
     # pagination
     offset: int = 0,
     limit: int = 20,
@@ -39,6 +41,7 @@ async def list_computations_latest_iteration_page(
         await comp_runs_repo.list_for_user__only_latest_iterations(
             product_name=product_name,
             user_id=user_id,
+            filter_only_running=filter_only_running,
             offset=offset,
             limit=limit,
             order_by=order_by,
diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py
@@ -195,6 +195,8 @@ async def list_for_user__only_latest_iterations(
         *,
         product_name: str,
         user_id: UserID,
+        # filters
+        filter_only_running: bool,
         # pagination
         offset: int,
         limit: int,
@@ -229,6 +231,16 @@ async def list_for_user__only_latest_iterations(
                 & (
                     comp_runs.c.metadata["product_name"].astext == product_name
                 )  # <-- NOTE: We might create a separate column for this for fast retrieval
+                & (
+                    comp_runs.c.result.in_(
+                        [
+                            RUNNING_STATE_TO_DB[item]
+                            for item in RunningState.list_running_states()
+                        ]
+                    )
+                )
+                if filter_only_running
+                else True
             )
             .group_by(comp_runs.c.project_uuid)
             .subquery("latest_runs")
diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml
@@ -2567,6 +2567,13 @@ paths:
           type: integer
           default: 0
           title: Offset
+      - name: filter_only_running
+        in: query
+        required: false
+        schema:
+          type: boolean
+          default: false
+          title: Filter Only Running
       responses:
         '200':
           description: Successful Response
@@ -9102,6 +9109,12 @@ components:
             format: date-time
           - type: 'null'
           title: Endedat
+        rootProjectName:
+          type: string
+          title: Rootprojectname
+        projectCustomMetadata:
+          type: object
+          title: Projectcustommetadata
       type: object
       required:
       - projectUuid
@@ -9111,6 +9124,8 @@ components:
       - submittedAt
       - startedAt
       - endedAt
+      - rootProjectName
+      - projectCustomMetadata
       title: ComputationRunRestGet
     ComputationStart:
       properties:
diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py
@@ -1,10 +1,10 @@
 from decimal import Decimal
 
 from aiohttp import web
-from models_library.api_schemas_directorv2.comp_runs import (
-    ComputationRunRpcGetPage,
+from models_library.computations import (
+    ComputationRunWithAttributes,
+    ComputationTaskWithAttributes,
 )
-from models_library.computations import ComputationTaskWithAttributes
 from models_library.products import ProductName
 from models_library.projects import ProjectID
 from models_library.rest_ordering import OrderBy
@@ -17,36 +17,86 @@
 from servicelib.utils import limited_gather
 
 from ..products.products_service import is_product_billable
-from ..projects.api import check_user_project_permission, get_project_dict_legacy
+from ..projects.api import (
+    batch_get_project_name,
+    check_user_project_permission,
+    get_project_dict_legacy,
+)
+from ..projects.projects_metadata_service import (
+    get_project_custom_metadata_or_empty_dict,
+)
 from ..rabbitmq import get_rabbitmq_rpc_client
 
 
 async def list_computations_latest_iteration(
     app: web.Application,
     product_name: ProductName,
     user_id: UserID,
+    # filters
+    filter_only_running: bool,  # noqa: FBT001
     # pagination
     offset: int,
     limit: NonNegativeInt,
     # ordering
     order_by: OrderBy,
-) -> ComputationRunRpcGetPage:
+) -> tuple[int, list[ComputationRunWithAttributes]]:
     """Returns the list of computations (only latest iterations)"""
     rpc_client = get_rabbitmq_rpc_client(app)
     _runs_get = await computations.list_computations_latest_iteration_page(
         rpc_client,
         product_name=product_name,
         user_id=user_id,
+        filter_only_running=filter_only_running,
         offset=offset,
         limit=limit,
         order_by=order_by,
     )
 
-    # NOTE: MD: Get project metadata
-    # NOTE: MD: Get Root project name
-    assert _runs_get  # nosec
+    # Get projects metadata (NOTE: MD: can be improved with a single batch call)
+    _projects_metadata = await limited_gather(
+        *[
+            get_project_custom_metadata_or_empty_dict(
+                app, project_uuid=item.project_uuid
+            )
+            for item in _runs_get.items
+        ],
+        limit=20,
+    )
+
+    # Get Root project names
+    _projects_root_uuids: list[ProjectID] = []
+    for item in _runs_get.items:
+        if (
+            prj_root_id := item.info.get("project_metadata", {}).get(
+                "root_parent_project_id", None
+            )
+        ) is not None:
+            _projects_root_uuids.append(ProjectID(prj_root_id))
+        else:
+            _projects_root_uuids.append(item.project_uuid)
+
+    _projects_root_names = await batch_get_project_name(
+        app, projects_uuids=_projects_root_uuids
+    )
+
+    _computational_runs_output = [
+        ComputationRunWithAttributes(
+            project_uuid=item.project_uuid,
+            iteration=item.iteration,
+            state=item.state,
+            info=item.info,
+            submitted_at=item.submitted_at,
+            started_at=item.started_at,
+            ended_at=item.ended_at,
+            root_project_name=project_name,
+            project_custom_metadata=project_metadata,
+        )
+        for item, project_metadata, project_name in zip(
+            _runs_get.items, _projects_metadata, _projects_root_names, strict=True
+        )
+    ]
 
-    return _runs_get
+    return _runs_get.total, _computational_runs_output
 
 
 async def list_computations_latest_iteration_tasks(
diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py b/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py
@@ -53,10 +53,12 @@ async def list_computations_latest_iteration(request: web.Request) -> web.Respon
         ComputationRunListQueryParams, request
     )
 
-    _get = await _computations_service.list_computations_latest_iteration(
+    total, items = await _computations_service.list_computations_latest_iteration(
         request.app,
         product_name=req_ctx.product_name,
         user_id=req_ctx.user_id,
+        # filters
+        filter_only_running=query_params.filter_only_running,
         # pagination
         offset=query_params.offset,
         limit=query_params.limit,
@@ -67,10 +69,10 @@ async def list_computations_latest_iteration(request: web.Request) -> web.Respon
     page = Page[ComputationRunRestGet].model_validate(
         paginate_data(
             chunk=[
-                ComputationRunRestGet.model_validate(task, from_attributes=True)
-                for task in _get.items
+                ComputationRunRestGet.model_validate(run, from_attributes=True)
+                for run in items
             ],
-            total=_get.total,
+            total=total,
             limit=query_params.limit,
             offset=query_params.offset,
             request_url=request.url,
diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/metadata_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/metadata_rest.py
@@ -47,7 +47,7 @@ async def get_project_metadata(request: web.Request) -> web.Response:
     req_ctx = RequestContext.model_validate(request)
     path_params = parse_request_path_parameters_as(ProjectPathParams, request)
 
-    custom_metadata = await _metadata_service.get_project_custom_metadata(
+    custom_metadata = await _metadata_service.get_project_custom_metadata_for_user(
         request.app, user_id=req_ctx.user_id, project_uuid=path_params.project_id
     )
 
diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py
@@ -11,20 +11,36 @@
 from ..db.plugin import get_database_engine
 from . import _metadata_repository
 from ._access_rights_service import validate_project_ownership
+from .exceptions import ProjectNotFoundError
 
 _logger = logging.getLogger(__name__)
 
 
-async def get_project_custom_metadata(
+async def get_project_custom_metadata_for_user(
     app: web.Application, user_id: UserID, project_uuid: ProjectID
 ) -> MetadataDict:
+    """raises: ProjectNotFoundError"""
     await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid)
 
     return await _metadata_repository.get_project_custom_metadata(
         engine=get_database_engine(app), project_uuid=project_uuid
     )
 
 
+async def get_project_custom_metadata_or_empty_dict(
+    app: web.Application, project_uuid: ProjectID
+) -> MetadataDict:
+    try:
+        output = await _metadata_repository.get_project_custom_metadata(
+            engine=get_database_engine(app), project_uuid=project_uuid
+        )
+    except ProjectNotFoundError:
+        # This is a valid case when the project is not found
+        # but we still want to return an empty dict
+        output = {}
+    return output
+
+
 async def set_project_custom_metadata(
     app: web.Application,
     user_id: UserID,
diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py
@@ -115,6 +115,42 @@ async def get_project(
         return ProjectDBGet.model_validate(row)
 
 
+async def batch_get_project_name(
+    app: web.Application,
+    connection: AsyncConnection | None = None,
+    *,
+    projects_uuids: list[ProjectID],
+) -> list[str | None]:
+    if not projects_uuids:
+        return []
+
+    projects_uuids_str = [f"{uuid}" for uuid in projects_uuids]
+
+    query = (
+        sql.select(
+            projects.c.uuid,
+            projects.c.name,
+        )
+        .select_from(projects)
+        .where(projects.c.uuid.in_(projects_uuids_str))
+    ).order_by(
+        # Preserves the order of projects_uuids
+        # SEE https://docs.sqlalchemy.org/en/20/core/sqlelement.html#sqlalchemy.sql.expression.case
+        sql.case(
+            {
+                project_uuid: index
+                for index, project_uuid in enumerate(projects_uuids_str)
+            },
+            value=projects.c.uuid,
+        )
+    )
+    async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn:
+        result = await conn.stream(query)
+        rows = {row.uuid: row.name async for row in result}
+
+    return [rows.get(project_uuid) for project_uuid in projects_uuids_str]
+
+
 def _select_trashed_by_primary_gid_query() -> sql.Select:
     return sql.select(
         projects.c.uuid,
@@ -159,7 +195,7 @@ async def batch_get_trashed_by_primary_gid(
             projects.c.uuid.in_(projects_uuids_str)
         )
     ).order_by(
-        # Preserves the order of folders_ids
+        # Preserves the order of project_uuids
         # SEE https://docs.sqlalchemy.org/en/20/core/sqlelement.html#sqlalchemy.sql.expression.case
         sql.case(
             {
diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_service.py b/services/web/server/src/simcore_service_webserver/projects/_projects_service.py
diff --git a/services/web/server/src/simcore_service_webserver/projects/api.py b/services/web/server/src/simcore_service_webserver/projects/api.py
diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_metadata_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_metadata_service.py
diff --git a/services/web/server/tests/unit/with_dbs/01/test_director_v2_handlers.py b/services/web/server/tests/unit/with_dbs/01/test_director_v2_handlers.py

Original file line number	Diff line number	Diff line change
`@@ -47,7 +47,7 @@ async def get_project_metadata(request: web.Request) -> web.Response:`
`47`	`47`	`req_ctx = RequestContext.model_validate(request)`
`48`	`48`	`path_params = parse_request_path_parameters_as(ProjectPathParams, request)`
`49`	`49`
`50`		`- custom_metadata = await _metadata_service.get_project_custom_metadata(`
	`50`	`+ custom_metadata = await _metadata_service.get_project_custom_metadata_for_user(`
`51`	`51`	`request.app, user_id=req_ctx.user_id, project_uuid=path_params.project_id`
`52`	`52`	`)`
`53`	`53`