Merge branch 'master' into is23/web-api-list-users-for-admin

Author: pcrespov

api/specs/web-server/_computations.py

@@ -6,17 +6,17 @@
 from models_library.api_schemas_webserver.computations import (
     ComputationGet,
     ComputationPathParams,
+    ComputationRunIterationsLatestListQueryParams,
+    ComputationRunIterationsListQueryParams,
     ComputationRunPathParams,
     ComputationRunRestGet,
-    ComputationRunWithFiltersListQueryParams,
     ComputationStart,
     ComputationStarted,
     ComputationTaskRestGet,
 )
 from models_library.generics import Envelope
 from simcore_service_webserver._meta import API_VTAG
 from simcore_service_webserver.director_v2._controller.computations_rest import (
-    ComputationRunListQueryParams,
     ComputationTaskListQueryParams,
     ComputationTaskPathParams,
 )
@@ -71,7 +71,9 @@ async def stop_computation(_path: Annotated[ComputationPathParams, Depends()]):
     response_model=Page[ComputationRunRestGet],
 )
 async def list_computations_latest_iteration(
-    _query: Annotated[as_query(ComputationRunWithFiltersListQueryParams), Depends()],
+    _query: Annotated[
+        as_query(ComputationRunIterationsLatestListQueryParams), Depends()
+    ],
 ): ...
 
 
@@ -80,7 +82,7 @@ async def list_computations_latest_iteration(
     response_model=Page[ComputationRunRestGet],
 )
 async def list_computation_iterations(
-    _query: Annotated[as_query(ComputationRunListQueryParams), Depends()],
+    _query: Annotated[as_query(ComputationRunIterationsListQueryParams), Depends()],
     _path: Annotated[ComputationRunPathParams, Depends()],
 ): ...
 

packages/dask-task-models-library/src/dask_task_models_library/container_tasks/io.py

@@ -195,7 +195,9 @@ def from_task_output(
             with suppress(json.JSONDecodeError):
                 # NOTE: The suppression here is ok, since if the data is empty,
                 # there will be a validation error anyway
-                data = json_loads(output_data_file.read_text())
+                loaded_data = json_loads(output_data_file.read_text())
+                # ignore what is not in the schema
+                data = {k: v for k, v in loaded_data.items() if k in schema}
 
         for output_key, output_params in schema.items():
             if isinstance(output_params, FilePortSchema):

packages/dask-task-models-library/tests/container_tasks/test_io.py

@@ -184,3 +184,39 @@ def test_objects_are_compatible_with_dask_requirements(model_cls, model_cls_exam
         model_instance = model_cls.model_validate(example)
         reloaded_instance = loads(dumps(model_instance))
         assert reloaded_instance == model_instance
+
+
+def test_create_task_output_from_task_ignores_additional_entries(
+    tmp_path: Path, faker: Faker
+):
+    task_output_schema = TaskOutputDataSchema.model_validate(
+        {
+            "some_output_1": {
+                "required": True,
+            },
+            "some_output_2": {
+                "required": True,
+            },
+        }
+    )
+    output_file = _create_fake_outputs(task_output_schema, tmp_path, False, faker)
+    assert output_file
+    # Add more data to the output file to simulate additional entries
+    file_path = tmp_path / output_file
+    data = json.loads(file_path.read_text())
+    # Ensure the file contains the expected keys first
+    for key in task_output_schema:
+        assert key in data
+    # Add an extra key
+    data["extra_key"] = "extra_value"
+    file_path.write_text(json.dumps(data))
+
+    task_output_data = TaskOutputData.from_task_output(
+        schema=task_output_schema,
+        output_folder=tmp_path,
+        output_file_ext=output_file,
+    )
+    # Only keys defined in the schema should be present
+    assert set(task_output_data.keys()) == set(
+        task_output_schema.keys()
+    ), "Should only contain the expected keys"

packages/models-library/src/models_library/api_schemas_catalog/services.py

@@ -172,23 +172,57 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
     )
 
 
-class _BaseServiceGetV2(CatalogOutputSchema):
-    # Model used in catalog's rpc and rest interfaces
+class ServiceSummary(CatalogOutputSchema):
     key: ServiceKey
     version: ServiceVersion
-
     name: str
-    thumbnail: HttpUrl | None = None
-    icon: HttpUrl | None = None
     description: str
 
-    description_ui: bool = False
-
     version_display: str | None = None
 
+    contact: LowerCaseEmailStr | None
+
+    @staticmethod
+    def _update_json_schema_extra(schema: JsonDict) -> None:
+        schema.update(
+            {
+                "examples": [
+                    {
+                        "key": _EXAMPLE_SLEEPER["key"],
+                        "version": _EXAMPLE_SLEEPER["version"],
+                        "name": _EXAMPLE_SLEEPER["name"],
+                        "description": _EXAMPLE_SLEEPER["description"],
+                        "version_display": _EXAMPLE_SLEEPER["version_display"],
+                        "contact": _EXAMPLE_SLEEPER["contact"],
+                    },
+                    {
+                        "key": _EXAMPLE_FILEPICKER["key"],
+                        "version": _EXAMPLE_FILEPICKER["version"],
+                        "name": _EXAMPLE_FILEPICKER["name"],
+                        "description": _EXAMPLE_FILEPICKER["description"],
+                        "version_display": None,
+                        "contact": _EXAMPLE_FILEPICKER["contact"],
+                    },
+                ]
+            }
+        )
+
+    model_config = ConfigDict(
+        extra="ignore",
+        populate_by_name=True,
+        alias_generator=snake_to_camel,
+        json_schema_extra=_update_json_schema_extra,
+    )
+
+
+class _BaseServiceGetV2(ServiceSummary):
     service_type: Annotated[ServiceType, Field(alias="type")]
 
-    contact: LowerCaseEmailStr | None
+    thumbnail: HttpUrl | None = None
+    icon: HttpUrl | None = None
+
+    description_ui: bool = False
+
     authors: Annotated[list[Author], Field(min_length=1)]
     owner: Annotated[
         LowerCaseEmailStr | None,
@@ -217,6 +251,7 @@ class _BaseServiceGetV2(CatalogOutputSchema):
         extra="forbid",
         populate_by_name=True,
         alias_generator=snake_to_camel,
+        json_schema_extra={"example": _EXAMPLE_SLEEPER},
     )
 
 
@@ -249,7 +284,6 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
 
 
 class ServiceGetV2(_BaseServiceGetV2):
-    # Model used in catalog's rpc and rest interfaces
     history: Annotated[
         list[ServiceRelease],
         Field(
@@ -338,6 +372,9 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
     ServiceRelease
 ]
 
+# Create PageRpc types
+PageRpcServiceSummary = PageRpc[ServiceSummary]
+
 ServiceResourcesGet: TypeAlias = ServiceResourcesDict
 
 

packages/models-library/src/models_library/api_schemas_webserver/computations.py

@@ -84,13 +84,20 @@ class ComputationRunListQueryParams(
 ): ...
 
 
-class ComputationRunWithFiltersListQueryParams(ComputationRunListQueryParams):
+class ComputationRunIterationsLatestListQueryParams(ComputationRunListQueryParams):
     filter_only_running: bool = Field(
         default=False,
         description="If true, only running computations are returned",
     )
 
 
+class ComputationRunIterationsListQueryParams(ComputationRunListQueryParams):
+    include_children: bool = Field(
+        default=False,
+        description="If true, all computational runs of the project and its children are returned (Currently supported only for root projects)",
+    )
+
+
 class ComputationRunRestGet(OutputSchema):
     project_uuid: ProjectID
     iteration: int
@@ -128,7 +135,11 @@ class ComputationTaskPathParams(BaseModel):
 class ComputationTaskListQueryParams(
     PageQueryParameters,
     ComputationTaskListOrderParams,  # type: ignore[misc, valid-type]
-): ...
+):
+    include_children: bool = Field(
+        default=False,
+        description="If true, all tasks of the project and its children are returned (Currently supported only for root projects)",
+    )
 
 
 class ComputationTaskRestGet(OutputSchema):

packages/postgres-database/src/simcore_postgres_database/migration/versions/4e7d8719855b_add_index_to_projects_metadata.py

@@ -0,0 +1,34 @@
+"""add index to projects_metadata
+
+Revision ID: 4e7d8719855b
+Revises: ba9c4816a31b
+Create Date: 2025-05-21 11:48:34.062860+00:00
+
+"""
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "4e7d8719855b"
+down_revision = "ba9c4816a31b"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(
+        "idx_projects_metadata_root_parent_project_uuid",
+        "projects_metadata",
+        ["root_parent_project_uuid"],
+        unique=False,
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(
+        "idx_projects_metadata_root_parent_project_uuid", table_name="projects_metadata"
+    )
+    # ### end Alembic commands ###

packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py

@@ -1,6 +1,6 @@
 """
-    These tables were designed to be controled by projects-plugin in
-    the webserver's service
+These tables were designed to be controled by projects-plugin in
+the webserver's service
 """
 
 import sqlalchemy as sa
@@ -100,6 +100,10 @@
         ondelete=RefActions.SET_NULL,
         name="fk_projects_metadata_root_parent_node_id",
     ),
+    #######
+    sa.Index(
+        "idx_projects_metadata_root_parent_project_uuid", "root_parent_project_uuid"
+    ),
 )
 
 

packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py

@@ -13,6 +13,7 @@
     LatestServiceGet,
     ServiceGetV2,
     ServiceListFilters,
+    ServiceSummary,
     ServiceUpdateV2,
 )
 from models_library.api_schemas_catalog.services_ports import ServicePortGet
@@ -200,6 +201,66 @@ async def get_service_ports(
             ServicePortGet.model_json_schema()["examples"],
         )
 
+    @validate_call(config={"arbitrary_types_allowed": True})
+    async def list_all_services_summaries_paginated(
+        self,
+        rpc_client: RabbitMQRPCClient | MockType,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        limit: PageLimitInt,
+        offset: NonNegativeInt,
+        filters: ServiceListFilters | None = None,
+    ):
+        assert rpc_client
+        assert product_name
+        assert user_id
+
+        service_summaries = TypeAdapter(list[ServiceSummary]).validate_python(
+            ServiceSummary.model_json_schema()["examples"],
+        )
+        if filters:
+            filtered_summaries = []
+            for summary in service_summaries:
+                # Match service type if specified
+                if (
+                    filters.service_type
+                    and {
+                        ServiceType.COMPUTATIONAL: "/comp/",
+                        ServiceType.DYNAMIC: "/dynamic/",
+                    }[filters.service_type]
+                    not in summary.key
+                ):
+                    continue
+
+                # Match service key pattern if specified
+                if filters.service_key_pattern and not fnmatch.fnmatch(
+                    summary.key, filters.service_key_pattern
+                ):
+                    continue
+
+                # Match version display pattern if specified
+                if filters.version_display_pattern and (
+                    summary.version_display is None
+                    or not fnmatch.fnmatch(
+                        summary.version_display, filters.version_display_pattern
+                    )
+                ):
+                    continue
+
+                filtered_summaries.append(summary)
+
+            service_summaries = filtered_summaries
+
+        total_count = len(service_summaries)
+
+        return PageRpc[ServiceSummary].create(
+            service_summaries[offset : offset + limit],
+            total=total_count,
+            limit=limit,
+            offset=offset,
+        )
+
 
 @dataclass
 class ZeroListingCatalogRpcSideEffects:
@@ -216,3 +277,11 @@ async def list_my_service_history_latest_first(self, *args, **kwargs):
             limit=10,
             offset=0,
         )
+
+    async def list_all_services_summaries_paginated(self, *args, **kwargs):
+        return PageRpc[ServiceSummary].create(
+            [],
+            total=0,
+            limit=10,
+            offset=0,
+        )