diff --git a/api/specs/web-server/_computations.py b/api/specs/web-server/_computations.py index fb60fce4175..4b565b9baa9 100644 --- a/api/specs/web-server/_computations.py +++ b/api/specs/web-server/_computations.py @@ -4,6 +4,9 @@ from fastapi import APIRouter, Depends, status from fastapi_pagination import Page from models_library.api_schemas_webserver.computations import ( + ComputationCollectionRunListQueryParams, + ComputationCollectionRunPathParams, + ComputationCollectionRunTaskListQueryParams, ComputationGet, ComputationPathParams, ComputationRunIterationsLatestListQueryParams, @@ -95,3 +98,22 @@ async def list_computations_latest_iteration_tasks( _query: Annotated[as_query(ComputationTaskListQueryParams), Depends()], _path: Annotated[ComputationTaskPathParams, Depends()], ): ... + + +@router.get( + "/computation-collection-runs", + response_model=Page[ComputationTaskRestGet], +) +async def list_computation_collection_runs( + _query: Annotated[as_query(ComputationCollectionRunListQueryParams), Depends()], +): ... + + +@router.get( + "/computation-collection-runs/{collection_run_id}/tasks", + response_model=Page[ComputationTaskRestGet], +) +async def list_computation_collection_run_tasks( + _query: Annotated[as_query(ComputationCollectionRunTaskListQueryParams), Depends()], + _path: Annotated[ComputationCollectionRunPathParams, Depends()], +): ... diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/comp_runs.py b/packages/models-library/src/models_library/api_schemas_directorv2/comp_runs.py index 7dc2b03c41b..37e3e2b29bb 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/comp_runs.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/comp_runs.py @@ -1,6 +1,7 @@ from datetime import datetime from typing import Any, NamedTuple +from models_library.computations import CollectionRunID from models_library.services_types import ServiceRunID from pydantic import ( AnyUrl, @@ -63,6 +64,55 @@ class ComputationRunRpcGetPage(NamedTuple): total: PositiveInt +class ComputationCollectionRunRpcGet(BaseModel): + collection_run_id: CollectionRunID + project_ids: list[str] + state: RunningState + info: dict[str, Any] + submitted_at: datetime + started_at: datetime | None + ended_at: datetime | None + + model_config = ConfigDict( + json_schema_extra={ + "examples": [ + { + "collection_run_id": "12e0c8b2-bad6-40fb-9948-8dec4f65d4d9", + "project_ids": ["beb16d18-d57d-44aa-a638-9727fa4a72ef"], + "state": "SUCCESS", + "info": { + "wallet_id": 9866, + "user_email": "test@example.net", + "wallet_name": "test", + "product_name": "osparc", + "project_name": "test", + "project_metadata": { + "parent_node_id": "12e0c8b2-bad6-40fb-9948-8dec4f65d4d9", + "parent_node_name": "UJyfwFVYySnPCaLuQIaz", + "parent_project_id": "beb16d18-d57d-44aa-a638-9727fa4a72ef", + "parent_project_name": "qTjDmYPxeqAWfCKCQCYF", + "root_parent_node_id": "37176e84-d977-4993-bc49-d76fcfc6e625", + "root_parent_node_name": "UEXExIZVPeFzGRmMglPr", + "root_parent_project_id": "beb16d18-d57d-44aa-a638-9727fa4a72ef", + "root_parent_project_name": "FuDpjjFIyeNTWRUWCuKo", + }, + "node_id_names_map": {}, + "simcore_user_agent": "agent", + }, + "submitted_at": "2023-01-11 13:11:47.293595", + "started_at": "2023-01-11 13:11:47.293595", + "ended_at": "2023-01-11 13:11:47.293595", + } + ] + } + ) + + +class ComputationCollectionRunRpcGetPage(NamedTuple): + items: list[ComputationCollectionRunRpcGet] + total: PositiveInt + + class ComputationTaskRpcGet(BaseModel): project_uuid: ProjectID node_id: NodeID @@ -100,3 +150,42 @@ class ComputationTaskRpcGet(BaseModel): class ComputationTaskRpcGetPage(NamedTuple): items: list[ComputationTaskRpcGet] total: PositiveInt + + +class ComputationCollectionRunTaskRpcGet(BaseModel): + project_uuid: ProjectID + node_id: NodeID + state: RunningState + progress: float + image: dict[str, Any] + started_at: datetime | None + ended_at: datetime | None + log_download_link: AnyUrl | None + service_run_id: ServiceRunID + + model_config = ConfigDict( + json_schema_extra={ + "examples": [ + { + "project_uuid": "beb16d18-d57d-44aa-a638-9727fa4a72ef", + "node_id": "12e0c8b2-bad6-40fb-9948-8dec4f65d4d9", + "state": "SUCCESS", + "progress": 0.0, + "image": { + "name": "simcore/services/comp/ti-solutions-optimizer", + "tag": "1.0.19", + "node_requirements": {"CPU": 8.0, "RAM": 25769803776}, + }, + "started_at": "2023-01-11 13:11:47.293595", + "ended_at": "2023-01-11 13:11:47.293595", + "log_download_link": "https://example.com/logs", + "service_run_id": "comp_1_12e0c8b2-bad6-40fb-9948-8dec4f65d4d9_1", + } + ] + } + ) + + +class ComputationCollectionRunTaskRpcGetPage(NamedTuple): + items: list[ComputationCollectionRunTaskRpcGet] + total: PositiveInt diff --git a/packages/models-library/src/models_library/api_schemas_directorv2/computations.py b/packages/models-library/src/models_library/api_schemas_directorv2/computations.py index 3691fdbf6ee..8a892676f63 100644 --- a/packages/models-library/src/models_library/api_schemas_directorv2/computations.py +++ b/packages/models-library/src/models_library/api_schemas_directorv2/computations.py @@ -1,5 +1,6 @@ from typing import Annotated, Any, TypeAlias +from models_library.computations import CollectionRunID from pydantic import ( AnyHttpUrl, AnyUrl, @@ -72,6 +73,12 @@ class ComputationCreate(BaseModel): description="contains information about the wallet used to bill the running service" ), ] = None + collection_run_id: Annotated[ + CollectionRunID | None, + Field( + description="In case start_pipeline is True, this is the collection run id to which the comp run belongs." + ), + ] = None @field_validator("product_name") @classmethod @@ -83,6 +90,20 @@ def _ensure_product_name_defined_if_computation_starts( raise ValueError(msg) return v + @field_validator("collection_run_id") + @classmethod + def _ensure_collection_run_id_dependency_on_start_pipeline( + cls, v, info: ValidationInfo + ): + start_pipeline = info.data.get("start_pipeline") + if start_pipeline and v is None: + msg = "collection_run_id must be provided when start_pipeline is True!" + raise ValueError(msg) + if not start_pipeline and v is not None: + msg = "collection_run_id must be None when start_pipeline is False!" + raise ValueError(msg) + return v + class ComputationStop(BaseModel): user_id: UserID diff --git a/packages/models-library/src/models_library/api_schemas_webserver/computations.py b/packages/models-library/src/models_library/api_schemas_webserver/computations.py index 0cd3d993b6d..ae4b0536020 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/computations.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/computations.py @@ -8,17 +8,20 @@ BaseModel, ConfigDict, Field, + field_validator, ) from ..api_schemas_directorv2.computations import ( ComputationGet as _DirectorV2ComputationGet, ) from ..basic_types import IDStr +from ..computations import CollectionRunID from ..projects import CommitID, ProjectID from ..projects_nodes_io import NodeID from ..projects_state import RunningState from ..rest_ordering import OrderBy, create_ordering_query_model_class from ..rest_pagination import PageQueryParameters +from ..utils.common_validators import null_or_none_str_to_none_validator from ._base import ( InputSchemaWithoutCamelCase, OutputSchema, @@ -153,3 +156,54 @@ class ComputationTaskRestGet(OutputSchema): log_download_link: AnyUrl | None node_name: str osparc_credits: Decimal | None + + +### Computation Collection Run + + +class ComputationCollectionRunListQueryParams( + PageQueryParameters, +): + filter_only_running: Annotated[ + bool, Field(description="If true, only running collection runs are returned") + ] = False + + filter_by_root_project_id: ProjectID | None = None + + _null_or_none_to_none = field_validator("filter_by_root_project_id", mode="before")( + null_or_none_str_to_none_validator + ) + + +class ComputationCollectionRunRestGet(OutputSchema): + collection_run_id: CollectionRunID + project_ids: list[str] + state: RunningState + info: dict[str, Any] + submitted_at: datetime + started_at: datetime | None + ended_at: datetime | None + name: str + + +class ComputationCollectionRunPathParams(BaseModel): + collection_run_id: CollectionRunID + model_config = ConfigDict(populate_by_name=True, extra="forbid") + + +class ComputationCollectionRunTaskListQueryParams( + PageQueryParameters, +): ... + + +class ComputationCollectionRunTaskRestGet(OutputSchema): + project_uuid: ProjectID + node_id: NodeID + state: RunningState + progress: float + image: dict[str, Any] + started_at: datetime | None + ended_at: datetime | None + log_download_link: AnyUrl | None + osparc_credits: Decimal | None + name: str diff --git a/packages/models-library/src/models_library/computations.py b/packages/models-library/src/models_library/computations.py index 6b88aff83ad..1efabc3fc73 100644 --- a/packages/models-library/src/models_library/computations.py +++ b/packages/models-library/src/models_library/computations.py @@ -1,6 +1,7 @@ from datetime import datetime from decimal import Decimal -from typing import Any +from typing import Any, TypeAlias +from uuid import UUID from pydantic import AnyUrl, BaseModel @@ -36,3 +37,34 @@ class ComputationRunWithAttributes(BaseModel): # Attributes added by the webserver root_project_name: str project_custom_metadata: dict[str, Any] + + +CollectionRunID: TypeAlias = UUID + + +class ComputationCollectionRunWithAttributes(BaseModel): + collection_run_id: CollectionRunID + project_ids: list[str] + state: RunningState + info: dict[str, Any] + submitted_at: datetime + started_at: datetime | None + ended_at: datetime | None + + # Attributes added by the webserver + name: str # Either root project name or collection name if provided by the client on start + + +class ComputationCollectionRunTaskWithAttributes(BaseModel): + project_uuid: ProjectID + node_id: NodeID + state: RunningState + progress: float + image: dict[str, Any] + started_at: datetime | None + ended_at: datetime | None + log_download_link: AnyUrl | None + + # Attributes added by the webserver + name: str # Either node name or job name if provided by the client on start + osparc_credits: Decimal | None diff --git a/packages/models-library/src/models_library/projects_state.py b/packages/models-library/src/models_library/projects_state.py index cef15bce5b5..560956a81a2 100644 --- a/packages/models-library/src/models_library/projects_state.py +++ b/packages/models-library/src/models_library/projects_state.py @@ -22,18 +22,32 @@ class RunningState(str, Enum): """State of execution of a project's computational workflow SEE StateType for task state + + # Computational backend states explained: + - UNKNOWN - The backend doesn't know about the task anymore, it has disappeared from the system or it was never created (eg. when we are asking for the task) + - NOT_STARTED - Default state when the task is created + - PUBLISHED - The task has been submitted to the computational backend (click on "Run" button in the UI) + - PENDING - Task has been transferred to the Dask scheduler and is waiting for a worker to pick it up (director-v2 --> Dask scheduler) + - But! it is also transition state (ex. PENDING -> WAITING_FOR_CLUSTER -> PENDING -> WAITING_FOR_RESOURCES -> PENDING -> STARTED) + - WAITING_FOR_CLUSTER - No cluster (Dask scheduler) is available to run the task; waiting for one to become available + - WAITING_FOR_RESOURCES - No worker (Dask worker) is available to run the task; waiting for one to become available + - STARTED - A worker has picked up the task and is executing it + - SUCCESS - Task finished successfully + - FAILED - Task finished with an error + - ABORTED - Task was aborted before completion + """ UNKNOWN = "UNKNOWN" - PUBLISHED = "PUBLISHED" NOT_STARTED = "NOT_STARTED" + PUBLISHED = "PUBLISHED" PENDING = "PENDING" + WAITING_FOR_CLUSTER = "WAITING_FOR_CLUSTER" WAITING_FOR_RESOURCES = "WAITING_FOR_RESOURCES" STARTED = "STARTED" SUCCESS = "SUCCESS" FAILED = "FAILED" ABORTED = "ABORTED" - WAITING_FOR_CLUSTER = "WAITING_FOR_CLUSTER" @staticmethod def list_running_states() -> list["RunningState"]: diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/42ec7816c0b4_computational_collection_runs.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/42ec7816c0b4_computational_collection_runs.py new file mode 100644 index 00000000000..b828e57e553 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/42ec7816c0b4_computational_collection_runs.py @@ -0,0 +1,121 @@ +"""computational collection runs + +Revision ID: 42ec7816c0b4 +Revises: d159ac30983c +Create Date: 2025-07-01 13:30:02.736058+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "42ec7816c0b4" +down_revision = "d159ac30983c" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "comp_runs_collections", + sa.Column( + "collection_run_id", + postgresql.UUID(as_uuid=True), + server_default=sa.text("gen_random_uuid()"), + nullable=False, + ), + sa.Column("client_or_system_generated_id", sa.String(), nullable=False), + sa.Column( + "client_or_system_generated_display_name", sa.String(), nullable=False + ), + sa.Column("is_generated_by_system", sa.Boolean(), nullable=False), + sa.Column( + "created", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("collection_run_id"), + ) + op.create_index( + "ix_comp_runs_collections_client_or_system_generated_id", + "comp_runs_collections", + ["client_or_system_generated_id"], + unique=False, + ) + op.add_column( + "comp_runs", sa.Column("collection_run_id", sa.String(), nullable=False) + ) + op.create_unique_constraint( + "comp_runs_project_collection_run_id_unique_constraint", + "comp_runs", + ["project_uuid", "collection_run_id"], + ) + + # Data migration: Create collection run records for existing comp_runs + op.execute( + """ + INSERT INTO comp_runs_collections ( + collection_run_id, + client_or_system_generated_id, + client_or_system_generated_display_name, + is_generated_by_system + ) + SELECT DISTINCT + gen_random_uuid(), + 'migration-generated-' || run_id::text, + 'Migration Generated Collection Run', + TRUE + FROM comp_runs + WHERE collection_run_id IS NULL + """ + ) + + # Update comp_runs to reference the newly created collection runs + op.execute( + """ + UPDATE comp_runs + SET collection_run_id = ( + SELECT collection_run_id::text + FROM comp_runs_collections + WHERE client_or_system_generated_id = 'migration-generated-' || comp_runs.run_id::text + ) + WHERE collection_run_id IS NULL + """ + ) + + op.alter_column( + "comp_runs", + "collection_run_id", + existing_type=sa.String(), + nullable=False, + ) + + op.create_index( + "ix_comp_runs_collection_run_id", + "comp_runs", + ["collection_run_id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_comp_runs_collection_run_id", table_name="comp_runs") + op.drop_column("comp_runs", "collection_run_id") + op.drop_index( + "ix_comp_runs_collections_client_or_system_generated_id", + table_name="comp_runs_collections", + ) + op.drop_table("comp_runs_collections") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/df61d1b2b967_computational_collection_runs_2.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/df61d1b2b967_computational_collection_runs_2.py new file mode 100644 index 00000000000..e4b986921b9 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/df61d1b2b967_computational_collection_runs_2.py @@ -0,0 +1,38 @@ +"""computational collection runs 2 + +Revision ID: df61d1b2b967 +Revises: 42ec7816c0b4 +Create Date: 2025-07-02 16:04:02.458800+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "df61d1b2b967" +down_revision = "42ec7816c0b4" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "funcapi_group_api_access_rights", + "read_functions", + existing_type=sa.BOOLEAN(), + nullable=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "funcapi_group_api_access_rights", + "read_functions", + existing_type=sa.BOOLEAN(), + nullable=True, + ) + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/comp_runs.py b/packages/postgres-database/src/simcore_postgres_database/models/comp_runs.py index efc1716cf10..13157505041 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/comp_runs.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/comp_runs.py @@ -105,6 +105,13 @@ server_default=sa.text("'{}'::jsonb"), nullable=False, ), + sa.Column( + "collection_run_id", + sa.String, + nullable=False, + ), sa.UniqueConstraint("project_uuid", "user_id", "iteration"), sa.Index("ix_comp_runs_user_id", "user_id"), + sa.Index("ix_comp_runs_collection_run_id", "collection_run_id"), + sa.UniqueConstraint("project_uuid", "collection_run_id"), ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/comp_runs_collections.py b/packages/postgres-database/src/simcore_postgres_database/models/comp_runs_collections.py new file mode 100644 index 00000000000..5953a90074b --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/comp_runs_collections.py @@ -0,0 +1,38 @@ +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import UUID + +from ._common import column_created_datetime, column_modified_datetime +from .base import metadata + +comp_runs_collections = sa.Table( + "comp_runs_collections", + metadata, + sa.Column( + "collection_run_id", + UUID(as_uuid=True), + server_default=sa.text("gen_random_uuid()"), + primary_key=True, + ), + sa.Column( + "client_or_system_generated_id", + sa.String, + nullable=False, + doc="Unique identifier for the collection run, generated by the client (ex. Third party using our public api) or system (ex. osparc webserver)", + ), + sa.Column( + "client_or_system_generated_display_name", + sa.String, + nullable=False, + ), + sa.Column( + "is_generated_by_system", + sa.Boolean, + nullable=False, + ), + column_created_datetime(timezone=True), + column_modified_datetime(timezone=True), + sa.Index( + "ix_comp_runs_collections_client_or_system_generated_id", + "client_or_system_generated_id", + ), +) diff --git a/packages/postgres-database/tests/test_utils_comp_runs.py b/packages/postgres-database/tests/test_utils_comp_runs.py index dc18abd8395..97f33bbdc18 100644 --- a/packages/postgres-database/tests/test_utils_comp_runs.py +++ b/packages/postgres-database/tests/test_utils_comp_runs.py @@ -4,13 +4,14 @@ import pytest import sqlalchemy as sa +from faker import Faker from simcore_postgres_database.models.comp_runs import comp_runs from simcore_postgres_database.utils_comp_runs import get_latest_run_id_for_project from sqlalchemy.ext.asyncio import AsyncEngine @pytest.fixture -async def sample_comp_runs(asyncpg_engine: AsyncEngine): +async def sample_comp_runs(asyncpg_engine: AsyncEngine, faker: Faker): async with asyncpg_engine.begin() as conn: await conn.execute(sa.text("SET session_replication_role = replica;")) await conn.execute(sa.delete(comp_runs)) @@ -37,6 +38,7 @@ async def sample_comp_runs(asyncpg_engine: AsyncEngine): "metadata": None, "use_on_demand_clusters": False, "dag_adjacency_list": {}, + "collection_run_id": faker.uuid4(), }, { "run_id": 2, @@ -58,6 +60,7 @@ async def sample_comp_runs(asyncpg_engine: AsyncEngine): "metadata": None, "use_on_demand_clusters": False, "dag_adjacency_list": {}, + "collection_run_id": faker.uuid4(), }, { "run_id": 3, @@ -79,6 +82,7 @@ async def sample_comp_runs(asyncpg_engine: AsyncEngine): "metadata": None, "use_on_demand_clusters": False, "dag_adjacency_list": {}, + "collection_run_id": faker.uuid4(), }, { "run_id": 4, @@ -100,6 +104,7 @@ async def sample_comp_runs(asyncpg_engine: AsyncEngine): "metadata": None, "use_on_demand_clusters": False, "dag_adjacency_list": {}, + "collection_run_id": faker.uuid4(), }, ], ) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_workspaces.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_workspaces.py index 1dbe5ebeb42..adf767b4945 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_workspaces.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_workspaces.py @@ -5,7 +5,7 @@ from simcore_postgres_database.models.workspaces_access_rights import ( workspaces_access_rights, ) -from simcore_service_webserver.db.plugin import get_database_engine +from simcore_service_webserver.db.plugin import get_database_engine_legacy from sqlalchemy.dialects.postgresql import insert as pg_insert @@ -18,7 +18,7 @@ async def update_or_insert_workspace_group( write: bool, delete: bool, ) -> None: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: insert_stmt = pg_insert(workspaces_access_rights).values( workspace_id=workspace_id, gid=group_id, diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py index a24ed19aba9..63d02b3a64a 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py @@ -6,9 +6,12 @@ DIRECTOR_V2_RPC_NAMESPACE, ) from models_library.api_schemas_directorv2.comp_runs import ( + ComputationCollectionRunRpcGetPage, + ComputationCollectionRunTaskRpcGetPage, ComputationRunRpcGetPage, ComputationTaskRpcGetPage, ) +from models_library.computations import CollectionRunID from models_library.products import ProductName from models_library.projects import ProjectID from models_library.rabbitmq_basic_types import RPCMethodName @@ -114,3 +117,60 @@ async def list_computations_latest_iteration_tasks_page( ) assert isinstance(result, ComputationTaskRpcGetPage) # nosec return result + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_computation_collection_runs_page( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + project_ids: list[ProjectID] | None, + # pagination + offset: int = 0, + limit: int = 20, +) -> ComputationCollectionRunRpcGetPage: + result = await rabbitmq_rpc_client.request( + DIRECTOR_V2_RPC_NAMESPACE, + _RPC_METHOD_NAME_ADAPTER.validate_python( + "list_computation_collection_runs_page" + ), + product_name=product_name, + user_id=user_id, + project_ids=project_ids, + offset=offset, + limit=limit, + timeout_s=_DEFAULT_TIMEOUT_S, + ) + assert isinstance(result, ComputationCollectionRunRpcGetPage) # nosec + return result + + +@log_decorator(_logger, level=logging.DEBUG) +async def list_computation_collection_run_tasks_page( + rabbitmq_rpc_client: RabbitMQRPCClient, + *, + product_name: ProductName, + user_id: UserID, + collection_run_id: CollectionRunID, + # pagination + offset: int = 0, + limit: int = 20, + # ordering + order_by: OrderBy | None = None, +) -> ComputationCollectionRunTaskRpcGetPage: + result = await rabbitmq_rpc_client.request( + DIRECTOR_V2_RPC_NAMESPACE, + _RPC_METHOD_NAME_ADAPTER.validate_python( + "list_computation_collection_run_tasks_page" + ), + product_name=product_name, + user_id=user_id, + collection_run_id=collection_run_id, + offset=offset, + limit=limit, + order_by=order_by, + timeout_s=_DEFAULT_TIMEOUT_S, + ) + assert isinstance(result, ComputationCollectionRunTaskRpcGetPage) # nosec + return result diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index 6d57f716398..883c72b81cb 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -10712,18 +10712,18 @@ "type": "string", "enum": [ "UNKNOWN", - "PUBLISHED", "NOT_STARTED", + "PUBLISHED", "PENDING", + "WAITING_FOR_CLUSTER", "WAITING_FOR_RESOURCES", "STARTED", "SUCCESS", "FAILED", - "ABORTED", - "WAITING_FOR_CLUSTER" + "ABORTED" ], "title": "RunningState", - "description": "State of execution of a project's computational workflow\n\nSEE StateType for task state" + "description": "State of execution of a project's computational workflow\n\nSEE StateType for task state\n\n# Computational backend states explained:\n- UNKNOWN - The backend doesn't know about the task anymore, it has disappeared from the system or it was never created (eg. when we are asking for the task)\n- NOT_STARTED - Default state when the task is created\n- PUBLISHED - The task has been submitted to the computational backend (click on \"Run\" button in the UI)\n- PENDING - Task has been transferred to the Dask scheduler and is waiting for a worker to pick it up (director-v2 --> Dask scheduler)\n - But! it is also transition state (ex. PENDING -> WAITING_FOR_CLUSTER -> PENDING -> WAITING_FOR_RESOURCES -> PENDING -> STARTED)\n- WAITING_FOR_CLUSTER - No cluster (Dask scheduler) is available to run the task; waiting for one to become available\n- WAITING_FOR_RESOURCES - No worker (Dask worker) is available to run the task; waiting for one to become available\n- STARTED - A worker has picked up the task and is executing it\n- SUCCESS - Task finished successfully\n- FAILED - Task finished with an error\n- ABORTED - Task was aborted before completion" }, "ServicePricingPlanGetLegacy": { "properties": { diff --git a/services/director-v2/VERSION b/services/director-v2/VERSION index 276cbf9e285..197c4d5c2d7 100644 --- a/services/director-v2/VERSION +++ b/services/director-v2/VERSION @@ -1 +1 @@ -2.3.0 +2.4.0 diff --git a/services/director-v2/openapi.json b/services/director-v2/openapi.json index b73eea273ef..1192f3e9425 100644 --- a/services/director-v2/openapi.json +++ b/services/director-v2/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "simcore-service-director-v2", "description": "Orchestrates the pipeline of services defined by the user", - "version": "2.3.0" + "version": "2.4.0" }, "servers": [ { @@ -1348,6 +1348,19 @@ } ], "description": "contains information about the wallet used to bill the running service" + }, + "collection_run_id": { + "anyOf": [ + { + "type": "string", + "format": "uuid" + }, + { + "type": "null" + } + ], + "title": "Collection Run Id", + "description": "In case start_pipeline is True, this is the collection run id to which the comp run belongs." } }, "type": "object", @@ -2789,18 +2802,18 @@ "type": "string", "enum": [ "UNKNOWN", - "PUBLISHED", "NOT_STARTED", + "PUBLISHED", "PENDING", + "WAITING_FOR_CLUSTER", "WAITING_FOR_RESOURCES", "STARTED", "SUCCESS", "FAILED", - "ABORTED", - "WAITING_FOR_CLUSTER" + "ABORTED" ], "title": "RunningState", - "description": "State of execution of a project's computational workflow\n\nSEE StateType for task state" + "description": "State of execution of a project's computational workflow\n\nSEE StateType for task state\n\n# Computational backend states explained:\n- UNKNOWN - The backend doesn't know about the task anymore, it has disappeared from the system or it was never created (eg. when we are asking for the task)\n- NOT_STARTED - Default state when the task is created\n- PUBLISHED - The task has been submitted to the computational backend (click on \"Run\" button in the UI)\n- PENDING - Task has been transferred to the Dask scheduler and is waiting for a worker to pick it up (director-v2 --> Dask scheduler)\n - But! it is also transition state (ex. PENDING -> WAITING_FOR_CLUSTER -> PENDING -> WAITING_FOR_RESOURCES -> PENDING -> STARTED)\n- WAITING_FOR_CLUSTER - No cluster (Dask scheduler) is available to run the task; waiting for one to become available\n- WAITING_FOR_RESOURCES - No worker (Dask worker) is available to run the task; waiting for one to become available\n- STARTED - A worker has picked up the task and is executing it\n- SUCCESS - Task finished successfully\n- FAILED - Task finished with an error\n- ABORTED - Task was aborted before completion" }, "SchedulerData": { "properties": { diff --git a/services/director-v2/setup.cfg b/services/director-v2/setup.cfg index f84ced2849b..20165c1866d 100644 --- a/services/director-v2/setup.cfg +++ b/services/director-v2/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.3.0 +current_version = 2.4.0 commit = True message = services/director-v2 version: {current_version} → {new_version} tag = False diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py index f8798aa3e63..dda8598ccf9 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/computations.py @@ -224,6 +224,11 @@ async def _try_start_pipeline( wallet_id = computation.wallet_info.wallet_id wallet_name = computation.wallet_info.wallet_name + if computation.collection_run_id is None: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail=f"Project {computation.project_id} has no collection run ID", + ) await run_new_pipeline( app, user_id=computation.user_id, @@ -245,6 +250,7 @@ async def _try_start_pipeline( ) or {}, use_on_demand_clusters=computation.use_on_demand_clusters, + collection_run_id=computation.collection_run_id, ) diff --git a/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py b/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py index eeb270c46cb..1335d6d86e5 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py @@ -1,11 +1,15 @@ # pylint: disable=too-many-arguments from fastapi import FastAPI from models_library.api_schemas_directorv2.comp_runs import ( + ComputationCollectionRunRpcGetPage, + ComputationCollectionRunTaskRpcGet, + ComputationCollectionRunTaskRpcGetPage, ComputationRunRpcGetPage, ComputationTaskRpcGet, ComputationTaskRpcGetPage, ) from models_library.api_schemas_directorv2.computations import TaskLogFileGet +from models_library.computations import CollectionRunID from models_library.products import ProductName from models_library.projects import ProjectID from models_library.rest_ordering import OrderBy @@ -15,9 +19,15 @@ from servicelib.utils import limited_gather from ...core.errors import ComputationalRunNotFoundError +from ...models.comp_run_snapshot_tasks import ( + CompRunSnapshotTaskDBGet, +) from ...models.comp_runs import CompRunsAtDB from ...models.comp_tasks import ComputationTaskForRpcDBGet from ...modules.db.repositories.comp_runs import CompRunsRepository +from ...modules.db.repositories.comp_runs_snapshot_tasks import ( + CompRunsSnapshotTasksRepository, +) from ...modules.db.repositories.comp_tasks import CompTasksRepository from ...utils import dask as dask_utils @@ -85,8 +95,33 @@ async def list_computations_iterations_page( ) +@router.expose(reraise_if_error_type=()) +async def list_computation_collection_runs_page( + app: FastAPI, + *, + product_name: ProductName, + user_id: UserID, + project_ids: list[ProjectID] | None, + # pagination + offset: int = 0, + limit: int = 20, +) -> ComputationCollectionRunRpcGetPage: + comp_runs_repo = CompRunsRepository.instance(db_engine=app.state.engine) + total, comp_runs_output = await comp_runs_repo.list_group_by_collection_run_id( + product_name=product_name, + user_id=user_id, + project_ids=project_ids, + offset=offset, + limit=limit, + ) + return ComputationCollectionRunRpcGetPage( + items=comp_runs_output, + total=total, + ) + + async def _fetch_task_log( - user_id: UserID, task: ComputationTaskForRpcDBGet + user_id: UserID, task: CompRunSnapshotTaskDBGet | ComputationTaskForRpcDBGet ) -> TaskLogFileGet | None: if not task.state.is_running(): return await dask_utils.get_task_log_file( @@ -182,3 +217,63 @@ async def list_computations_latest_iteration_tasks_page( items=comp_tasks_output, total=total, ) + + +@router.expose(reraise_if_error_type=()) +async def list_computation_collection_run_tasks_page( + app: FastAPI, + *, + product_name: ProductName, + user_id: UserID, + collection_run_id: CollectionRunID, + # pagination + offset: int = 0, + limit: int = 20, + # ordering + order_by: OrderBy | None = None, +) -> ComputationCollectionRunTaskRpcGetPage: + comp_runs_snapshot_tasks_repo = CompRunsSnapshotTasksRepository.instance( + db_engine=app.state.engine + ) + + total, comp_tasks = ( + await comp_runs_snapshot_tasks_repo.list_computation_collection_run_tasks( + product_name=product_name, + user_id=user_id, + collection_run_id=collection_run_id, + offset=offset, + limit=limit, + order_by=order_by, + ) + ) + + # Run all log fetches concurrently + log_files = await limited_gather( + *[_fetch_task_log(user_id, task) for task in comp_tasks], + limit=20, + ) + + comp_tasks_output = [ + ComputationCollectionRunTaskRpcGet( + project_uuid=task.project_uuid, + node_id=task.node_id, + state=task.state, + progress=task.progress, + image=task.image, + started_at=task.started_at, + ended_at=task.ended_at, + log_download_link=log_file.download_link if log_file else None, + service_run_id=ServiceRunID.get_resource_tracking_run_id_for_computational( + user_id, + task.project_uuid, + task.node_id, + task.iteration, + ), + ) + for task, log_file in zip(comp_tasks, log_files, strict=True) + ] + + return ComputationCollectionRunTaskRpcGetPage( + items=comp_tasks_output, + total=total, + ) diff --git a/services/director-v2/src/simcore_service_director_v2/models/comp_run_snapshot_tasks.py b/services/director-v2/src/simcore_service_director_v2/models/comp_run_snapshot_tasks.py index cb781452435..435ba460d01 100644 --- a/services/director-v2/src/simcore_service_director_v2/models/comp_run_snapshot_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/models/comp_run_snapshot_tasks.py @@ -1,5 +1,11 @@ +from datetime import datetime +from typing import Annotated, Any + +from models_library.projects import ProjectID +from models_library.projects_nodes_io import NodeID +from models_library.projects_state import RunningState from models_library.resource_tracker import HardwareInfo -from pydantic import ConfigDict, PositiveInt +from pydantic import BaseModel, BeforeValidator, ConfigDict, PositiveInt from .comp_tasks import BaseCompTaskAtDB, Image @@ -75,3 +81,22 @@ class CompRunSnapshotTaskAtDBGet(BaseCompTaskAtDB): class CompRunSnapshotTaskAtDBCreate(BaseCompTaskAtDB): run_id: PositiveInt + + +def _none_to_zero_float_pre_validator(value: Any): + if value is None: + return 0.0 + return value + + +class CompRunSnapshotTaskDBGet(BaseModel): + snapshot_task_id: PositiveInt + run_id: PositiveInt + project_uuid: ProjectID + node_id: NodeID + state: RunningState + progress: Annotated[float, BeforeValidator(_none_to_zero_float_pre_validator)] + image: dict[str, Any] + started_at: datetime | None + ended_at: datetime | None + iteration: PositiveInt diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_manager.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_manager.py index a52a0b43938..430dc0f94b2 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_manager.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_manager.py @@ -5,6 +5,7 @@ import networkx as nx from common_library.async_tools import cancel_wait_task from fastapi import FastAPI +from models_library.computations import CollectionRunID from models_library.projects import ProjectID from models_library.users import UserID from servicelib.background_task import create_periodic_task @@ -45,6 +46,7 @@ async def run_new_pipeline( project_id: ProjectID, run_metadata: RunMetadataDict, use_on_demand_clusters: bool, + collection_run_id: CollectionRunID, ) -> None: """Sets a new pipeline to be scheduled on the computational resources.""" # ensure the pipeline exists and is populated with something @@ -77,6 +79,7 @@ async def run_new_pipeline( metadata=run_metadata, use_on_demand_clusters=use_on_demand_clusters, dag_adjacency_list=comp_pipeline_at_db.dag_adjacency_list, + collection_run_id=collection_run_id, ) tasks_to_run = await _get_pipeline_tasks_at_db(db_engine, project_id, dag) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py index f623663b4aa..e9a2f26ceab 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py @@ -6,8 +6,12 @@ import asyncpg # type: ignore[import-untyped] import sqlalchemy as sa import sqlalchemy.exc as sql_exc -from models_library.api_schemas_directorv2.comp_runs import ComputationRunRpcGet +from models_library.api_schemas_directorv2.comp_runs import ( + ComputationCollectionRunRpcGet, + ComputationRunRpcGet, +) from models_library.basic_types import IDStr +from models_library.computations import CollectionRunID from models_library.projects import ProjectID from models_library.projects_state import RunningState from models_library.rest_ordering import OrderBy, OrderDirection @@ -89,6 +93,29 @@ def _handle_foreign_key_violation( raise exc_type(**{k: error_keys.get(k) for k in exc_keys}) +def _resolve_grouped_state(states: list[RunningState]) -> RunningState: + # If any state is not a final state, return STARTED + final_states = { + RunningState.FAILED, + RunningState.ABORTED, + RunningState.SUCCESS, + RunningState.UNKNOWN, + } + if any(state not in final_states for state in states): + return RunningState.STARTED + # All are final states + if all(state == RunningState.SUCCESS for state in states): + return RunningState.SUCCESS + if any(state == RunningState.FAILED for state in states): + return RunningState.FAILED + if any(state == RunningState.ABORTED for state in states): + return RunningState.ABORTED + if any(state == RunningState.UNKNOWN for state in states): + return RunningState.UNKNOWN + # Fallback (should not happen) + return RunningState.STARTED + + class CompRunsRepository(BaseRepository): async def get( self, @@ -350,6 +377,78 @@ async def list_for_user_and_project_all_iterations( return cast(int, total_count), items + async def list_group_by_collection_run_id( + self, + *, + product_name: str, + user_id: UserID, + project_ids: list[ProjectID] | None = None, + # pagination + offset: int, + limit: int, + ) -> tuple[int, list[ComputationCollectionRunRpcGet]]: + base_select_query = sa.select( + comp_runs.c.collection_run_id, + sa.func.array_agg(comp_runs.c.project_uuid).label("project_ids"), + sa.func.array_agg(comp_runs.c.result).label("states"), + # For simplicity, we use any metadata from the collection (first one in aggregation order): + sa.literal_column("(jsonb_agg(comp_runs.metadata))[1]").label("info"), + sa.func.min(comp_runs.c.created).label("submitted_at"), + sa.func.min(comp_runs.c.started).label("started_at"), + sa.func.min(comp_runs.c.run_id).label("min_run_id"), + sa.case( + ( + sa.func.bool_or(comp_runs.c.ended.is_(None)), + None, + ), + else_=sa.func.max(comp_runs.c.ended), + ).label("ended_at"), + ).where( + (comp_runs.c.user_id == user_id) + & (comp_runs.c.metadata["product_name"].astext == product_name) + ) + + if project_ids: + base_select_query = base_select_query.where( + comp_runs.c.project_uuid.in_( + [f"{project_id}" for project_id in project_ids] + ) + ) + + base_select_query_with_group_by = base_select_query.group_by( + comp_runs.c.collection_run_id + ) + + count_query = sa.select(sa.func.count()).select_from( + base_select_query_with_group_by.subquery() + ) + + # Default ordering by min_run_id descending (biggest first) + list_query = base_select_query_with_group_by.order_by( + desc(literal_column("min_run_id")) + ) + + list_query = list_query.offset(offset).limit(limit) + + async with pass_or_acquire_connection(self.db_engine) as conn: + total_count = await conn.scalar(count_query) + items = [] + async for row in await conn.stream(list_query): + db_states = [DB_TO_RUNNING_STATE[s] for s in row["states"]] + resolved_state = _resolve_grouped_state(db_states) + items.append( + ComputationCollectionRunRpcGet( + collection_run_id=row["collection_run_id"], + project_ids=row["project_ids"], + state=resolved_state, + info={} if row["info"] is None else row["info"], + submitted_at=row["submitted_at"], + started_at=row["started_at"], + ended_at=row["ended_at"], + ) + ) + return cast(int, total_count), items + async def create( self, *, @@ -359,6 +458,7 @@ async def create( metadata: RunMetadataDict, use_on_demand_clusters: bool, dag_adjacency_list: dict[str, list[str]], + collection_run_id: CollectionRunID, ) -> CompRunsAtDB: try: async with transaction_context(self.db_engine) as conn: @@ -375,6 +475,7 @@ async def create( metadata=jsonable_encoder(metadata), use_on_demand_clusters=use_on_demand_clusters, dag_adjacency_list=dag_adjacency_list, + collection_run_id=f"{collection_run_id}", ) .returning(literal_column("*")) ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs_snapshot_tasks.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs_snapshot_tasks.py index fc3c096e09b..23db21a26eb 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs_snapshot_tasks.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs_snapshot_tasks.py @@ -1,5 +1,11 @@ import logging +from typing import cast +import sqlalchemy as sa +from models_library.basic_types import IDStr +from models_library.computations import CollectionRunID +from models_library.products import ProductName +from models_library.rest_ordering import OrderBy, OrderDirection from simcore_postgres_database.utils_comp_run_snapshot_tasks import ( COMP_RUN_SNAPSHOT_TASKS_DB_COLS, ) @@ -7,7 +13,9 @@ transaction_context, ) -from ..tables import comp_run_snapshot_tasks +from ....models.comp_run_snapshot_tasks import CompRunSnapshotTaskDBGet +from ....utils.db import DB_TO_RUNNING_STATE +from ..tables import comp_run_snapshot_tasks, comp_runs from ._base import BaseRepository logger = logging.getLogger(__name__) @@ -36,3 +44,79 @@ async def batch_create( except Exception: logger.exception("Failed to batch create comp run snapshot tasks") raise + + async def list_computation_collection_run_tasks( + self, + *, + product_name: ProductName, + user_id: int, + collection_run_id: CollectionRunID, + # pagination + offset: int = 0, + limit: int = 20, + # ordering + order_by: OrderBy | None = None, + ) -> tuple[int, list[CompRunSnapshotTaskDBGet]]: + if order_by is None: + order_by = OrderBy(field=IDStr("snapshot_task_id")) # default ordering + + prefiltered_comp_runs = ( + sa.select( + comp_runs.c.run_id, + comp_runs.c.iteration, + ).where( + (comp_runs.c.user_id == user_id) + & (comp_runs.c.metadata["product_name"].astext == product_name) + & (comp_runs.c.collection_run_id == f"{collection_run_id}") + ) + ).subquery("prefiltered_comp_runs") + + base_select_query = sa.select( + comp_run_snapshot_tasks.c.snapshot_task_id, + comp_run_snapshot_tasks.c.run_id, + comp_run_snapshot_tasks.c.project_id.label("project_uuid"), + comp_run_snapshot_tasks.c.node_id, + comp_run_snapshot_tasks.c.state, + comp_run_snapshot_tasks.c.progress, + comp_run_snapshot_tasks.c.image, + comp_run_snapshot_tasks.c.start.label("started_at"), + comp_run_snapshot_tasks.c.end.label("ended_at"), + prefiltered_comp_runs.c.iteration, + ).select_from( + comp_run_snapshot_tasks.join( + prefiltered_comp_runs, + comp_run_snapshot_tasks.c.run_id == prefiltered_comp_runs.c.run_id, + ) + ) + + # Select total count from base_query + count_query = sa.select(sa.func.count()).select_from( + base_select_query.subquery() + ) + + # Ordering and pagination + if order_by.direction == OrderDirection.ASC: + list_query = base_select_query.order_by( + sa.asc(getattr(comp_run_snapshot_tasks.c, order_by.field)), + comp_run_snapshot_tasks.c.snapshot_task_id, + ) + else: + list_query = base_select_query.order_by( + sa.desc(getattr(comp_run_snapshot_tasks.c, order_by.field)), + comp_run_snapshot_tasks.c.snapshot_task_id, + ) + list_query = list_query.offset(offset).limit(limit) + + async with self.db_engine.connect() as conn: + total_count = await conn.scalar(count_query) + + items = [ + CompRunSnapshotTaskDBGet.model_validate( + { + **row, + "state": DB_TO_RUNNING_STATE[row["state"]], # Convert the state + } + ) + async for row in await conn.stream(list_query) + ] + return cast(int, total_count), items diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index a261dc56262..f5dce1567de 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -7,6 +7,7 @@ import asyncio import json +import uuid from collections.abc import Awaitable, Callable from copy import deepcopy from dataclasses import dataclass @@ -979,6 +980,7 @@ async def test_pipeline_with_control_loop_made_of_dynamic_services_is_allowed( "start_pipeline": True, "product_name": osparc_product_name, "product_api_base_url": osparc_product_api_base_url, + "collection_run_id": str(uuid.uuid4()), }, ) assert ( @@ -1064,6 +1066,7 @@ async def test_pipeline_with_cycle_containing_a_computational_service_is_forbidd "start_pipeline": True, "product_name": osparc_product_name, "product_api_base_url": osparc_product_api_base_url, + "collection_run_id": str(uuid.uuid4()), }, ) assert ( diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index cc4c32899ae..13a56f99e98 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -3,6 +3,7 @@ # pylint: disable=unused-import import asyncio +import uuid from collections.abc import AsyncIterator, Awaitable, Callable from unittest.mock import AsyncMock @@ -99,6 +100,9 @@ async def _creator( "start_pipeline": start_pipeline, "product_name": product_name, "product_api_base_url": product_api_base_url, + "collection_run_id": ( + str(uuid.uuid4()) if start_pipeline is True else None + ), **kwargs, }, ) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py index fa54799cf31..44604713fa0 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_route_computations.py @@ -33,6 +33,7 @@ RutPricingPlanGet, RutPricingUnitGet, ) +from models_library.computations import CollectionRunID from models_library.projects import ProjectAtDB from models_library.projects_nodes import NodeID, NodeState from models_library.projects_pipeline import PipelineDetails @@ -376,6 +377,7 @@ async def test_computation_create_validators( product_name: str, product_api_base_url: AnyHttpUrl, with_product: dict[str, Any], + fake_collection_run_id: CollectionRunID, ): user = create_registered_user() proj = await create_project(user, workbench=fake_workbench_without_outputs) @@ -405,6 +407,7 @@ async def test_create_computation( create_registered_user: Callable[..., dict[str, Any]], create_project: Callable[..., Awaitable[ProjectAtDB]], async_client: httpx.AsyncClient, + fake_collection_run_id: CollectionRunID, ): user = create_registered_user() proj = await create_project(user, workbench=fake_workbench_without_outputs) @@ -512,6 +515,7 @@ async def test_create_computation_with_wallet( sqlalchemy_async_engine: AsyncEngine, fake_ec2_cpus: PositiveInt, fake_ec2_ram: ByteSize, + fake_collection_run_id: CollectionRunID, ): # In billable product a wallet is passed, with a selected pricing plan # the pricing plan contains information about the hardware that should be used @@ -619,6 +623,7 @@ async def test_create_computation_with_wallet_with_invalid_pricing_unit_name_rai create_project: Callable[..., Awaitable[ProjectAtDB]], async_client: httpx.AsyncClient, wallet_info: WalletInfo, + fake_collection_run_id: CollectionRunID, ): user = create_registered_user() proj = await create_project( @@ -662,6 +667,7 @@ async def test_create_computation_with_wallet_with_no_clusters_keeper_raises_503 create_project: Callable[..., Awaitable[ProjectAtDB]], async_client: httpx.AsyncClient, wallet_info: WalletInfo, + fake_collection_run_id: CollectionRunID, ): user = create_registered_user() proj = await create_project(user, workbench=fake_workbench_without_outputs) @@ -690,6 +696,7 @@ async def test_start_computation_without_product_fails( create_registered_user: Callable[..., dict[str, Any]], create_project: Callable[..., Awaitable[ProjectAtDB]], async_client: httpx.AsyncClient, + fake_collection_run_id: CollectionRunID, ): user = create_registered_user() proj = await create_project(user, workbench=fake_workbench_without_outputs) @@ -700,6 +707,32 @@ async def test_start_computation_without_product_fails( "user_id": f"{user['id']}", "project_id": f"{proj.uuid}", "start_pipeline": f"{True}", + "collection_run_id": f"{fake_collection_run_id}", + }, + ) + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY, response.text + + +async def test_start_computation_without_collection_run_id_fails( + minimal_configuration: None, + mocked_director_service_fcts: respx.MockRouter, + mocked_catalog_service_fcts: respx.MockRouter, + product_name: str, + fake_workbench_without_outputs: dict[str, Any], + create_registered_user: Callable[..., dict[str, Any]], + create_project: Callable[..., Awaitable[ProjectAtDB]], + async_client: httpx.AsyncClient, +): + user = create_registered_user() + proj = await create_project(user, workbench=fake_workbench_without_outputs) + create_computation_url = httpx.URL("/v2/computations") + response = await async_client.post( + create_computation_url, + json={ + "product_name": product_name, + "user_id": f"{user['id']}", + "project_id": f"{proj.uuid}", + "start_pipeline": f"{True}", }, ) assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY, response.text @@ -715,6 +748,7 @@ async def test_start_computation( create_registered_user: Callable[..., dict[str, Any]], create_project: Callable[..., Awaitable[ProjectAtDB]], async_client: httpx.AsyncClient, + fake_collection_run_id: CollectionRunID, ): user = create_registered_user() proj = await create_project(user, workbench=fake_workbench_without_outputs) @@ -728,6 +762,7 @@ async def test_start_computation( start_pipeline=True, product_name=product_name, product_api_base_url=product_api_base_url, + collection_run_id=fake_collection_run_id, ) ), ) @@ -749,6 +784,7 @@ async def test_start_computation_with_project_node_resources_defined( create_registered_user: Callable[..., dict[str, Any]], create_project: Callable[..., Awaitable[ProjectAtDB]], async_client: httpx.AsyncClient, + fake_collection_run_id: CollectionRunID, ): user = create_registered_user() assert "json_schema_extra" in ServiceResourcesDictHelpers.model_config @@ -777,6 +813,7 @@ async def test_start_computation_with_project_node_resources_defined( start_pipeline=True, product_name=product_name, product_api_base_url=product_api_base_url, + collection_run_id=fake_collection_run_id, ) ), ) @@ -797,6 +834,7 @@ async def test_start_computation_with_deprecated_services_raises_406( create_registered_user: Callable[..., dict[str, Any]], create_project: Callable[..., Awaitable[ProjectAtDB]], async_client: httpx.AsyncClient, + fake_collection_run_id: CollectionRunID, ): user = create_registered_user() proj = await create_project(user, workbench=fake_workbench_without_outputs) @@ -810,6 +848,7 @@ async def test_start_computation_with_deprecated_services_raises_406( start_pipeline=True, product_name=product_name, product_api_base_url=product_api_base_url, + collection_run_id=fake_collection_run_id, ) ), ) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_rpc_computations.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_rpc_computations.py index e062f65fd4b..336f5366c38 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_rpc_computations.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_rpc_computations.py @@ -10,10 +10,14 @@ from datetime import UTC, datetime, timedelta from typing import Any +from faker import Faker from models_library.api_schemas_directorv2.comp_runs import ( + ComputationCollectionRunRpcGetPage, + ComputationCollectionRunTaskRpcGetPage, ComputationRunRpcGetPage, ComputationTaskRpcGetPage, ) +from models_library.computations import CollectionRunID from models_library.projects import ProjectAtDB from models_library.projects_state import RunningState from servicelib.rabbitmq import RabbitMQRPCClient @@ -22,6 +26,9 @@ ) from simcore_postgres_database.models.comp_pipeline import StateType from simcore_service_director_v2.models.comp_pipelines import CompPipelineAtDB +from simcore_service_director_v2.models.comp_run_snapshot_tasks import ( + CompRunSnapshotTaskDBGet, +) from simcore_service_director_v2.models.comp_runs import CompRunsAtDB from simcore_service_director_v2.models.comp_tasks import CompTaskAtDB @@ -226,3 +233,87 @@ async def test_rpc_list_computation_runs_history( ) assert output.total == 3 assert isinstance(output, ComputationRunRpcGetPage) + + +async def test_rpc_list_computation_collection_runs_page_and_collection_run_tasks_page( + fake_workbench_without_outputs: dict[str, Any], # <-- Has 4 nodes + fake_workbench_adjacency: dict[str, Any], + create_registered_user: Callable[..., dict[str, Any]], + create_project: Callable[..., Awaitable[ProjectAtDB]], + create_pipeline: Callable[..., Awaitable[CompPipelineAtDB]], + create_tasks_from_project: Callable[..., Awaitable[list[CompTaskAtDB]]], + create_comp_run_snapshot_tasks: Callable[ + ..., Awaitable[list[CompRunSnapshotTaskDBGet]] + ], + create_comp_run: Callable[..., Awaitable[CompRunsAtDB]], + rpc_client: RabbitMQRPCClient, + faker: Faker, + with_product: dict[str, Any], +): + user = create_registered_user() + projects = [ + await create_project(user, workbench=fake_workbench_without_outputs) + for _ in range(3) + ] + + default_collection_run_id = CollectionRunID(f"{faker.uuid4(cast_to=None)}") + not_default_collection_run_id = CollectionRunID(f"{faker.uuid4(cast_to=None)}") + + collection_run_id_project_list = [ + default_collection_run_id, + default_collection_run_id, + not_default_collection_run_id, + ] + + for proj, collection_run_id in zip(projects, collection_run_id_project_list): + await create_pipeline( + project_id=f"{proj.uuid}", + dag_adjacency_list=fake_workbench_adjacency, + ) + await create_tasks_from_project( + user=user, project=proj, state=StateType.PUBLISHED, progress=None + ) + run = await create_comp_run( + user=user, + project=proj, + result=RunningState.SUCCESS, + started=datetime.now(tz=UTC) - timedelta(minutes=120), + ended=datetime.now(tz=UTC) - timedelta(minutes=100), + iteration=1, + dag_adjacency_list=fake_workbench_adjacency, + collection_run_id=f"{collection_run_id}", + ) + await create_comp_run_snapshot_tasks( + user=user, + project=proj, + run_id=run.run_id, + ) + + output = await rpc_computations.list_computation_collection_runs_page( + rpc_client, product_name="osparc", user_id=user["id"], project_ids=None + ) + assert output.total == 2 + assert len(output.items) == 2 + assert isinstance(output, ComputationCollectionRunRpcGetPage) + assert len(output.items[0].project_ids) == 1 + assert len(output.items[1].project_ids) == 2 + + output = await rpc_computations.list_computation_collection_run_tasks_page( + rpc_client, + product_name="osparc", + user_id=user["id"], + collection_run_id=default_collection_run_id, + ) + assert output.total == 8 + assert len(output.items) == 8 + isinstance(output, ComputationCollectionRunTaskRpcGetPage) + + output = await rpc_computations.list_computation_collection_run_tasks_page( + rpc_client, + product_name="osparc", + user_id=user["id"], + collection_run_id=not_default_collection_run_id, + ) + assert output.total == 4 + assert len(output.items) == 4 + isinstance(output, ComputationCollectionRunTaskRpcGetPage) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_db_repositories_comp_runs.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_db_repositories_comp_runs.py index 547aa51d52c..34d26ce135c 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_db_repositories_comp_runs.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_db_repositories_comp_runs.py @@ -8,6 +8,7 @@ import asyncio import datetime import random +import uuid from collections.abc import Awaitable, Callable from typing import Any, cast @@ -15,6 +16,7 @@ import pytest from _helpers import PublishedProject from faker import Faker +from models_library.computations import CollectionRunID from models_library.projects import ProjectID from models_library.projects_state import RunningState from models_library.users import UserID @@ -89,6 +91,7 @@ async def test_list( publish_project: Callable[[], Awaitable[PublishedProject]], run_metadata: RunMetadataDict, faker: Faker, + fake_collection_run_id: CollectionRunID, with_product: dict[str, Any], ): assert await CompRunsRepository(sqlalchemy_async_engine).list_() == [] @@ -103,6 +106,7 @@ async def test_list( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=fake_collection_run_id, ) assert await CompRunsRepository(sqlalchemy_async_engine).list_() == [created] @@ -115,6 +119,7 @@ async def test_list( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=faker.uuid4(), ) for n in range(50) ) @@ -281,6 +286,7 @@ async def test_create( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list={}, + collection_run_id=faker.uuid4(), ) published_project = await publish_project() with pytest.raises(UserNotFoundError): @@ -291,6 +297,7 @@ async def test_create( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=faker.uuid4(), ) created = await CompRunsRepository(sqlalchemy_async_engine).create( @@ -300,6 +307,7 @@ async def test_create( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=faker.uuid4(), ) got = await CompRunsRepository(sqlalchemy_async_engine).get( user_id=published_project.user["id"], @@ -315,6 +323,7 @@ async def test_create( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=faker.uuid4(), ) assert created != got assert created.iteration == got.iteration + 1 @@ -334,6 +343,7 @@ async def test_update( run_metadata: RunMetadataDict, faker: Faker, publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, with_product: dict[str, Any], ): # this updates nothing but also does not complain @@ -350,6 +360,7 @@ async def test_update( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=fake_collection_run_id, ) got = await CompRunsRepository(sqlalchemy_async_engine).get( @@ -375,6 +386,7 @@ async def test_set_run_result( run_metadata: RunMetadataDict, faker: Faker, publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, with_product: dict[str, Any], ): published_project = await publish_project() @@ -385,6 +397,7 @@ async def test_set_run_result( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=fake_collection_run_id, ) got = await CompRunsRepository(sqlalchemy_async_engine).get( user_id=published_project.user["id"], @@ -424,6 +437,7 @@ async def test_mark_for_cancellation( run_metadata: RunMetadataDict, faker: Faker, publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, with_product: dict[str, Any], ): published_project = await publish_project() @@ -434,6 +448,7 @@ async def test_mark_for_cancellation( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=fake_collection_run_id, ) got = await CompRunsRepository(sqlalchemy_async_engine).get( user_id=published_project.user["id"], @@ -457,6 +472,7 @@ async def test_mark_for_scheduling( run_metadata: RunMetadataDict, faker: Faker, publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, with_product: dict[str, Any], ): published_project = await publish_project() @@ -467,6 +483,7 @@ async def test_mark_for_scheduling( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=fake_collection_run_id, ) got = await CompRunsRepository(sqlalchemy_async_engine).get( user_id=published_project.user["id"], @@ -492,6 +509,7 @@ async def test_mark_scheduling_done( run_metadata: RunMetadataDict, faker: Faker, publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, with_product: dict[str, Any], ): published_project = await publish_project() @@ -502,6 +520,7 @@ async def test_mark_scheduling_done( metadata=run_metadata, use_on_demand_clusters=faker.pybool(), dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=fake_collection_run_id, ) got = await CompRunsRepository(sqlalchemy_async_engine).get( user_id=published_project.user["id"], @@ -520,3 +539,709 @@ async def test_mark_scheduling_done( assert updated != created assert updated.scheduled is None assert updated.processed is not None + + +def _normalize_uuids(data): + """Recursively convert UUID objects to strings in a nested dictionary.""" + if isinstance(data, dict): + return {k: _normalize_uuids(v) for k, v in data.items()} + if isinstance(data, list): + return [_normalize_uuids(i) for i in data] + if isinstance(data, uuid.UUID): + return str(data) + return data + + +async def test_list_group_by_collection_run_id( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + """Test list_group_by_collection_run_id function with simple data insertion and retrieval.""" + # Create a few published projects + published_project_1 = await publish_project() + published_project_2 = ( + await publish_project() + ) # Create a shared collection run ID for grouping + collection_run_id = fake_collection_run_id + + # Create computation runs with the same collection_run_id + await asyncio.gather( + CompRunsRepository(sqlalchemy_async_engine).create( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_1.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ), + CompRunsRepository(sqlalchemy_async_engine).create( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_2.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ), + ) + + # Test the list_group_by_collection_run_id function + total_count, items = await CompRunsRepository( + sqlalchemy_async_engine + ).list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project_1.user["id"], + offset=0, + limit=10, + ) + + # Assertions + assert total_count == 1 # One collection group + assert len(items) == 1 + + collection_item = items[0] + assert collection_item.collection_run_id == collection_run_id + assert len(collection_item.project_ids) == 2 # Two projects in the collection + assert str(published_project_1.project.uuid) in collection_item.project_ids + assert str(published_project_2.project.uuid) in collection_item.project_ids + assert ( + collection_item.state + == RunningState.STARTED # Initial state returned to activity overview + ) + assert collection_item.info == _normalize_uuids(run_metadata) + assert collection_item.submitted_at is not None + assert collection_item.started_at is None # Not started yet + assert collection_item.ended_at is None # Not ended yet + + +async def test_list_group_by_collection_run_id_with_mixed_states_returns_started( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + """Test that if any state is not final, the grouped state returns STARTED.""" + # Create published projects + published_project_1 = await publish_project() + published_project_2 = await publish_project() + published_project_3 = await publish_project() + + collection_run_id = fake_collection_run_id + repo = CompRunsRepository(sqlalchemy_async_engine) + + # Create computation runs with same collection_run_id + comp_run_1 = await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_1.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + comp_run_2 = await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_2.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + comp_run_3 = await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_3.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_3.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + + # Set mixed states: one SUCCESS (final), one FAILED (final), one STARTED (non-final) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=comp_run_1.iteration, + result_state=RunningState.SUCCESS, + final_state=True, + ) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=comp_run_2.iteration, + result_state=RunningState.FAILED, + final_state=True, + ) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=published_project_3.project.uuid, + iteration=comp_run_3.iteration, + result_state=RunningState.STARTED, + final_state=False, + ) + + # Test the list_group_by_collection_run_id function + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project_1.user["id"], + offset=0, + limit=10, + ) + + # Assertions + assert total_count == 1 + assert len(items) == 1 + collection_item = items[0] + assert collection_item.collection_run_id == collection_run_id + assert collection_item.state == RunningState.STARTED # Non-final state wins + + +async def test_list_group_by_collection_run_id_all_success_returns_success( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + """Test that if all states are SUCCESS, the grouped state returns SUCCESS.""" + published_project_1 = await publish_project() + published_project_2 = await publish_project() + + collection_run_id = fake_collection_run_id + repo = CompRunsRepository(sqlalchemy_async_engine) + + # Create computation runs + comp_run_1 = await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_1.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + comp_run_2 = await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_2.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + + # Set both to SUCCESS + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=comp_run_1.iteration, + result_state=RunningState.SUCCESS, + final_state=True, + ) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=comp_run_2.iteration, + result_state=RunningState.SUCCESS, + final_state=True, + ) + + # Test the function + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project_1.user["id"], + offset=0, + limit=10, + ) + + # Assertions + assert total_count == 1 + assert len(items) == 1 + collection_item = items[0] + assert collection_item.state == RunningState.SUCCESS + + +async def test_list_group_by_collection_run_id_with_failed_returns_failed( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + """Test that if any state is FAILED (among final states), the grouped state returns FAILED.""" + published_project_1 = await publish_project() + published_project_2 = await publish_project() + published_project_3 = await publish_project() + + collection_run_id = fake_collection_run_id + repo = CompRunsRepository(sqlalchemy_async_engine) + + # Create computation runs + comp_runs = [] + for project in [published_project_1, published_project_2, published_project_3]: + comp_run = await repo.create( + user_id=published_project_1.user["id"], + project_id=project.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=project.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + comp_runs.append((project, comp_run)) + + # Set states: SUCCESS, FAILED, ABORTED (all final states, but FAILED is present) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=comp_runs[0][0].project.uuid, + iteration=comp_runs[0][1].iteration, + result_state=RunningState.SUCCESS, + final_state=True, + ) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=comp_runs[1][0].project.uuid, + iteration=comp_runs[1][1].iteration, + result_state=RunningState.FAILED, + final_state=True, + ) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=comp_runs[2][0].project.uuid, + iteration=comp_runs[2][1].iteration, + result_state=RunningState.ABORTED, + final_state=True, + ) + + # Test the function + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project_1.user["id"], + offset=0, + limit=10, + ) + + # Assertions + assert total_count == 1 + assert len(items) == 1 + collection_item = items[0] + assert collection_item.state == RunningState.FAILED # FAILED takes precedence + + +async def test_list_group_by_collection_run_id_with_aborted_returns_aborted( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + """Test that if any state is ABORTED (but no FAILED), the grouped state returns ABORTED.""" + published_project_1 = await publish_project() + published_project_2 = await publish_project() + + collection_run_id = fake_collection_run_id + repo = CompRunsRepository(sqlalchemy_async_engine) + + # Create computation runs + comp_run_1 = await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_1.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + comp_run_2 = await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_2.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + + # Set states: SUCCESS, ABORTED (final states, no FAILED) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=comp_run_1.iteration, + result_state=RunningState.SUCCESS, + final_state=True, + ) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=comp_run_2.iteration, + result_state=RunningState.ABORTED, + final_state=True, + ) + + # Test the function + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project_1.user["id"], + offset=0, + limit=10, + ) + + # Assertions + assert total_count == 1 + assert len(items) == 1 + collection_item = items[0] + assert collection_item.state == RunningState.ABORTED + + +async def test_list_group_by_collection_run_id_with_unknown_returns_unknown( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + """Test that if any state is UNKNOWN (but no FAILED/ABORTED), the grouped state returns UNKNOWN.""" + published_project_1 = await publish_project() + published_project_2 = await publish_project() + + collection_run_id = fake_collection_run_id + repo = CompRunsRepository(sqlalchemy_async_engine) + + # Create computation runs + comp_run_1 = await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_1.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + comp_run_2 = await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_2.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + + # Set states: SUCCESS, UNKNOWN (final states, no FAILED/ABORTED) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=comp_run_1.iteration, + result_state=RunningState.SUCCESS, + final_state=True, + ) + await repo.set_run_result( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=comp_run_2.iteration, + result_state=RunningState.UNKNOWN, # --> is setup to be FAILED + final_state=True, + ) + + # Test the function + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project_1.user["id"], + offset=0, + limit=10, + ) + + # Assertions + assert total_count == 1 + assert len(items) == 1 + collection_item = items[0] + assert collection_item.state == RunningState.FAILED + + +async def test_list_group_by_collection_run_id_with_project_filter( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + with_product: dict[str, Any], +): + """Test list_group_by_collection_run_id with project_ids filter.""" + published_project_1 = await publish_project() + published_project_2 = await publish_project() + published_project_3 = await publish_project() + + collection_run_id_1 = CollectionRunID(f"{faker.uuid4(cast_to=None)}") + collection_run_id_2 = CollectionRunID(f"{faker.uuid4(cast_to=None)}") + repo = CompRunsRepository(sqlalchemy_async_engine) + + # Create computation runs with different collection_run_ids + await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_1.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_1.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id_1, + ) + await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_2.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_2.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id_1, + ) + await repo.create( + user_id=published_project_1.user["id"], + project_id=published_project_3.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_3.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id_2, + ) + + # Test with project filter for only first two projects + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project_1.user["id"], + project_ids=[ + published_project_1.project.uuid, + published_project_2.project.uuid, + ], + offset=0, + limit=10, + ) + + # Should only return collection_run_id_1 + assert total_count == 1 + assert len(items) == 1 + collection_item = items[0] + assert collection_item.collection_run_id == collection_run_id_1 + assert len(collection_item.project_ids) == 2 + + +async def test_list_group_by_collection_run_id_pagination( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + with_product: dict[str, Any], +): + """Test pagination functionality of list_group_by_collection_run_id.""" + published_project = await publish_project() + repo = CompRunsRepository(sqlalchemy_async_engine) + + # Create multiple collection runs + collection_run_ids = [] + for _ in range(5): + collection_run_id = CollectionRunID(f"{faker.uuid4(cast_to=None)}") + collection_run_ids.append(collection_run_id) + + project = await publish_project() + await repo.create( + user_id=published_project.user["id"], + project_id=project.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=project.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + + # Test first page + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project.user["id"], + offset=0, + limit=2, + ) + + assert total_count == 5 + assert len(items) == 2 + + # Test second page + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project.user["id"], + offset=2, + limit=2, + ) + + assert total_count == 5 + assert len(items) == 2 + + # Test last page + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_project.user["id"], + offset=4, + limit=2, + ) + + assert total_count == 5 + assert len(items) == 1 + + +async def test_list_group_by_collection_run_id_empty_result( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + fake_user_id: UserID, + with_product: dict[str, Any], +): + """Test list_group_by_collection_run_id returns empty when no runs exist.""" + repo = CompRunsRepository(sqlalchemy_async_engine) + + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=fake_user_id, + offset=0, + limit=10, + ) + + assert total_count == 0 + assert len(items) == 0 + + +async def test_list_group_by_collection_run_id_with_different_users( + sqlalchemy_async_engine: AsyncEngine, + create_registered_user: Callable[..., dict[str, Any]], + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + with_product: dict[str, Any], +): + """Test that list_group_by_collection_run_id filters by user_id correctly.""" + published_project_user1 = await publish_project() + published_project_user2 = await publish_project() + + user1 = create_registered_user() + user2 = create_registered_user() + + collection_run_id_1 = CollectionRunID(f"{faker.uuid4(cast_to=None)}") + collection_run_id_2 = CollectionRunID(f"{faker.uuid4(cast_to=None)}") + + repo = CompRunsRepository(sqlalchemy_async_engine) + + # Create runs for different users with same collection_run_id + await repo.create( + user_id=user1["id"], + project_id=published_project_user1.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_user1.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id_1, + ) + await repo.create( + user_id=user2["id"], + project_id=published_project_user2.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=published_project_user2.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id_2, + ) + + # Test for user1 - should only see their own runs + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=user1["id"], + offset=0, + limit=10, + ) + + assert total_count == 1 + assert len(items) == 1 + collection_item = items[0] + assert len(collection_item.project_ids) == 1 + assert str(published_project_user1.project.uuid) in collection_item.project_ids + assert str(published_project_user2.project.uuid) not in collection_item.project_ids + + # Test for user2 - should only see their own runs + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=user2["id"], + offset=0, + limit=10, + ) + + assert total_count == 1 + assert len(items) == 1 + collection_item = items[0] + assert len(collection_item.project_ids) == 1 + assert str(published_project_user2.project.uuid) in collection_item.project_ids + assert str(published_project_user1.project.uuid) not in collection_item.project_ids + + +async def test_list_group_by_collection_run_id_state_priority_precedence( + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, + publish_project: Callable[[], Awaitable[PublishedProject]], + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + """Test that state resolution follows correct priority: FAILED > ABORTED > UNKNOWN.""" + published_projects = [await publish_project() for _ in range(4)] + + collection_run_id = fake_collection_run_id + repo = CompRunsRepository(sqlalchemy_async_engine) + + # Create computation runs + comp_runs = [] + for project in published_projects: + comp_run = await repo.create( + user_id=published_projects[0].user["id"], + project_id=project.project.uuid, + iteration=None, + metadata=run_metadata, + use_on_demand_clusters=faker.pybool(), + dag_adjacency_list=project.pipeline.dag_adjacency_list, + collection_run_id=collection_run_id, + ) + comp_runs.append((project, comp_run)) + + # Set states: SUCCESS, UNKNOWN, ABORTED, FAILED - should return FAILED + states = [ + RunningState.SUCCESS, + RunningState.UNKNOWN, + RunningState.ABORTED, + RunningState.FAILED, + ] + for i, (project, comp_run) in enumerate(comp_runs): + await repo.set_run_result( + user_id=published_projects[0].user["id"], + project_id=project.project.uuid, + iteration=comp_run.iteration, + result_state=states[i], + final_state=True, + ) + + # Test the function + total_count, items = await repo.list_group_by_collection_run_id( + product_name=run_metadata.get("product_name"), + user_id=published_projects[0].user["id"], + offset=0, + limit=10, + ) + + # Assertions - FAILED should have highest priority + assert total_count == 1 + assert len(items) == 1 + collection_item = items[0] + assert collection_item.state == RunningState.FAILED diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_db_repositories_comp_runs_snapshot_tasks.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_db_repositories_comp_runs_snapshot_tasks.py new file mode 100644 index 00000000000..1696d6f2fd9 --- /dev/null +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_db_repositories_comp_runs_snapshot_tasks.py @@ -0,0 +1,246 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments + + +from collections.abc import Awaitable, Callable +from typing import Any + +from _helpers import PublishedProject +from models_library.computations import CollectionRunID +from models_library.products import ProductName +from simcore_service_director_v2.models.comp_run_snapshot_tasks import ( + CompRunSnapshotTaskDBGet, +) +from simcore_service_director_v2.models.comp_runs import CompRunsAtDB +from simcore_service_director_v2.modules.db.repositories.comp_runs_snapshot_tasks import ( + CompRunsSnapshotTasksRepository, +) +from sqlalchemy.ext.asyncio import AsyncEngine + +pytest_simcore_core_services_selection = [ + "postgres", +] +pytest_simcore_ops_services_selection = [ + "adminer", +] + + +async def test_list_computation_collection_run_tasks( + sqlalchemy_async_engine: AsyncEngine, + publish_project: Callable[[], Awaitable[PublishedProject]], + create_comp_run: Callable[..., Awaitable[CompRunsAtDB]], + create_comp_run_snapshot_tasks: Callable[ + ..., Awaitable[list[CompRunSnapshotTaskDBGet]] + ], + osparc_product_name: ProductName, + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + repo = CompRunsSnapshotTasksRepository(db_engine=sqlalchemy_async_engine) + + # 1. create a project + published_project = await publish_project() + user_id = published_project.user["id"] + + # 2. create a comp_run + run = await create_comp_run( + published_project.user, + published_project.project, + dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=f"{fake_collection_run_id}", + ) + + # 3. create snapshot tasks for that run + snapshot_tasks = await create_comp_run_snapshot_tasks( + user=published_project.user, + project=published_project.project, + run_id=run.run_id, + ) + + # 4. list them + total_count, tasks = await repo.list_computation_collection_run_tasks( + product_name=osparc_product_name, + user_id=user_id, + collection_run_id=fake_collection_run_id, + ) + + assert total_count == len(snapshot_tasks) + assert tasks + assert len(tasks) == len(snapshot_tasks) + assert {t.snapshot_task_id for t in tasks} == { + t.snapshot_task_id for t in snapshot_tasks + } + + +async def test_list_computation_collection_run_tasks_empty( + sqlalchemy_async_engine: AsyncEngine, + osparc_product_name: ProductName, + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + repo = CompRunsSnapshotTasksRepository(db_engine=sqlalchemy_async_engine) + # Use a random user_id unlikely to have tasks + user_id = 999999 + total_count, tasks = await repo.list_computation_collection_run_tasks( + product_name=osparc_product_name, + user_id=user_id, + collection_run_id=fake_collection_run_id, + ) + assert total_count == 0 + assert tasks == [] + + +async def test_list_computation_collection_run_tasks_pagination( + sqlalchemy_async_engine: AsyncEngine, + publish_project: Callable[[], Awaitable[PublishedProject]], + create_comp_run: Callable[..., Awaitable[CompRunsAtDB]], + create_comp_run_snapshot_tasks: Callable[ + ..., Awaitable[list[CompRunSnapshotTaskDBGet]] + ], + osparc_product_name: ProductName, + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + repo = CompRunsSnapshotTasksRepository(db_engine=sqlalchemy_async_engine) + published_project = await publish_project() + user_id = published_project.user["id"] + run = await create_comp_run( + published_project.user, + published_project.project, + dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=f"{fake_collection_run_id}", + ) + snapshot_tasks = await create_comp_run_snapshot_tasks( + user=published_project.user, + project=published_project.project, + run_id=run.run_id, + ) + # Test pagination: limit=1 + total_count, tasks = await repo.list_computation_collection_run_tasks( + product_name=osparc_product_name, + user_id=user_id, + collection_run_id=fake_collection_run_id, + limit=1, + offset=0, + ) + assert total_count == len(snapshot_tasks) + assert len(tasks) == 1 + # Test pagination: offset=1 + _, tasks_offset = await repo.list_computation_collection_run_tasks( + product_name=osparc_product_name, + user_id=user_id, + collection_run_id=fake_collection_run_id, + limit=1, + offset=1, + ) + assert len(tasks_offset) == 1 or ( + len(snapshot_tasks) == 1 and len(tasks_offset) == 0 + ) + + +async def test_list_computation_collection_run_tasks_wrong_user( + sqlalchemy_async_engine: AsyncEngine, + publish_project: Callable[[], Awaitable[PublishedProject]], + create_comp_run: Callable[..., Awaitable[CompRunsAtDB]], + create_comp_run_snapshot_tasks: Callable[ + ..., Awaitable[list[CompRunSnapshotTaskDBGet]] + ], + osparc_product_name: ProductName, + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + repo = CompRunsSnapshotTasksRepository(db_engine=sqlalchemy_async_engine) + published_project = await publish_project() + run = await create_comp_run( + published_project.user, + published_project.project, + dag_adjacency_list=published_project.pipeline.dag_adjacency_list, + collection_run_id=f"{fake_collection_run_id}", + ) + await create_comp_run_snapshot_tasks( + user=published_project.user, + project=published_project.project, + run_id=run.run_id, + ) + # Use a different user_id + wrong_user_id = 123456789 + total_count, tasks = await repo.list_computation_collection_run_tasks( + product_name=osparc_product_name, + user_id=wrong_user_id, + collection_run_id=fake_collection_run_id, + ) + assert total_count == 0 + assert tasks == [] + + +async def test_list_computation_collection_run_tasks_multiple_comp_runs_same_collection( + sqlalchemy_async_engine: AsyncEngine, + publish_project: Callable[[], Awaitable[PublishedProject]], + create_comp_run: Callable[..., Awaitable[CompRunsAtDB]], + create_comp_run_snapshot_tasks: Callable[ + ..., Awaitable[list[CompRunSnapshotTaskDBGet]] + ], + osparc_product_name: ProductName, + fake_collection_run_id: CollectionRunID, + with_product: dict[str, Any], +): + repo = CompRunsSnapshotTasksRepository(db_engine=sqlalchemy_async_engine) + published_project1 = await publish_project() + published_project2 = await publish_project() + published_project3 = await publish_project() + user_id = published_project1.user["id"] + + # Create 3 comp_runs, 2 with the same collection_run_id, 1 with a different one + run1 = await create_comp_run( + published_project1.user, + published_project1.project, + dag_adjacency_list=published_project1.pipeline.dag_adjacency_list, + collection_run_id=f"{fake_collection_run_id}", + ) + run2 = await create_comp_run( + published_project2.user, + published_project2.project, + dag_adjacency_list=published_project2.pipeline.dag_adjacency_list, + collection_run_id=f"{fake_collection_run_id}", + ) + other_collection_run_id = CollectionRunID("00000000-0000-0000-0000-000000000001") + run3 = await create_comp_run( + published_project3.user, + published_project3.project, + dag_adjacency_list=published_project3.pipeline.dag_adjacency_list, + collection_run_id=f"{other_collection_run_id}", + ) + + # Create snapshot tasks for each run + tasks_run1 = await create_comp_run_snapshot_tasks( + user=published_project1.user, + project=published_project1.project, + run_id=run1.run_id, + ) + tasks_run2 = await create_comp_run_snapshot_tasks( + user=published_project2.user, + project=published_project2.project, + run_id=run2.run_id, + ) + tasks_run3 = await create_comp_run_snapshot_tasks( + user=published_project3.user, + project=published_project3.project, + run_id=run3.run_id, + ) + + # Query for tasks with the shared collection_run_id + total_count, tasks = await repo.list_computation_collection_run_tasks( + product_name=osparc_product_name, + user_id=user_id, + collection_run_id=fake_collection_run_id, + ) + expected_task_ids = {t.snapshot_task_id for t in tasks_run1 + tasks_run2} + actual_task_ids = {t.snapshot_task_id for t in tasks} + assert total_count == len(expected_task_ids) + assert actual_task_ids == expected_task_ids + # Ensure tasks from run3 are not included + assert not any( + t.snapshot_task_id in {tt.snapshot_task_id for tt in tasks_run3} for t in tasks + ) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py index 596aaaf5bb6..fdd5a184bf2 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_manager.py @@ -18,6 +18,7 @@ import pytest from _helpers import PublishedProject, assert_comp_runs, assert_comp_runs_empty from fastapi import FastAPI +from models_library.computations import CollectionRunID from models_library.projects import ProjectAtDB from models_library.projects_state import RunningState from pytest_mock.plugin import MockerFixture @@ -149,6 +150,7 @@ async def test_schedule_all_pipelines( sqlalchemy_async_engine: AsyncEngine, run_metadata: RunMetadataDict, scheduler_rabbit_client_parser: mock.AsyncMock, + fake_collection_run_id: CollectionRunID, ): await assert_comp_runs_empty(sqlalchemy_async_engine) assert published_project.project.prj_owner @@ -159,6 +161,7 @@ async def test_schedule_all_pipelines( project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) # this directly schedule a new pipeline scheduler_rabbit_client_parser.assert_called_once_with( @@ -250,6 +253,7 @@ async def test_schedule_all_pipelines_logs_error_if_it_find_old_pipelines( run_metadata: RunMetadataDict, scheduler_rabbit_client_parser: mock.AsyncMock, caplog: pytest.LogCaptureFixture, + fake_collection_run_id: CollectionRunID, ): await assert_comp_runs_empty(sqlalchemy_async_engine) assert published_project.project.prj_owner @@ -260,6 +264,7 @@ async def test_schedule_all_pipelines_logs_error_if_it_find_old_pipelines( project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) # this directly schedule a new pipeline scheduler_rabbit_client_parser.assert_called_once_with( @@ -331,6 +336,7 @@ async def test_empty_pipeline_is_not_scheduled( sqlalchemy_async_engine: AsyncEngine, scheduler_rabbit_client_parser: mock.AsyncMock, caplog: pytest.LogCaptureFixture, + fake_collection_run_id: CollectionRunID, ): await assert_comp_runs_empty(sqlalchemy_async_engine) user = create_registered_user() @@ -344,6 +350,7 @@ async def test_empty_pipeline_is_not_scheduled( project_id=empty_project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) await assert_comp_runs_empty(sqlalchemy_async_engine) scheduler_rabbit_client_parser.assert_not_called() @@ -359,6 +366,7 @@ async def test_empty_pipeline_is_not_scheduled( project_id=empty_project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) assert len(caplog.records) == 1 assert "no computational dag defined" in caplog.records[0].message diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_scheduler_dask.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_scheduler_dask.py index cb24d649f93..4e298139cb5 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_scheduler_dask.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_scheduler_dask.py @@ -32,6 +32,7 @@ from dask_task_models_library.container_tasks.protocol import TaskOwner from faker import Faker from fastapi.applications import FastAPI +from models_library.computations import CollectionRunID from models_library.projects import ProjectAtDB, ProjectID from models_library.projects_nodes_io import NodeID from models_library.projects_state import RunningState @@ -164,6 +165,7 @@ async def _assert_start_pipeline( published_project: PublishedProject, run_metadata: RunMetadataDict, computational_pipeline_rabbit_client_parser: mock.AsyncMock, + collection_run_id: CollectionRunID, ) -> tuple[CompRunsAtDB, list[CompTaskAtDB]]: exp_published_tasks = deepcopy(published_project.tasks) assert published_project.project.prj_owner @@ -173,6 +175,7 @@ async def _assert_start_pipeline( project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=collection_run_id, ) # check the database is correctly updated, the run is published @@ -472,6 +475,7 @@ async def test_proper_pipeline_is_scheduled( # noqa: PLR0915 resource_tracking_rabbit_client_parser: mock.AsyncMock, computational_pipeline_rabbit_client_parser: mock.AsyncMock, run_metadata: RunMetadataDict, + fake_collection_run_id: CollectionRunID, ): with_disabled_auto_scheduling.assert_called_once() _with_mock_send_computation_tasks(published_project.tasks, mocked_dask_client) @@ -485,6 +489,7 @@ async def test_proper_pipeline_is_scheduled( # noqa: PLR0915 published_project=published_project, run_metadata=run_metadata, computational_pipeline_rabbit_client_parser=computational_pipeline_rabbit_client_parser, + collection_run_id=fake_collection_run_id, ) with_disabled_scheduler_publisher.assert_called() @@ -965,6 +970,7 @@ async def with_started_project( instrumentation_rabbit_client_parser: mock.AsyncMock, resource_tracking_rabbit_client_parser: mock.AsyncMock, computational_pipeline_rabbit_client_parser: mock.AsyncMock, + fake_collection_run_id: CollectionRunID, with_product: dict[str, Any], ) -> RunningProject: with_disabled_auto_scheduling.assert_called_once() @@ -978,6 +984,7 @@ async def with_started_project( published_project=published_project, run_metadata=run_metadata, computational_pipeline_rabbit_client_parser=computational_pipeline_rabbit_client_parser, + collection_run_id=fake_collection_run_id, ) with_disabled_scheduler_publisher.assert_called_once() @@ -1211,6 +1218,7 @@ async def test_broken_pipeline_configuration_is_not_scheduled_and_aborted( sqlalchemy_async_engine: AsyncEngine, run_metadata: RunMetadataDict, computational_pipeline_rabbit_client_parser: mock.AsyncMock, + fake_collection_run_id: CollectionRunID, with_product: dict[str, Any], ): """A pipeline which comp_tasks are missing should not be scheduled. @@ -1234,6 +1242,7 @@ async def test_broken_pipeline_configuration_is_not_scheduled_and_aborted( project_id=sleepers_project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) with_disabled_scheduler_publisher.assert_called_once() # we shall have a a new comp_runs row with the new pipeline job @@ -1286,6 +1295,7 @@ async def test_task_progress_triggers( mocked_clean_task_output_and_log_files_if_invalid: mock.Mock, run_metadata: RunMetadataDict, computational_pipeline_rabbit_client_parser: mock.AsyncMock, + fake_collection_run_id: CollectionRunID, ): _with_mock_send_computation_tasks(published_project.tasks, mocked_dask_client) _run_in_db, expected_published_tasks = await _assert_start_pipeline( @@ -1294,6 +1304,7 @@ async def test_task_progress_triggers( published_project=published_project, run_metadata=run_metadata, computational_pipeline_rabbit_client_parser=computational_pipeline_rabbit_client_parser, + collection_run_id=fake_collection_run_id, ) # ------------------------------------------------------------------------------- @@ -1361,6 +1372,7 @@ async def test_handling_of_disconnected_scheduler_dask( backend_error: ComputationalSchedulerError, run_metadata: RunMetadataDict, computational_pipeline_rabbit_client_parser: mock.AsyncMock, + fake_collection_run_id: CollectionRunID, ): # this will create a non connected backend issue that will trigger re-connection mocked_dask_client_send_task = mocker.patch( @@ -1377,6 +1389,7 @@ async def test_handling_of_disconnected_scheduler_dask( project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) await _assert_message_received( computational_pipeline_rabbit_client_parser, @@ -1805,6 +1818,7 @@ async def test_running_pipeline_triggers_heartbeat( resource_tracking_rabbit_client_parser: mock.AsyncMock, run_metadata: RunMetadataDict, computational_pipeline_rabbit_client_parser: mock.AsyncMock, + fake_collection_run_id: CollectionRunID, ): _with_mock_send_computation_tasks(published_project.tasks, mocked_dask_client) run_in_db, expected_published_tasks = await _assert_start_pipeline( @@ -1813,6 +1827,7 @@ async def test_running_pipeline_triggers_heartbeat( published_project=published_project, run_metadata=run_metadata, computational_pipeline_rabbit_client_parser=computational_pipeline_rabbit_client_parser, + collection_run_id=fake_collection_run_id, ) # ------------------------------------------------------------------------------- # 1. first run will move comp_tasks to PENDING so the dask-worker can take them @@ -1922,6 +1937,7 @@ async def test_pipeline_with_on_demand_cluster_with_not_ready_backend_waits( mocked_get_or_create_cluster: mock.Mock, faker: Faker, computational_pipeline_rabbit_client_parser: mock.AsyncMock, + fake_collection_run_id: CollectionRunID, ): mocked_get_or_create_cluster.side_effect = ( ComputationalBackendOnDemandNotReadyError( @@ -1936,6 +1952,7 @@ async def test_pipeline_with_on_demand_cluster_with_not_ready_backend_waits( project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=True, + collection_run_id=fake_collection_run_id, ) # we ask to use an on-demand cluster, therefore the tasks are published first @@ -2042,6 +2059,7 @@ async def test_pipeline_with_on_demand_cluster_with_no_clusters_keeper_fails( mocked_get_or_create_cluster: mock.Mock, get_or_create_exception: Exception, computational_pipeline_rabbit_client_parser: mock.AsyncMock, + fake_collection_run_id: CollectionRunID, ): # needs to change: https://github.com/ITISFoundation/osparc-simcore/issues/6817 @@ -2054,6 +2072,7 @@ async def test_pipeline_with_on_demand_cluster_with_no_clusters_keeper_fails( project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=True, + collection_run_id=fake_collection_run_id, ) # we ask to use an on-demand cluster, therefore the tasks are published first @@ -2155,6 +2174,7 @@ async def test_run_new_pipeline_called_twice_prevents_duplicate_runs( published_project: PublishedProject, run_metadata: RunMetadataDict, computational_pipeline_rabbit_client_parser: mock.AsyncMock, + fake_collection_run_id: CollectionRunID, ): # Ensure we start with an empty database await assert_comp_runs_empty(sqlalchemy_async_engine) @@ -2167,6 +2187,7 @@ async def test_run_new_pipeline_called_twice_prevents_duplicate_runs( project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) # Verify first run was created and published @@ -2195,6 +2216,7 @@ async def test_run_new_pipeline_called_twice_prevents_duplicate_runs( project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) # Verify still only one run exists with same run_id diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_worker.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_worker.py index 8e1a6fd8424..5c7354e093e 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_worker.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_worker.py @@ -15,6 +15,7 @@ import pytest from _helpers import PublishedProject from fastapi import FastAPI +from models_library.computations import CollectionRunID from pytest_mock import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -62,6 +63,7 @@ async def test_worker_properly_autocalls_scheduler_api( mocked_get_scheduler_worker: mock.Mock, published_project: PublishedProject, run_metadata: RunMetadataDict, + fake_collection_run_id: CollectionRunID, ): assert published_project.project.prj_owner await run_new_pipeline( @@ -70,6 +72,7 @@ async def test_worker_properly_autocalls_scheduler_api( project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) mocked_get_scheduler_worker.assert_called_once_with(initialized_app) mocked_get_scheduler_worker.return_value.apply.assert_called_once_with( @@ -110,6 +113,7 @@ async def test_worker_scheduling_parallelism( initialized_app: FastAPI, publish_project: Callable[[], Awaitable[PublishedProject]], run_metadata: RunMetadataDict, + fake_collection_run_id: CollectionRunID, with_product: dict[str, Any], ): with_disabled_auto_scheduling.assert_called_once() @@ -128,6 +132,7 @@ async def _project_pipeline_creation_workflow() -> None: project_id=published_project.project.uuid, run_metadata=run_metadata, use_on_demand_clusters=False, + collection_run_id=fake_collection_run_id, ) # whatever scheduling concurrency we call in here, we shall always see the same number of calls to the scheduler diff --git a/services/director-v2/tests/unit/with_dbs/conftest.py b/services/director-v2/tests/unit/with_dbs/conftest.py index 8b09138d651..22209ec67c5 100644 --- a/services/director-v2/tests/unit/with_dbs/conftest.py +++ b/services/director-v2/tests/unit/with_dbs/conftest.py @@ -16,6 +16,7 @@ from dask_task_models_library.container_tasks.utils import generate_dask_job_id from faker import Faker from fastapi.encoders import jsonable_encoder +from models_library.computations import CollectionRunID from models_library.projects import ProjectAtDB, ProjectID from models_library.projects_nodes_io import NodeID from pydantic import PositiveInt @@ -144,9 +145,16 @@ def run_metadata( ) +@pytest.fixture +def fake_collection_run_id(faker: Faker) -> CollectionRunID: + return CollectionRunID(f"{faker.uuid4(cast_to=None)}") + + @pytest.fixture async def create_comp_run( - sqlalchemy_async_engine: AsyncEngine, run_metadata: RunMetadataDict + sqlalchemy_async_engine: AsyncEngine, + run_metadata: RunMetadataDict, + faker: Faker, ) -> AsyncIterator[Callable[..., Awaitable[CompRunsAtDB]]]: created_run_ids: list[int] = [] @@ -161,6 +169,7 @@ async def _( "metadata": jsonable_encoder(run_metadata), "use_on_demand_clusters": False, "dag_adjacency_list": {}, + "collection_run_id": faker.uuid4(), } run_config.update(**run_kwargs) async with sqlalchemy_async_engine.begin() as conn: @@ -244,6 +253,7 @@ async def _( project_id=project.uuid, node_id=NodeID(node_id), ), + "state": StateType.PUBLISHED.value, } task_config.update(**overrides_kwargs) async with sqlalchemy_async_engine.begin() as conn: diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index a28fa1d7276..185929e596a 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2776,6 +2776,87 @@ paths: application/json: schema: $ref: '#/components/schemas/Page_ComputationTaskRestGet_' + /v0/computation-collection-runs: + get: + tags: + - computations + - projects + summary: List Computation Collection Runs + operationId: list_computation_collection_runs + parameters: + - name: limit + in: query + required: false + schema: + type: integer + default: 20 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + default: 0 + title: Offset + - name: filter_only_running + in: query + required: false + schema: + type: boolean + default: false + title: Filter Only Running + - name: filter_by_root_project_id + in: query + required: false + schema: + anyOf: + - type: string + format: uuid + - type: 'null' + title: Filter By Root Project Id + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Page_ComputationTaskRestGet_' + /v0/computation-collection-runs/{collection_run_id}/tasks: + get: + tags: + - computations + - projects + summary: List Computation Collection Run Tasks + operationId: list_computation_collection_run_tasks + parameters: + - name: collection_run_id + in: path + required: true + schema: + type: string + format: uuid + title: Collection Run Id + - name: limit + in: query + required: false + schema: + type: integer + default: 20 + title: Limit + - name: offset + in: query + required: false + schema: + type: integer + default: 0 + title: Offset + responses: + '200': + description: Successful Response + content: + application/json: + schema: + $ref: '#/components/schemas/Page_ComputationTaskRestGet_' /v0/projects/{project_id}:xport: post: tags: @@ -15855,20 +15936,32 @@ components: type: string enum: - UNKNOWN - - PUBLISHED - NOT_STARTED + - PUBLISHED - PENDING + - WAITING_FOR_CLUSTER - WAITING_FOR_RESOURCES - STARTED - SUCCESS - FAILED - ABORTED - - WAITING_FOR_CLUSTER title: RunningState - description: 'State of execution of a project''s computational workflow - - - SEE StateType for task state' + description: "State of execution of a project's computational workflow\n\nSEE\ + \ StateType for task state\n\n# Computational backend states explained:\n\ + - UNKNOWN - The backend doesn't know about the task anymore, it has disappeared\ + \ from the system or it was never created (eg. when we are asking for the\ + \ task)\n- NOT_STARTED - Default state when the task is created\n- PUBLISHED\ + \ - The task has been submitted to the computational backend (click on \"\ + Run\" button in the UI)\n- PENDING - Task has been transferred to the Dask\ + \ scheduler and is waiting for a worker to pick it up (director-v2 --> Dask\ + \ scheduler)\n - But! it is also transition state (ex. PENDING -> WAITING_FOR_CLUSTER\ + \ -> PENDING -> WAITING_FOR_RESOURCES -> PENDING -> STARTED)\n- WAITING_FOR_CLUSTER\ + \ - No cluster (Dask scheduler) is available to run the task; waiting for\ + \ one to become available\n- WAITING_FOR_RESOURCES - No worker (Dask worker)\ + \ is available to run the task; waiting for one to become available\n- STARTED\ + \ - A worker has picked up the task and is executing it\n- SUCCESS - Task\ + \ finished successfully\n- FAILED - Task finished with an error\n- ABORTED\ + \ - Task was aborted before completion" SelectBox: properties: structure: diff --git a/services/web/server/src/simcore_service_webserver/db/plugin.py b/services/web/server/src/simcore_service_webserver/db/plugin.py index a01efdbc40c..d99e7f90658 100644 --- a/services/web/server/src/simcore_service_webserver/db/plugin.py +++ b/services/web/server/src/simcore_service_webserver/db/plugin.py @@ -1,6 +1,4 @@ -""" database submodule associated to the postgres uservice - -""" +"""database submodule associated to the postgres uservice""" import logging @@ -14,7 +12,7 @@ # API -get_database_engine = _aiopg.get_database_engine +get_database_engine_legacy = _aiopg.get_database_engine get_engine_state = _aiopg.get_engine_state is_service_responsive = _aiopg.is_service_responsive is_service_enabled = _aiopg.is_service_enabled @@ -34,7 +32,7 @@ def setup_db(app: web.Application): # ensures keys exist app[APP_AIOPG_ENGINE_KEY] = None - assert get_database_engine(app) is None # nosec + assert get_database_engine_legacy(app) is None # nosec # init engines app.cleanup_ctx.append(_aiopg.postgres_cleanup_ctx) diff --git a/services/web/server/src/simcore_service_webserver/db_listener/_db_comp_tasks_listening_task.py b/services/web/server/src/simcore_service_webserver/db_listener/_db_comp_tasks_listening_task.py index 2c72213b094..629bfaa95a0 100644 --- a/services/web/server/src/simcore_service_webserver/db_listener/_db_comp_tasks_listening_task.py +++ b/services/web/server/src/simcore_service_webserver/db_listener/_db_comp_tasks_listening_task.py @@ -22,7 +22,7 @@ from simcore_postgres_database.webserver_models import DB_CHANNEL_NAME, projects from sqlalchemy.sql import select -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from ..projects import _projects_service, exceptions from ..projects.nodes_utils import update_node_outputs from ._models import CompTaskNotificationPayload @@ -126,7 +126,7 @@ async def _handle_db_notification( async def _listen(app: web.Application) -> NoReturn: listen_query = f"LISTEN {DB_CHANNEL_NAME};" - db_engine = get_database_engine(app) + db_engine = get_database_engine_legacy(app) async with db_engine.acquire() as conn: assert conn.connection # nosec await conn.execute(listen_query) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_comp_runs_collections_models.py b/services/web/server/src/simcore_service_webserver/director_v2/_comp_runs_collections_models.py new file mode 100644 index 00000000000..7812025920c --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/director_v2/_comp_runs_collections_models.py @@ -0,0 +1,17 @@ +import logging +from datetime import datetime + +from models_library.computations import CollectionRunID +from pydantic import BaseModel, ConfigDict + +_logger = logging.getLogger(__name__) + + +class CompRunCollectionDBGet(BaseModel): + collection_run_id: CollectionRunID + client_or_system_generated_id: str + client_or_system_generated_display_name: str + is_generated_by_system: bool + created: datetime + + model_config = ConfigDict(from_attributes=True) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_comp_runs_collections_repository.py b/services/web/server/src/simcore_service_webserver/director_v2/_comp_runs_collections_repository.py new file mode 100644 index 00000000000..22eee161826 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/director_v2/_comp_runs_collections_repository.py @@ -0,0 +1,65 @@ +import logging +from uuid import UUID + +from models_library.computations import CollectionRunID +from pydantic import TypeAdapter +from simcore_postgres_database.models.comp_runs_collections import comp_runs_collections +from sqlalchemy import func + +from ._comp_runs_collections_models import CompRunCollectionDBGet + +_logger = logging.getLogger(__name__) + + +# Comp run collections CRUD operations + + +async def create_comp_run_collection( + conn, + client_or_system_generated_id: str, + client_or_system_generated_display_name: str, + is_generated_by_system: bool, +) -> CollectionRunID: + """Create a new computational run collection.""" + result = await conn.execute( + comp_runs_collections.insert() + .values( + client_or_system_generated_id=client_or_system_generated_id, + client_or_system_generated_display_name=client_or_system_generated_display_name, + is_generated_by_system=is_generated_by_system, + created=func.now(), + modified=func.now(), + ) + .returning(comp_runs_collections.c.collection_run_id) + ) + collection_id_tuple: tuple[UUID] = result.one() + return TypeAdapter(CollectionRunID).validate_python(collection_id_tuple[0]) + + +async def get_comp_run_collection_or_none_by_id( + conn, collection_run_id: CollectionRunID +) -> CompRunCollectionDBGet | None: + result = await conn.execute( + comp_runs_collections.select().where( + comp_runs_collections.c.collection_run_id == f"{collection_run_id}" + ) + ) + row = result.one_or_none() + if row is None: + return None + return CompRunCollectionDBGet.model_validate(row) + + +async def get_comp_run_collection_or_none_by_client_generated_id( + conn, client_or_system_generated_id: str +) -> CompRunCollectionDBGet | None: + result = await conn.execute( + comp_runs_collections.select().where( + comp_runs_collections.c.client_or_system_generated_id + == client_or_system_generated_id + ) + ) + row = result.one_or_none() + if row is None: + return None + return CompRunCollectionDBGet.model_validate(row) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_comp_runs_collections_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_comp_runs_collections_service.py new file mode 100644 index 00000000000..0bf82c01f7f --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/director_v2/_comp_runs_collections_service.py @@ -0,0 +1,47 @@ +import logging + +from aiohttp import web +from models_library.computations import CollectionRunID +from simcore_postgres_database.utils_repos import transaction_context + +from ..db.plugin import get_asyncpg_engine +from . import _comp_runs_collections_repository +from ._comp_runs_collections_models import CompRunCollectionDBGet + +_logger = logging.getLogger(__name__) + + +async def create_comp_run_collection( + app: web.Application, + *, + client_or_system_generated_id: str, + client_or_system_generated_display_name: str, + is_generated_by_system: bool, +) -> CollectionRunID: + async with transaction_context(get_asyncpg_engine(app)) as conn: + return await _comp_runs_collections_repository.create_comp_run_collection( + conn=conn, + client_or_system_generated_id=client_or_system_generated_id, + client_or_system_generated_display_name=client_or_system_generated_display_name, + is_generated_by_system=is_generated_by_system, + ) + + +async def get_comp_run_collection_or_none_by_id( + app: web.Application, *, collection_run_id: CollectionRunID +) -> CompRunCollectionDBGet | None: + async with transaction_context(get_asyncpg_engine(app)) as conn: + return await _comp_runs_collections_repository.get_comp_run_collection_or_none_by_id( + conn=conn, collection_run_id=collection_run_id + ) + + +async def get_comp_run_collection_or_none_by_client_generated_id( + app: web.Application, + *, + client_or_system_generated_id: str, +) -> CompRunCollectionDBGet | None: + async with transaction_context(get_asyncpg_engine(app)) as conn: + return await _comp_runs_collections_repository.get_comp_run_collection_or_none_by_client_generated_id( + conn=conn, client_or_system_generated_id=client_or_system_generated_id + ) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py index 22667ded1c8..2068a97fe9b 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py @@ -1,8 +1,14 @@ from decimal import Decimal from aiohttp import web -from models_library.api_schemas_directorv2.comp_runs import ComputationRunRpcGet +from models_library.api_schemas_directorv2.comp_runs import ( + ComputationCollectionRunRpcGet, + ComputationRunRpcGet, +) from models_library.computations import ( + CollectionRunID, + ComputationCollectionRunTaskWithAttributes, + ComputationCollectionRunWithAttributes, ComputationRunWithAttributes, ComputationTaskWithAttributes, ) @@ -35,6 +41,7 @@ get_project_uuids_by_root_parent_project_id, ) from ..rabbitmq import get_rabbitmq_rpc_client +from ._comp_runs_collections_service import get_comp_run_collection_or_none_by_id async def _get_projects_metadata( @@ -289,3 +296,175 @@ async def list_computations_latest_iteration_tasks( ) ] return _tasks_get.total, _tasks_get_output + + +async def _get_root_project_names_v2( + app: web.Application, items: list[ComputationCollectionRunRpcGet] +) -> list[str]: + root_uuids: list[ProjectID] = [] + for item in items: + if root_id := item.info.get("project_metadata", {}).get( + "root_parent_project_id" + ): + root_uuids.append(ProjectID(root_id)) + else: + assert len(item.project_ids) > 0 # nosec + root_uuids.append(ProjectID(item.project_ids[0])) + + return await batch_get_project_name(app, projects_uuids=root_uuids) + + +async def list_computation_collection_runs( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + # filters + filter_by_root_project_id: ProjectID | None = None, + # pagination + offset: int, + limit: NonNegativeInt, +) -> tuple[int, list[ComputationCollectionRunWithAttributes]]: + child_projects_with_root = None + if filter_by_root_project_id: + await check_user_project_permission( + app, + project_id=filter_by_root_project_id, + user_id=user_id, + product_name=product_name, + ) + # NOTE: Can be improved with checking if the provided project is a root project + child_projects = await get_project_uuids_by_root_parent_project_id( + app, root_parent_project_uuid=filter_by_root_project_id + ) + child_projects_with_root = [*child_projects, filter_by_root_project_id] + + rpc_client = get_rabbitmq_rpc_client(app) + _runs_get = await computations.list_computation_collection_runs_page( + rpc_client, + product_name=product_name, + user_id=user_id, + project_ids=child_projects_with_root, + offset=offset, + limit=limit, + ) + + # NOTE: MD: can be improved with a single batch call + _comp_runs_collections = await limited_gather( + *[ + get_comp_run_collection_or_none_by_id( + app, collection_run_id=_run.collection_run_id + ) + for _run in _runs_get.items + ], + limit=20, + ) + # Get Root project names + _projects_root_names = await _get_root_project_names_v2(app, _runs_get.items) + + _computational_runs_output = [ + ComputationCollectionRunWithAttributes( + collection_run_id=item.collection_run_id, + project_ids=item.project_ids, + state=item.state, + info=item.info, + submitted_at=item.submitted_at, + started_at=item.started_at, + ended_at=item.ended_at, + name=( + run_collection.client_or_system_generated_display_name + if run_collection and run_collection.is_generated_by_system is False + else project_root_name + ), + ) + for item, run_collection, project_root_name in zip( + _runs_get.items, _comp_runs_collections, _projects_root_names, strict=True + ) + ] + + return _runs_get.total, _computational_runs_output + + +async def list_computation_collection_run_tasks( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + collection_run_id: CollectionRunID, + # pagination + offset: int, + limit: NonNegativeInt, +) -> tuple[int, list[ComputationCollectionRunTaskWithAttributes]]: + rpc_client = get_rabbitmq_rpc_client(app) + _tasks_get = await computations.list_computation_collection_run_tasks_page( + rpc_client, + product_name=product_name, + user_id=user_id, + collection_run_id=collection_run_id, + offset=offset, + limit=limit, + ) + + # Get unique set of all project_uuids from comp_tasks + unique_project_uuids = {task.project_uuid for task in _tasks_get.items} + # NOTE: MD: can be improved with a single batch call + project_dicts = await limited_gather( + *[ + get_project_dict_legacy(app, project_uuid=project_uuid) + for project_uuid in unique_project_uuids + ], + limit=20, + ) + # Build a dict: project_uuid -> workbench + project_uuid_to_workbench = {prj["uuid"]: prj["workbench"] for prj in project_dicts} + + # Fetch projects metadata concurrently + _projects_metadata = await _get_projects_metadata( + app, project_uuids=[item.project_uuid for item in _tasks_get.items] + ) + + _service_run_ids = [item.service_run_id for item in _tasks_get.items] + _is_product_billable = await is_product_billable(app, product_name=product_name) + _service_run_osparc_credits: list[Decimal | None] + if _is_product_billable: + # NOTE: MD: can be improved with a single batch call + _service_run_osparc_credits = await limited_gather( + *[ + _get_credits_or_zero_by_service_run_id( + rpc_client, service_run_id=_run_id + ) + for _run_id in _service_run_ids + ], + limit=20, + ) + else: + _service_run_osparc_credits = [None for _ in _service_run_ids] + + # Final output + _tasks_get_output = [ + ComputationCollectionRunTaskWithAttributes( + project_uuid=item.project_uuid, + node_id=item.node_id, + state=item.state, + progress=item.progress, + image=item.image, + started_at=item.started_at, + ended_at=item.ended_at, + log_download_link=item.log_download_link, + name=( + custom_metadata.get("job_name") + or project_uuid_to_workbench[f"{item.project_uuid}"][ + f"{item.node_id}" + ].get("label") + or "Unknown" + ), + osparc_credits=credits_or_none, + ) + for item, credits_or_none, custom_metadata in zip( + _tasks_get.items, + _service_run_osparc_credits, + _projects_metadata, + strict=True, + ) + ] + return _tasks_get.total, _tasks_get_output diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py b/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py index 96766fe97af..98c3ef73ca5 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py @@ -2,6 +2,11 @@ from aiohttp import web from models_library.api_schemas_webserver.computations import ( + ComputationCollectionRunListQueryParams, + ComputationCollectionRunPathParams, + ComputationCollectionRunRestGet, + ComputationCollectionRunTaskListQueryParams, + ComputationCollectionRunTaskRestGet, ComputationRunIterationsLatestListQueryParams, ComputationRunIterationsListQueryParams, ComputationRunPathParams, @@ -187,3 +192,105 @@ async def list_computations_latest_iteration_tasks( text=page.model_dump_json(**RESPONSE_MODEL_POLICY), content_type=MIMETYPE_APPLICATION_JSON, ) + + +#### NEW: + + +@routes.get( + f"/{VTAG}/computation-collection-runs", + name="list_computation_collection_runs", +) +@login_required +@permission_required("services.pipeline.*") +@permission_required("project.read") +async def list_computation_collection_runs(request: web.Request) -> web.Response: + req_ctx = ComputationsRequestContext.model_validate(request) + query_params: ComputationCollectionRunListQueryParams = ( + parse_request_query_parameters_as( + ComputationCollectionRunListQueryParams, request + ) + ) + + if query_params.filter_only_running is True: + raise NotImplementedError + + total, items = await _computations_service.list_computation_collection_runs( + request.app, + product_name=req_ctx.product_name, + user_id=req_ctx.user_id, + # filters + filter_by_root_project_id=query_params.filter_by_root_project_id, + # pagination + offset=query_params.offset, + limit=query_params.limit, + ) + + page = Page[ComputationCollectionRunRestGet].model_validate( + paginate_data( + chunk=[ + ComputationCollectionRunRestGet.model_validate( + run, from_attributes=True + ) + for run in items + ], + total=total, + limit=query_params.limit, + offset=query_params.offset, + request_url=request.url, + ) + ) + + return web.Response( + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), + content_type=MIMETYPE_APPLICATION_JSON, + ) + + +@routes.get( + f"/{VTAG}/computation-collection-runs/{{collection_run_id}}/tasks", + name="list_computation_collection_run_tasks", +) +@login_required +@permission_required("services.pipeline.*") +@permission_required("project.read") +async def list_computation_collection_run_tasks(request: web.Request) -> web.Response: + req_ctx = ComputationsRequestContext.model_validate(request) + path_params = parse_request_path_parameters_as( + ComputationCollectionRunPathParams, request + ) + query_params: ComputationCollectionRunTaskListQueryParams = ( + parse_request_query_parameters_as( + ComputationCollectionRunTaskListQueryParams, request + ) + ) + + total, items = await _computations_service.list_computation_collection_run_tasks( + request.app, + product_name=req_ctx.product_name, + user_id=req_ctx.user_id, + collection_run_id=path_params.collection_run_id, + # pagination + offset=query_params.offset, + limit=query_params.limit, + ) + + page = Page[ComputationCollectionRunTaskRestGet].model_validate( + paginate_data( + chunk=[ + ComputationCollectionRunTaskRestGet.model_validate( + run, from_attributes=True + ) + for run in items + ], + total=total, + limit=query_params.limit, + offset=query_params.offset, + request_url=request.url, + ) + ) + + return web.Response( + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), + content_type=MIMETYPE_APPLICATION_JSON, + ) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_controller/rest.py b/services/web/server/src/simcore_service_webserver/director_v2/_controller/rest.py index b0ad4b5cd07..990c958ba67 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_controller/rest.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_controller/rest.py @@ -1,5 +1,7 @@ import asyncio import logging +import uuid +from datetime import UTC, datetime, timedelta from typing import Any from aiohttp import web @@ -25,10 +27,14 @@ from ...login.decorators import login_required from ...models import AuthenticatedRequestContext from ...products import products_web +from ...projects.projects_metadata_service import ( + get_project_custom_metadata_or_empty_dict, +) from ...security.decorators import permission_required from ...utils_aiohttp import envelope_json_response, get_api_base_url -from .. import _director_v2_service +from .. import _comp_runs_collections_service, _director_v2_service from .._client import DirectorV2RestClient +from .._comp_runs_collections_models import CompRunCollectionDBGet from .._director_v2_abc_service import get_project_run_policy from ._rest_exceptions import handle_rest_requests_exceptions @@ -72,6 +78,46 @@ async def start_computation(request: web.Request) -> web.Response: product_name=req_ctx.product_name, ) + # Get Project custom metadata information + # inject the collection_id to the options + custom_metadata = await get_project_custom_metadata_or_empty_dict( + request.app, project_uuid=path_params.project_id + ) + group_id_or_none = custom_metadata.get("group_id") + + comp_run_collection: CompRunCollectionDBGet | None = None + if group_id_or_none: + comp_run_collection = await _comp_runs_collections_service.get_comp_run_collection_or_none_by_client_generated_id( + request.app, client_or_system_generated_id=group_id_or_none # type: ignore + ) + if comp_run_collection is not None: + created_at: datetime = comp_run_collection.created + now = datetime.now(UTC) + if now - created_at > timedelta(minutes=5): + raise web.HTTPBadRequest( + reason=( + "This client generated collection is not new, " + "it was created more than 5 minutes ago. " + "Therefore, the client is probably wrongly generating it." + ) + ) + is_generated_by_system = False + if group_id_or_none in {None, "", "00000000-0000-0000-0000-000000000000"}: + is_generated_by_system = True + client_or_system_generated_id = ( + f"system-generated/{path_params.project_id}/{uuid.uuid4()}" + ) + else: + client_or_system_generated_id = f"{group_id_or_none}" + group_name = custom_metadata.get("group_name", "No Group Name") + + collection_run_id = await _comp_runs_collections_service.create_comp_run_collection( + request.app, + client_or_system_generated_id=client_or_system_generated_id, + client_or_system_generated_display_name=group_name, # type: ignore + is_generated_by_system=is_generated_by_system, + ) + options = { "start_pipeline": True, "subgraph": list(subgraph), # sets are not natively json serializable @@ -79,6 +125,7 @@ async def start_computation(request: web.Request) -> web.Response: "simcore_user_agent": simcore_user_agent, "use_on_demand_clusters": group_properties.use_on_demand_clusters, "wallet_info": wallet_info, + "collection_run_id": collection_run_id, } run_policy = get_project_run_policy(request.app) diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py index 2bbc971f887..b52be5f1554 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_director_v2_service.py @@ -24,7 +24,7 @@ ) from ..application_settings import get_application_settings -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from ..products import products_service from ..products.models import Product from ..projects import projects_wallets_service @@ -277,7 +277,7 @@ async def get_group_properties( product_name: ProductName, user_id: UserID, ) -> GroupExtraProperties: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: return await GroupExtraPropertiesRepo.get_aggregated_properties_for_user( conn, user_id=user_id, product_name=product_name ) diff --git a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py index 2c06edb571f..481635b05f4 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py +++ b/services/web/server/src/simcore_service_webserver/groups/_classifiers_service.py @@ -26,7 +26,7 @@ from servicelib.logging_errors import create_troubleshootting_log_kwargs from simcore_postgres_database.models.classifiers import group_classifiers -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from ..scicrunch.db import ResearchResourceRepository from ..scicrunch.service_client import SciCrunch @@ -78,7 +78,7 @@ class Classifiers(BaseModel): class GroupClassifierRepository: def __init__(self, app: web.Application): - self.engine = get_database_engine(app) + self.engine = get_database_engine_legacy(app) async def _get_bundle(self, gid: int) -> RowProxy | None: async with self.engine.acquire() as conn: diff --git a/services/web/server/src/simcore_service_webserver/payments/_autorecharge_db.py b/services/web/server/src/simcore_service_webserver/payments/_autorecharge_db.py index 813fa6b9eb1..b3ac90985f2 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_autorecharge_db.py +++ b/services/web/server/src/simcore_service_webserver/payments/_autorecharge_db.py @@ -9,7 +9,7 @@ from pydantic import BaseModel, ConfigDict, PositiveInt from simcore_postgres_database.utils_payments_autorecharge import AutoRechargeStmts -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from .errors import InvalidPaymentMethodError _logger = logging.getLogger(__name__) @@ -32,7 +32,7 @@ async def get_wallet_autorecharge( *, wallet_id: WalletID, ) -> PaymentsAutorechargeDB | None: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: stmt = AutoRechargeStmts.get_wallet_autorecharge(wallet_id) result = await conn.execute(stmt) row = await result.first() @@ -51,7 +51,7 @@ async def replace_wallet_autorecharge( InvalidPaymentMethodError: if `new` includes some invalid 'primary_payment_method_id' """ - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: stmt = AutoRechargeStmts.is_valid_payment_method( user_id=user_id, wallet_id=new.wallet_id, diff --git a/services/web/server/src/simcore_service_webserver/payments/_methods_db.py b/services/web/server/src/simcore_service_webserver/payments/_methods_db.py index 135eaf41a9e..745c0e7caac 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_methods_db.py +++ b/services/web/server/src/simcore_service_webserver/payments/_methods_db.py @@ -16,7 +16,7 @@ from sqlalchemy import literal_column from sqlalchemy.sql import func -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from .errors import ( PaymentMethodAlreadyAckedError, PaymentMethodNotFoundError, @@ -46,7 +46,7 @@ async def insert_init_payment_method( wallet_id: WalletID, initiated_at: datetime.datetime, ) -> None: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: try: await conn.execute( payments_methods.insert().values( @@ -68,7 +68,7 @@ async def list_successful_payment_methods( user_id: UserID, wallet_id: WalletID, ) -> list[PaymentsMethodsDB]: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result: ResultProxy = await conn.execute( payments_methods.select() .where( @@ -89,7 +89,7 @@ async def get_successful_payment_method( wallet_id: WalletID, payment_method_id: PaymentMethodID, ) -> PaymentsMethodsDB: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result: ResultProxy = await conn.execute( payments_methods.select().where( (payments_methods.c.user_id == user_id) @@ -108,7 +108,7 @@ async def get_successful_payment_method( async def get_pending_payment_methods_ids( app: web.Application, ) -> list[PaymentMethodID]: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute( sa.select(payments_methods.c.payment_method_id) .where(payments_methods.c.completed_at.is_(None)) @@ -142,7 +142,7 @@ async def udpate_payment_method( if state_message: optional["state_message"] = state_message - async with get_database_engine(app).acquire() as conn, conn.begin(): + async with get_database_engine_legacy(app).acquire() as conn, conn.begin(): row = await ( await conn.execute( sa.select( @@ -179,7 +179,7 @@ async def delete_payment_method( wallet_id: WalletID, payment_method_id: PaymentMethodID, ): - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: await conn.execute( payments_methods.delete().where( (payments_methods.c.user_id == user_id) diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py index ef74187c5df..e2fda301f98 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_api.py @@ -24,7 +24,7 @@ from simcore_postgres_database.utils_payments import insert_init_payment_transaction from yarl import URL -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from ..products import products_service from ..resource_usage.service import add_credits_to_wallet from ..users.api import get_user_display_and_id_names, get_user_invoice_address @@ -91,7 +91,7 @@ async def _fake_init_payment( .with_query(id=payment_id) ) # (2) Annotate INIT transaction - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: await insert_init_payment_transaction( conn, payment_id=payment_id, diff --git a/services/web/server/src/simcore_service_webserver/payments/_onetime_db.py b/services/web/server/src/simcore_service_webserver/payments/_onetime_db.py index d6146cd0f81..d6014509461 100644 --- a/services/web/server/src/simcore_service_webserver/payments/_onetime_db.py +++ b/services/web/server/src/simcore_service_webserver/payments/_onetime_db.py @@ -21,7 +21,7 @@ update_payment_transaction_state, ) -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from .errors import PaymentCompletedError, PaymentNotFoundError _logger = logging.getLogger(__name__) @@ -58,7 +58,7 @@ async def list_user_payment_transactions( Sorted by newest-first """ - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: total_number_of_items, rows = await get_user_payments_transactions( conn, user_id=user_id, offset=offset, limit=limit ) @@ -67,7 +67,7 @@ async def list_user_payment_transactions( async def get_pending_payment_transactions_ids(app: web.Application) -> list[PaymentID]: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute( sa.select(payments_transactions.c.payment_id) .where(payments_transactions.c.completed_at == None) # noqa: E711 @@ -95,7 +95,7 @@ async def complete_payment_transaction( if invoice_url: optional_kwargs["invoice_url"] = invoice_url - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: row = await update_payment_transaction_state( conn, payment_id=payment_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_access_rights_service.py b/services/web/server/src/simcore_service_webserver/projects/_access_rights_service.py index 7007dc72e33..ddc790d5ece 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_access_rights_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_access_rights_service.py @@ -3,7 +3,7 @@ from models_library.projects import ProjectID from models_library.users import UserID -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from ..workspaces.api import get_workspace from ._access_rights_repository import get_project_owner from ._projects_repository_legacy import APP_PROJECT_DBAPI, ProjectDBAPI @@ -20,7 +20,9 @@ async def validate_project_ownership( ProjectInvalidRightsError: if 'user_id' does not own 'project_uuid' """ if ( - await get_project_owner(get_database_engine(app), project_uuid=project_uuid) + await get_project_owner( + get_database_engine_legacy(app), project_uuid=project_uuid + ) != user_id ): raise ProjectInvalidRightsError(user_id=user_id, project_uuid=project_uuid) diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_repository.py b/services/web/server/src/simcore_service_webserver/projects/_folders_repository.py index 6c1755eb245..8270bdcb7ba 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_repository.py @@ -13,7 +13,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection from sqlalchemy.sql import select -from ..db.plugin import get_asyncpg_engine, get_database_engine +from ..db.plugin import get_asyncpg_engine, get_database_engine_legacy _logger = logging.getLogger(__name__) @@ -38,7 +38,7 @@ async def insert_project_to_folder( folder_id: FolderID, private_workspace_user_id_or_none: UserID | None, ) -> ProjectToFolderDB: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute( projects_to_folders.insert() .values( @@ -71,7 +71,7 @@ async def get_project_to_folder( & (projects_to_folders.c.user_id == private_workspace_user_id_or_none) ) - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute(stmt) row = await result.first() if row is None: @@ -85,7 +85,7 @@ async def delete_project_to_folder( folder_id: FolderID, private_workspace_user_id_or_none: UserID | None, ) -> None: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: await conn.execute( projects_to_folders.delete().where( (projects_to_folders.c.project_uuid == f"{project_id}") diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_repository.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_repository.py index d943e72e6f9..2def2375f0f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_repository.py @@ -16,6 +16,7 @@ DBProjectInvalidParentNodeError, DBProjectInvalidParentProjectError, DBProjectNotFoundError, + ProjectMetadata, ) from simcore_postgres_database.utils_projects_nodes import ( ProjectNodesNodeNotFoundError, @@ -95,6 +96,19 @@ async def get_project_custom_metadata( return TypeAdapter(MetadataDict).validate_python(metadata.custom or {}) +@_handle_projects_metadata_exceptions +async def get_project_metadata_or_none( + engine: Engine, project_uuid: ProjectID +) -> ProjectMetadata | None: + async with engine.acquire() as connection: + try: + return await utils_projects_metadata.get( + connection, project_uuid=project_uuid + ) + except DBProjectNotFoundError: + return None + + @_handle_projects_metadata_exceptions async def set_project_custom_metadata( engine: Engine, diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py index 3d77fa6a74f..0620f9c58e0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py @@ -8,7 +8,7 @@ from models_library.users import UserID from pydantic import TypeAdapter -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from . import _metadata_repository from ._access_rights_service import validate_project_ownership from .exceptions import ProjectNotFoundError @@ -23,7 +23,7 @@ async def get_project_custom_metadata_for_user( await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid) return await _metadata_repository.get_project_custom_metadata( - engine=get_database_engine(app), project_uuid=project_uuid + engine=get_database_engine_legacy(app), project_uuid=project_uuid ) @@ -32,7 +32,7 @@ async def get_project_custom_metadata_or_empty_dict( ) -> MetadataDict: try: output = await _metadata_repository.get_project_custom_metadata( - engine=get_database_engine(app), project_uuid=project_uuid + engine=get_database_engine_legacy(app), project_uuid=project_uuid ) except ProjectNotFoundError: # This is a valid case when the project is not found @@ -50,7 +50,7 @@ async def set_project_custom_metadata( await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid) return await _metadata_repository.set_project_custom_metadata( - engine=get_database_engine(app), + engine=get_database_engine_legacy(app), project_uuid=project_uuid, custom_metadata=value, ) @@ -65,7 +65,7 @@ async def _project_has_ancestors( await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid) return await _metadata_repository.project_has_ancestors( - engine=get_database_engine(app), project_uuid=project_uuid + engine=get_database_engine_legacy(app), project_uuid=project_uuid ) @@ -90,11 +90,11 @@ async def set_project_ancestors_from_custom_metadata( # let's try to get the parent project UUID parent_project_uuid = await _metadata_repository.get_project_id_from_node_id( - get_database_engine(app), node_id=parent_node_id + get_database_engine_legacy(app), node_id=parent_node_id ) await _metadata_repository.set_project_ancestors( - get_database_engine(app), + get_database_engine_legacy(app), project_uuid=project_uuid, parent_project_uuid=parent_project_uuid, parent_node_id=parent_node_id, @@ -111,7 +111,7 @@ async def set_project_ancestors( await validate_project_ownership(app, user_id=user_id, project_uuid=project_uuid) await _metadata_repository.set_project_ancestors( - get_database_engine(app), + get_database_engine_legacy(app), project_uuid=project_uuid, parent_project_uuid=parent_project_uuid, parent_node_id=parent_node_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py index e5060360265..f55d22fac82 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_repository.py @@ -3,7 +3,7 @@ from models_library.services_types import ServiceKey, ServiceVersion from simcore_postgres_database.utils_projects_nodes import ProjectNodesRepo -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy async def get_project_nodes_services( @@ -11,7 +11,7 @@ async def get_project_nodes_services( ) -> list[tuple[ServiceKey, ServiceVersion]]: repo = ProjectNodesRepo(project_uuid=project_uuid) - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: nodes = await repo.list(conn) # removes duplicates by preserving order diff --git a/services/web/server/src/simcore_service_webserver/scicrunch/db.py b/services/web/server/src/simcore_service_webserver/scicrunch/db.py index 57e19bbed35..c99c82f2aed 100644 --- a/services/web/server/src/simcore_service_webserver/scicrunch/db.py +++ b/services/web/server/src/simcore_service_webserver/scicrunch/db.py @@ -1,5 +1,5 @@ """ - Access to postgres database scicrunch_resources table where USED rrids get stored +Access to postgres database scicrunch_resources table where USED rrids get stored """ import logging @@ -10,7 +10,7 @@ from simcore_postgres_database.models.scicrunch_resources import scicrunch_resources from sqlalchemy.dialects.postgresql import insert as sa_pg_insert -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from .models import ResearchResource, ResearchResourceAtdB logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ class ResearchResourceRepository: # WARNING: interfaces to both ResarchResource and ResearchResourceAtDB def __init__(self, app: web.Application): - self._engine = get_database_engine(app) + self._engine = get_database_engine_legacy(app) async def list_resources(self) -> list[ResearchResource]: async with self._engine.acquire() as conn: @@ -39,7 +39,9 @@ async def list_resources(self) -> list[ResearchResource]: ) res: ResultProxy = await conn.execute(stmt) rows: list[RowProxy] = await res.fetchall() - return [ResearchResource.model_validate(row) for row in rows] if rows else [] + return ( + [ResearchResource.model_validate(row) for row in rows] if rows else [] + ) async def get(self, rrid: str) -> ResearchResourceAtdB | None: async with self._engine.acquire() as conn: diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py index 752b1c3e2ee..1a17651e13e 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_catalog.py @@ -24,7 +24,7 @@ ) from simcore_postgres_database.utils_services import create_select_latest_services_query -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from ._errors import ServiceNotFound from .settings import StudiesDispatcherSettings, get_plugin_settings @@ -68,7 +68,7 @@ async def iter_latest_product_services( assert page_number >= 1 # nosec assert ((page_number - 1) * page_size) >= 0 # nosec - engine: Engine = get_database_engine(app) + engine: Engine = get_database_engine_legacy(app) settings: StudiesDispatcherSettings = get_plugin_settings(app) # Select query for latest version of the service @@ -140,7 +140,7 @@ async def validate_requested_service( service_key: ServiceKey, service_version: ServiceVersion, ) -> ValidService: - engine: Engine = get_database_engine(app) + engine: Engine = get_database_engine_legacy(app) async with engine.acquire() as conn: query = sa.select( diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py index f0fa876bf0d..b5660f40241 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_core.py @@ -14,7 +14,7 @@ ) from sqlalchemy.dialects.postgresql import ARRAY, INTEGER -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from ._errors import FileToLarge, IncompatibleService from ._models import ViewerInfo from .settings import get_plugin_settings @@ -40,7 +40,7 @@ async def list_viewers_info( # consumers: deque = deque() - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: # FIXME: ADD CONDITION: service MUST be shared with EVERYBODY! query = services_consume_filetypes.select() if file_type: @@ -117,7 +117,7 @@ def _version(column_or_value): return await get_default_viewer(app, file_type, file_size) if service_key and service_version: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: query = ( services_consume_filetypes.select() .where( diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py index 362bb7509b8..92206532c6b 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects_permalinks.py @@ -10,7 +10,7 @@ TypedDict, ) -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from ..projects.exceptions import PermalinkNotAllowedError, ProjectNotFoundError from ..projects.projects_permalink_service import ( ProjectPermalink, @@ -84,7 +84,7 @@ async def permalink_factory( """ # NOTE: next iterations will mobe this as part of the project repository pattern - engine = get_database_engine(request.app) + engine = get_database_engine_legacy(request.app) async with engine.acquire() as conn: access_rights_subquery = ( sa.select( diff --git a/services/web/server/src/simcore_service_webserver/users/_preferences_service.py b/services/web/server/src/simcore_service_webserver/users/_preferences_service.py index 0a5893141e1..b5083732d70 100644 --- a/services/web/server/src/simcore_service_webserver/users/_preferences_service.py +++ b/services/web/server/src/simcore_service_webserver/users/_preferences_service.py @@ -19,7 +19,7 @@ GroupExtraPropertiesRepo, ) -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from . import _preferences_repository from ._preferences_models import ( ALL_FRONTEND_PREFERENCES, @@ -75,7 +75,7 @@ async def get_frontend_user_preference( async def get_frontend_user_preferences_aggregation( app: web.Application, *, user_id: UserID, product_name: ProductName ) -> AggregatedPreferences: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: group_extra_properties = ( await GroupExtraPropertiesRepo.get_aggregated_properties_for_user( conn, user_id=user_id, product_name=product_name diff --git a/services/web/server/src/simcore_service_webserver/users/_tokens_service.py b/services/web/server/src/simcore_service_webserver/users/_tokens_service.py index 18e2f6323fd..f8ceba3651c 100644 --- a/services/web/server/src/simcore_service_webserver/users/_tokens_service.py +++ b/services/web/server/src/simcore_service_webserver/users/_tokens_service.py @@ -1,21 +1,22 @@ -""" Private user tokens from external services (e.g. dat-core) +"""Private user tokens from external services (e.g. dat-core) - Implemented as a stand-alone API but currently only exposed to the handlers +Implemented as a stand-alone API but currently only exposed to the handlers """ + import sqlalchemy as sa from aiohttp import web from models_library.users import UserID, UserThirdPartyToken from sqlalchemy import and_, literal_column from ..db.models import tokens -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from .exceptions import TokenNotFoundError async def create_token( app: web.Application, user_id: UserID, token: UserThirdPartyToken ) -> UserThirdPartyToken: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: await conn.execute( tokens.insert().values( user_id=user_id, @@ -30,7 +31,7 @@ async def list_tokens( app: web.Application, user_id: UserID ) -> list[UserThirdPartyToken]: user_tokens: list[UserThirdPartyToken] = [] - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: async for row in conn.execute( sa.select(tokens.c.token_data).where(tokens.c.user_id == user_id) ): @@ -41,7 +42,7 @@ async def list_tokens( async def get_token( app: web.Application, user_id: UserID, service_id: str ) -> UserThirdPartyToken: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute( sa.select(tokens.c.token_data).where( and_(tokens.c.user_id == user_id, tokens.c.token_service == service_id) @@ -55,7 +56,7 @@ async def get_token( async def update_token( app: web.Application, user_id: UserID, service_id: str, token_data: dict[str, str] ) -> UserThirdPartyToken: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute( sa.select(tokens.c.token_data, tokens.c.token_id).where( (tokens.c.user_id == user_id) & (tokens.c.token_service == service_id) @@ -82,7 +83,7 @@ async def update_token( async def delete_token(app: web.Application, user_id: UserID, service_id: str) -> None: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: await conn.execute( tokens.delete().where( and_(tokens.c.user_id == user_id, tokens.c.token_service == service_id) diff --git a/services/web/server/src/simcore_service_webserver/wallets/_db.py b/services/web/server/src/simcore_service_webserver/wallets/_db.py index 4d17c742925..3466f39b644 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_db.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_db.py @@ -12,7 +12,7 @@ from sqlalchemy.dialects.postgresql import BOOLEAN, INTEGER from sqlalchemy.sql import select -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from .errors import WalletAccessForbiddenError, WalletNotFoundError _logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ async def create_wallet( description: str | None, thumbnail: str | None, ) -> WalletDB: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute( wallets.insert() .values( @@ -90,7 +90,7 @@ async def list_wallets_for_user( ) ) - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute(stmt) rows = await result.fetchall() or [] output: list[UserWalletDB] = [UserWalletDB.model_validate(row) for row in rows] @@ -112,7 +112,7 @@ async def list_wallets_owned_by_user( & (wallets.c.product_name == product_name) ) ) - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: results = await conn.execute(stmt) rows = await results.fetchall() or [] return [row.wallet_id for row in rows] @@ -145,7 +145,7 @@ async def get_wallet_for_user( ) ) - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute(stmt) row = await result.first() if row is None: @@ -178,7 +178,7 @@ async def get_wallet( & (wallets.c.product_name == product_name) ) ) - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute(stmt) row = await result.first() if row is None: @@ -195,7 +195,7 @@ async def update_wallet( status: WalletStatus, product_name: ProductName, ) -> WalletDB: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute( wallets.update() .values( @@ -222,7 +222,7 @@ async def delete_wallet( wallet_id: WalletID, product_name: ProductName, ) -> None: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: await conn.execute( wallets.delete().where( (wallets.c.wallet_id == wallet_id) diff --git a/services/web/server/src/simcore_service_webserver/wallets/_groups_db.py b/services/web/server/src/simcore_service_webserver/wallets/_groups_db.py index c7e24fff4b8..8d6860b6ea1 100644 --- a/services/web/server/src/simcore_service_webserver/wallets/_groups_db.py +++ b/services/web/server/src/simcore_service_webserver/wallets/_groups_db.py @@ -9,7 +9,7 @@ from sqlalchemy import func, literal_column from sqlalchemy.sql import select -from ..db.plugin import get_database_engine +from ..db.plugin import get_database_engine_legacy from .errors import WalletGroupNotFoundError _logger = logging.getLogger(__name__) @@ -38,7 +38,7 @@ async def create_wallet_group( write: bool, delete: bool, ) -> WalletGroupGetDB: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute( wallet_to_groups.insert() .values( @@ -73,7 +73,7 @@ async def list_wallet_groups( .where(wallet_to_groups.c.wallet_id == wallet_id) ) - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute(stmt) rows = await result.fetchall() or [] return TypeAdapter(list[WalletGroupGetDB]).validate_python(rows) @@ -100,7 +100,7 @@ async def get_wallet_group( ) ) - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute(stmt) row = await result.first() if row is None: @@ -119,7 +119,7 @@ async def update_wallet_group( write: bool, delete: bool, ) -> WalletGroupGetDB: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: result = await conn.execute( wallet_to_groups.update() .values( @@ -146,7 +146,7 @@ async def delete_wallet_group( wallet_id: WalletID, group_id: GroupID, ) -> None: - async with get_database_engine(app).acquire() as conn: + async with get_database_engine_legacy(app).acquire() as conn: await conn.execute( wallet_to_groups.delete().where( (wallet_to_groups.c.wallet_id == wallet_id) diff --git a/services/web/server/tests/unit/with_dbs/01/test_director_v2_handlers.py b/services/web/server/tests/unit/with_dbs/01/test_director_v2_handlers.py index 4556878b0e0..8e8ef6faee4 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_director_v2_handlers.py +++ b/services/web/server/tests/unit/with_dbs/01/test_director_v2_handlers.py @@ -4,15 +4,22 @@ import pytest +import sqlalchemy as sa from aiohttp.test_utils import TestClient from faker import Faker from models_library.api_schemas_directorv2.comp_runs import ( + ComputationCollectionRunRpcGet, + ComputationCollectionRunRpcGetPage, + ComputationCollectionRunTaskRpcGet, + ComputationCollectionRunTaskRpcGetPage, ComputationRunRpcGet, ComputationRunRpcGetPage, ComputationTaskRpcGet, ComputationTaskRpcGetPage, ) from models_library.api_schemas_webserver.computations import ( + ComputationCollectionRunRestGet, + ComputationCollectionRunTaskRestGet, ComputationRunRestGet, ComputationTaskRestGet, ) @@ -26,6 +33,8 @@ ) from pytest_simcore.services_api_mocks_for_aiohttp_clients import AioResponsesMock from servicelib.aiohttp import status +from simcore_postgres_database.models.comp_runs_collections import comp_runs_collections +from simcore_postgres_database.models.projects_metadata import projects_metadata from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects.models import ProjectDict @@ -241,3 +250,187 @@ async def test_list_computations_latest_iteration( ) if user_role != UserRole.ANONYMOUS: assert ComputationTaskRestGet.model_validate(data[0]) + + +@pytest.fixture +def mock_rpc_list_computation_collection_runs_page( + mocker: MockerFixture, + user_project: ProjectDict, +) -> ComputationCollectionRunRpcGetPage: + project_uuid = user_project["uuid"] + example = ComputationCollectionRunRpcGet.model_config["json_schema_extra"][ + "examples" + ][0] + example["project_ids"] = [project_uuid] + example["info"]["project_metadata"]["root_parent_project_id"] = project_uuid + + return mocker.patch( + "simcore_service_webserver.director_v2._computations_service.computations.list_computation_collection_runs_page", + spec=True, + return_value=ComputationCollectionRunRpcGetPage( + items=[ComputationCollectionRunRpcGet.model_validate(example)], + total=1, + ), + ) + + +@pytest.fixture +def mock_rpc_list_computation_collection_run_tasks_page( + mocker: MockerFixture, + user_project: ProjectDict, +) -> str: + project_uuid = user_project["uuid"] + workbench_ids = list(user_project["workbench"].keys()) + example = ComputationCollectionRunTaskRpcGet.model_config["json_schema_extra"][ + "examples" + ][0] + example["node_id"] = workbench_ids[0] + example["project_uuid"] = project_uuid + + mocker.patch( + "simcore_service_webserver.director_v2._computations_service.computations.list_computation_collection_run_tasks_page", + spec=True, + return_value=ComputationCollectionRunTaskRpcGetPage( + items=[ComputationCollectionRunTaskRpcGet.model_validate(example)], + total=1, + ), + ) + + return workbench_ids[0] + + +@pytest.mark.parametrize(*standard_role_response(), ids=str) +async def test_list_computation_collection_runs_and_tasks( + director_v2_service_mock: AioResponsesMock, + user_project: ProjectDict, + client: TestClient, + logged_user: LoggedUser, + user_role: UserRole, + expected: ExpectedResponse, + mock_rpc_list_computation_collection_runs_page: None, + mock_rpc_list_computation_collection_run_tasks_page: str, + faker: Faker, +): + assert client.app + url = client.app.router["list_computation_collection_runs"].url_for() + resp = await client.get(f"{url}") + data, _ = await assert_status( + resp, status.HTTP_200_OK if user_role == UserRole.GUEST else expected.ok + ) + if user_role != UserRole.ANONYMOUS: + assert ComputationCollectionRunRestGet.model_validate(data[0]) + assert data[0]["name"] == user_project["name"] + + url = client.app.router["list_computation_collection_run_tasks"].url_for( + collection_run_id=faker.uuid4() + ) + resp = await client.get(f"{url}") + data, _ = await assert_status( + resp, status.HTTP_200_OK if user_role == UserRole.GUEST else expected.ok + ) + if user_role != UserRole.ANONYMOUS: + assert ComputationCollectionRunTaskRestGet.model_validate(data[0]) + assert len(data) == 1 + assert ( + data[0]["name"] + == user_project["workbench"][ + mock_rpc_list_computation_collection_run_tasks_page + ]["label"] + ) + + +@pytest.fixture +async def populated_comp_run_collection( + client: TestClient, + postgres_db: sa.engine.Engine, +): + assert client.app + example = ComputationCollectionRunRpcGet.model_config["json_schema_extra"][ + "examples" + ][0] + collection_run_id = example["collection_run_id"] + + with postgres_db.connect() as con: + con.execute( + comp_runs_collections.insert() + .values( + collection_run_id=collection_run_id, + client_or_system_generated_id=collection_run_id, + client_or_system_generated_display_name="My Collection Run", + is_generated_by_system=False, + created=sa.func.now(), + modified=sa.func.now(), + ) + .returning(comp_runs_collections.c.collection_run_id) + ) + yield + con.execute(comp_runs_collections.delete()) + + +@pytest.mark.parametrize(*standard_role_response(), ids=str) +async def test_list_computation_collection_runs_with_client_defined_name( + director_v2_service_mock: AioResponsesMock, + user_project: ProjectDict, + client: TestClient, + logged_user: LoggedUser, + user_role: UserRole, + expected: ExpectedResponse, + populated_comp_run_collection: None, + mock_rpc_list_computation_collection_runs_page: None, +): + assert client.app + url = client.app.router["list_computation_collection_runs"].url_for() + resp = await client.get(f"{url}") + data, _ = await assert_status( + resp, status.HTTP_200_OK if user_role == UserRole.GUEST else expected.ok + ) + if user_role != UserRole.ANONYMOUS: + assert ComputationCollectionRunRestGet.model_validate(data[0]) + assert data[0]["name"] == "My Collection Run" + + +@pytest.fixture +async def populated_project_metadata( + client: TestClient, + logged_user: LoggedUser, + user_project: ProjectDict, + postgres_db: sa.engine.Engine, +): + assert client.app + project_uuid = user_project["uuid"] + with postgres_db.connect() as con: + con.execute( + projects_metadata.insert().values( + **{ + "project_uuid": project_uuid, + "custom": {"job_name": "My Job Name"}, + } + ) + ) + yield + con.execute(projects_metadata.delete()) + + +@pytest.mark.parametrize(*standard_role_response(), ids=str) +async def test_list_computation_collection_runs_and_tasks_with_different_names( + director_v2_service_mock: AioResponsesMock, + user_project: ProjectDict, + client: TestClient, + logged_user: LoggedUser, + user_role: UserRole, + expected: ExpectedResponse, + populated_project_metadata: None, + mock_rpc_list_computation_collection_run_tasks_page: str, + faker: Faker, +): + assert client.app + url = client.app.router["list_computation_collection_run_tasks"].url_for( + collection_run_id=faker.uuid4() + ) + resp = await client.get(f"{url}") + data, _ = await assert_status( + resp, status.HTTP_200_OK if user_role == UserRole.GUEST else expected.ok + ) + if user_role != UserRole.ANONYMOUS: + assert ComputationCollectionRunTaskRestGet.model_validate(data[0]) + assert data[0]["name"] == "My Job Name" diff --git a/services/web/server/tests/unit/with_dbs/03/tags/test_tags.py b/services/web/server/tests/unit/with_dbs/03/tags/test_tags.py index ee15b1d1494..df2b7541756 100644 --- a/services/web/server/tests/unit/with_dbs/03/tags/test_tags.py +++ b/services/web/server/tests/unit/with_dbs/03/tags/test_tags.py @@ -29,7 +29,7 @@ from servicelib.aiohttp import status from simcore_postgres_database.models.tags import tags from simcore_service_webserver.db.models import UserRole -from simcore_service_webserver.db.plugin import get_database_engine +from simcore_service_webserver.db.plugin import get_database_engine_legacy from simcore_service_webserver.products._service import get_product from simcore_service_webserver.projects.models import ProjectDict @@ -121,7 +121,7 @@ async def test_tags_to_studies( @pytest.fixture async def everybody_tag_id(client: TestClient) -> AsyncIterator[int]: assert client.app - engine = get_database_engine(client.app) + engine = get_database_engine_legacy(client.app) assert engine async with engine.acquire() as conn: diff --git a/services/web/server/tests/unit/with_dbs/03/test_users__tokens.py b/services/web/server/tests/unit/with_dbs/03/test_users__tokens.py index 690862f3a8a..56af58d1efe 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_users__tokens.py +++ b/services/web/server/tests/unit/with_dbs/03/test_users__tokens.py @@ -25,7 +25,7 @@ from pytest_simcore.helpers.webserver_users import UserInfoDict from servicelib.aiohttp import status from simcore_postgres_database.models.users import UserRole -from simcore_service_webserver.db.plugin import get_database_engine +from simcore_service_webserver.db.plugin import get_database_engine_legacy @pytest.fixture @@ -48,7 +48,7 @@ async def tokens_db_cleanup( client: TestClient, ) -> AsyncIterator[None]: assert client.app - engine = get_database_engine(client.app) + engine = get_database_engine_legacy(client.app) yield None @@ -75,7 +75,7 @@ async def fake_tokens( "token_secret": faker.md5(raw_output=False), } await create_token_in_db( - get_database_engine(client.app), + get_database_engine_legacy(client.app), user_id=logged_user["id"], token_service=data["service"], token_data=data, @@ -115,7 +115,7 @@ async def test_create_token( data, error = await assert_status(resp, expected) if not error: db_token = await get_token_from_db( - get_database_engine(client.app), token_data=token + get_database_engine_legacy(client.app), token_data=token ) assert db_token assert db_token["token_data"] == token @@ -191,5 +191,7 @@ async def test_delete_token( if not error: assert not ( - await get_token_from_db(get_database_engine(client.app), token_service=sid) + await get_token_from_db( + get_database_engine_legacy(client.app), token_service=sid + ) ) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index e446da4e19c..539006a5d95 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -72,7 +72,7 @@ from simcore_service_webserver.application import create_application from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.constants import INDEX_RESOURCE_NAME -from simcore_service_webserver.db.plugin import get_database_engine +from simcore_service_webserver.db.plugin import get_database_engine_legacy from simcore_service_webserver.projects.models import ProjectDict from simcore_service_webserver.projects.utils import NodesMap from simcore_service_webserver.statics._constants import ( @@ -262,7 +262,7 @@ def osparc_product_api_base_url() -> str: @pytest.fixture async def default_product_name(client: TestClient) -> ProductName: assert client.app - async with get_database_engine(client.app).acquire() as conn: + async with get_database_engine_legacy(client.app).acquire() as conn: return await get_default_product_name(conn)