From d6ae5e6871c4a604deb5eab787e6551d7012517d Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Mon, 19 May 2025 13:56:16 +0200 Subject: [PATCH 1/2] migration --- api/specs/web-server/_projects_comments.py | 116 ------- api/specs/web-server/openapi.py | 1 - ...14f_preparation_for_osparc_io_migration.py | 298 ++++++++++++++++++ .../models/file_meta_data.py | 15 +- .../models/folders_v2.py | 2 + .../models/payments_autorecharge.py | 8 +- .../models/payments_methods.py | 14 + .../models/payments_transactions.py | 20 ++ .../models/projects_comments.py | 53 ---- ..._service_runs__osparc_io_history_202508.py | 228 ++++++++++++++ .../models/tokens.py | 14 +- .../api/v0/openapi.yaml | 268 ---------------- .../projects/_comments_repository.py | 98 ------ .../projects/_comments_service.py | 96 ------ .../projects/_controller/comments_rest.py | 228 -------------- .../projects/_projects_repository_legacy.py | 54 ---- .../projects/plugin.py | 2 - 17 files changed, 594 insertions(+), 921 deletions(-) delete mode 100644 api/specs/web-server/_projects_comments.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/e98c45ff314f_preparation_for_osparc_io_migration.py delete mode 100644 packages/postgres-database/src/simcore_postgres_database/models/projects_comments.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_service_runs__osparc_io_history_202508.py delete mode 100644 services/web/server/src/simcore_service_webserver/projects/_comments_repository.py delete mode 100644 services/web/server/src/simcore_service_webserver/projects/_comments_service.py delete mode 100644 services/web/server/src/simcore_service_webserver/projects/_controller/comments_rest.py diff --git a/api/specs/web-server/_projects_comments.py b/api/specs/web-server/_projects_comments.py deleted file mode 100644 index 04ad1f1fa43..00000000000 --- a/api/specs/web-server/_projects_comments.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Helper script to automatically generate OAS - -This OAS are the source of truth -""" - -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable -# pylint: disable=too-many-arguments - -from typing import Literal - -from _common import assert_handler_signature_against_model -from fastapi import APIRouter, status -from models_library.generics import Envelope -from models_library.projects import ProjectID -from models_library.projects_comments import CommentID, ProjectsCommentsAPI -from pydantic import NonNegativeInt -from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.projects._controller.comments_rest import ( - _ProjectCommentsBodyParams, - _ProjectCommentsPathParams, - _ProjectCommentsWithCommentPathParams, -) - -router = APIRouter( - prefix=f"/{API_VTAG}", - tags=[ - "projects", - "comments", - ], -) - - -# -# API entrypoints -# - - -@router.post( - "/projects/{project_uuid}/comments", - response_model=Envelope[dict[Literal["comment_id"], CommentID]], - description="Create a new comment for a specific project. The request body should contain the comment contents and user information.", - status_code=status.HTTP_201_CREATED, - deprecated=True, -) -async def create_project_comment( - project_uuid: ProjectID, body: _ProjectCommentsBodyParams -): ... - - -assert_handler_signature_against_model( - create_project_comment, _ProjectCommentsPathParams -) - - -@router.get( - "/projects/{project_uuid}/comments", - response_model=Envelope[list[ProjectsCommentsAPI]], - description="Retrieve all comments for a specific project.", - deprecated=True, -) -async def list_project_comments( - project_uuid: ProjectID, limit: int = 20, offset: NonNegativeInt = 0 -): ... - - -assert_handler_signature_against_model( - list_project_comments, _ProjectCommentsPathParams -) - - -@router.put( - "/projects/{project_uuid}/comments/{comment_id}", - response_model=Envelope[ProjectsCommentsAPI], - description="Update the contents of a specific comment for a project. The request body should contain the updated comment contents.", - deprecated=True, -) -async def update_project_comment( - project_uuid: ProjectID, - comment_id: CommentID, - body: _ProjectCommentsBodyParams, -): ... - - -assert_handler_signature_against_model( - update_project_comment, _ProjectCommentsWithCommentPathParams -) - - -@router.delete( - "/projects/{project_uuid}/comments/{comment_id}", - description="Delete a specific comment associated with a project.", - status_code=status.HTTP_204_NO_CONTENT, - deprecated=True, -) -async def delete_project_comment(project_uuid: ProjectID, comment_id: CommentID): ... - - -assert_handler_signature_against_model( - delete_project_comment, _ProjectCommentsWithCommentPathParams -) - - -@router.get( - "/projects/{project_uuid}/comments/{comment_id}", - response_model=Envelope[ProjectsCommentsAPI], - description="Retrieve a specific comment by its ID within a project.", - deprecated=True, -) -async def get_project_comment(project_uuid: ProjectID, comment_id: CommentID): ... - - -assert_handler_signature_against_model( - get_project_comment, _ProjectCommentsWithCommentPathParams -) diff --git a/api/specs/web-server/openapi.py b/api/specs/web-server/openapi.py index fa0c86abcc7..fae410644d1 100644 --- a/api/specs/web-server/openapi.py +++ b/api/specs/web-server/openapi.py @@ -44,7 +44,6 @@ "_nih_sparc_redirections", "_projects", "_projects_access_rights", - "_projects_comments", "_projects_conversations", "_projects_folders", "_projects_metadata", diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e98c45ff314f_preparation_for_osparc_io_migration.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e98c45ff314f_preparation_for_osparc_io_migration.py new file mode 100644 index 00000000000..70b06ca060b --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e98c45ff314f_preparation_for_osparc_io_migration.py @@ -0,0 +1,298 @@ +"""preparation for osparc.io migration + +Revision ID: e98c45ff314f +Revises: b39f2dc87ccd +Create Date: 2025-05-19 11:55:04.513120+00:00 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "e98c45ff314f" +down_revision = "b39f2dc87ccd" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "resource_tracker_service_runs__osparc_io_history_202508", + sa.Column("product_name", sa.String(), nullable=False), + sa.Column("service_run_id", sa.String(), nullable=False), + sa.Column("wallet_id", sa.BigInteger(), nullable=True), + sa.Column("wallet_name", sa.String(), nullable=True), + sa.Column("pricing_plan_id", sa.BigInteger(), nullable=True), + sa.Column("pricing_unit_id", sa.BigInteger(), nullable=True), + sa.Column("pricing_unit_cost_id", sa.BigInteger(), nullable=True), + sa.Column("pricing_unit_cost", sa.Numeric(scale=2), nullable=True), + sa.Column("simcore_user_agent", sa.String(), nullable=True), + sa.Column("user_id", sa.BigInteger(), nullable=False), + sa.Column("user_email", sa.String(), nullable=True), + sa.Column("project_id", sa.String(), nullable=False), + sa.Column("project_name", sa.String(), nullable=False), + sa.Column("node_id", sa.String(), nullable=False), + sa.Column("node_name", sa.String(), nullable=False), + sa.Column("parent_project_id", sa.String(), nullable=False), + sa.Column("root_parent_project_id", sa.String(), nullable=False), + sa.Column("root_parent_project_name", sa.String(), nullable=False), + sa.Column("parent_node_id", sa.String(), nullable=False), + sa.Column("root_parent_node_id", sa.String(), nullable=False), + sa.Column("service_key", sa.String(), nullable=False), + sa.Column("service_version", sa.String(), nullable=False), + sa.Column( + "service_type", + sa.Enum( + "COMPUTATIONAL_SERVICE", + "DYNAMIC_SERVICE", + name="resourcetrackerservicetypeosparciohistory", + ), + nullable=False, + ), + sa.Column( + "service_resources", postgresql.JSONB(astext_type=sa.Text()), nullable=False + ), + sa.Column( + "service_additional_metadata", + postgresql.JSONB(astext_type=sa.Text()), + nullable=False, + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("stopped_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "service_run_status", + sa.Enum( + "RUNNING", + "SUCCESS", + "ERROR", + name="resourcetrackerservicerunstatusosparciohistory", + ), + nullable=False, + ), + sa.Column( + "modified", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("last_heartbeat_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("service_run_status_msg", sa.String(), nullable=True), + sa.Column("missed_heartbeat_counter", sa.SmallInteger(), nullable=False), + sa.PrimaryKeyConstraint("product_name", "service_run_id"), + ) + op.create_index( + op.f("ix_resource_tracker_service_runs__osparc_io_history_202508_wallet_id"), + "resource_tracker_service_runs__osparc_io_history_202508", + ["wallet_id"], + unique=False, + ) + op.create_index( + op.f("ix_resource_tracker_service_runs__osparc_io_history_202508_user_id"), + "resource_tracker_service_runs__osparc_io_history_202508", + ["user_id"], + unique=False, + ) + op.create_index( + op.f("ix_resource_tracker_service_runs__osparc_io_history_202508_started_at"), + "resource_tracker_service_runs__osparc_io_history_202508", + ["started_at"], + unique=False, + ) + op.drop_index("ix_projects_comments_project_uuid", table_name="projects_comments") + op.drop_table("projects_comments") + op.drop_column("file_meta_data", "node_id") + op.drop_constraint("fk_new_folders_to_folders_id", "folders_v2", type_="foreignkey") + op.drop_constraint("fk_new_folders_to_groups_gid", "folders_v2", type_="foreignkey") + op.create_foreign_key( + "fk_new_folders_to_folders_id", + "folders_v2", + "folders_v2", + ["parent_folder_id"], + ["folder_id"], + onupdate="CASCADE", + ) + op.create_foreign_key( + "fk_new_folders_to_groups_gid", + "folders_v2", + "groups", + ["created_by_gid"], + ["gid"], + onupdate="CASCADE", + ondelete="SET NULL", + ) + op.create_foreign_key( + "fk_payments_autorecharge_id_wallets", + "payments_autorecharge", + "wallets", + ["wallet_id"], + ["wallet_id"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + op.create_foreign_key( + "fk_payments_methods_to_user_id", + "payments_methods", + "users", + ["user_id"], + ["id"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + op.create_foreign_key( + "fk_payments_methods_to_wallet_id", + "payments_methods", + "wallets", + ["wallet_id"], + ["wallet_id"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + op.create_foreign_key( + "fk_payments_transactions_to_wallet_id", + "payments_transactions", + "wallets", + ["wallet_id"], + ["wallet_id"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + op.create_foreign_key( + "fk_payments_transactions_to_user_id", + "payments_transactions", + "users", + ["user_id"], + ["id"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + op.create_foreign_key( + "fk_payments_transactions_to_products_name", + "payments_transactions", + "products", + ["product_name"], + ["name"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + op.drop_constraint("tokens_user_id_fkey", "tokens", type_="foreignkey") + op.create_foreign_key( + None, + "tokens", + "users", + ["user_id"], + ["id"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, "tokens", type_="foreignkey") + op.create_foreign_key("tokens_user_id_fkey", "tokens", "users", ["user_id"], ["id"]) + op.drop_constraint( + "fk_payments_transactions_to_products_name", + "payments_transactions", + type_="foreignkey", + ) + op.drop_constraint( + "fk_payments_transactions_to_user_id", + "payments_transactions", + type_="foreignkey", + ) + op.drop_constraint( + "fk_payments_transactions_to_wallet_id", + "payments_transactions", + type_="foreignkey", + ) + op.drop_constraint( + "fk_payments_methods_to_wallet_id", "payments_methods", type_="foreignkey" + ) + op.drop_constraint( + "fk_payments_methods_to_user_id", "payments_methods", type_="foreignkey" + ) + op.drop_constraint( + "fk_payments_autorecharge_id_wallets", + "payments_autorecharge", + type_="foreignkey", + ) + op.drop_constraint("fk_new_folders_to_groups_gid", "folders_v2", type_="foreignkey") + op.drop_constraint("fk_new_folders_to_folders_id", "folders_v2", type_="foreignkey") + op.create_foreign_key( + "fk_new_folders_to_groups_gid", + "folders_v2", + "groups", + ["created_by_gid"], + ["gid"], + ondelete="SET NULL", + ) + op.create_foreign_key( + "fk_new_folders_to_folders_id", + "folders_v2", + "folders_v2", + ["parent_folder_id"], + ["folder_id"], + ) + op.add_column( + "file_meta_data", + sa.Column("node_id", sa.VARCHAR(), autoincrement=False, nullable=True), + ) + op.create_table( + "projects_comments", + sa.Column("comment_id", sa.BIGINT(), autoincrement=True, nullable=False), + sa.Column("project_uuid", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("user_id", sa.BIGINT(), autoincrement=False, nullable=True), + sa.Column("contents", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column( + "created", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "modified", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.ForeignKeyConstraint( + ["project_uuid"], + ["projects.uuid"], + name="fk_projects_comments_project_uuid", + onupdate="CASCADE", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name="fk_projects_comments_user_id", + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("comment_id", name="projects_comments_pkey"), + ) + op.create_index( + "ix_projects_comments_project_uuid", + "projects_comments", + ["project_uuid"], + unique=False, + ) + op.drop_index( + op.f("ix_resource_tracker_service_runs__osparc_io_history_202508_started_at"), + table_name="resource_tracker_service_runs__osparc_io_history_202508", + ) + op.drop_index( + op.f("ix_resource_tracker_service_runs__osparc_io_history_202508_user_id"), + table_name="resource_tracker_service_runs__osparc_io_history_202508", + ) + op.drop_index( + op.f("ix_resource_tracker_service_runs__osparc_io_history_202508_wallet_id"), + table_name="resource_tracker_service_runs__osparc_io_history_202508", + ) + op.drop_table("resource_tracker_service_runs__osparc_io_history_202508") + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/file_meta_data.py b/packages/postgres-database/src/simcore_postgres_database/models/file_meta_data.py index 9ece039863f..21d6ce899be 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/file_meta_data.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/file_meta_data.py @@ -1,4 +1,5 @@ import sqlalchemy as sa +from simcore_postgres_database.models._common import RefActions from .base import metadata @@ -10,7 +11,18 @@ sa.Column("bucket_name", sa.String()), sa.Column("object_name", sa.String()), sa.Column("project_id", sa.String(), index=True), - sa.Column("node_id", sa.String()), + sa.Column( + "user_id", + sa.BigInteger(), + sa.ForeignKey( + "users.id", + name="fk_file_meta_data_user_id_users", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + ), + nullable=False, + doc="The user id with which the run entry is associated", + ), sa.Column("user_id", sa.String(), index=True), sa.Column("file_id", sa.String(), primary_key=True), sa.Column("created_at", sa.String()), @@ -56,4 +68,5 @@ doc="SHA256 checksum of the file content", index=True, ), + ### ) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/folders_v2.py b/packages/postgres-database/src/simcore_postgres_database/models/folders_v2.py index eebfd2079f8..5d06503f78e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/folders_v2.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/folders_v2.py @@ -33,6 +33,7 @@ sa.BigInteger, sa.ForeignKey( "folders_v2.folder_id", + onupdate=RefActions.CASCADE, name="fk_new_folders_to_folders_id", ), nullable=True, @@ -77,6 +78,7 @@ "groups.gid", name="fk_new_folders_to_groups_gid", ondelete=RefActions.SET_NULL, + onupdate=RefActions.CASCADE, ), nullable=True, ), diff --git a/packages/postgres-database/src/simcore_postgres_database/models/payments_autorecharge.py b/packages/postgres-database/src/simcore_postgres_database/models/payments_autorecharge.py index df30251c50c..b96ffdb2e11 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/payments_autorecharge.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/payments_autorecharge.py @@ -9,6 +9,7 @@ ) from .base import metadata from .payments_methods import payments_methods +from .wallets import wallets # # NOTE: @@ -29,7 +30,12 @@ sa.Column( "wallet_id", sa.BigInteger, - # NOTE: cannot use foreign-key because it would require a link to wallets table + sa.ForeignKey( + wallets.c.wallet_id, + name="fk_payments_autorecharge_id_wallets", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + ), nullable=False, doc="Wallet associated to the auto-recharge", unique=True, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/payments_methods.py b/packages/postgres-database/src/simcore_postgres_database/models/payments_methods.py index 3aabc3f992c..9d9d84ee741 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/payments_methods.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/payments_methods.py @@ -3,11 +3,13 @@ import sqlalchemy as sa from ._common import ( + RefActions, column_created_datetime, column_modified_datetime, register_modified_datetime_auto_update_trigger, ) from .base import metadata +from .wallets import wallets @enum.unique @@ -41,6 +43,12 @@ class InitPromptAckFlowState(str, enum.Enum): sa.Column( "user_id", sa.BigInteger, + sa.ForeignKey( + "users.id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_payments_methods_to_user_id", + ), nullable=False, doc="Unique identifier of the user", index=True, @@ -48,6 +56,12 @@ class InitPromptAckFlowState(str, enum.Enum): sa.Column( "wallet_id", sa.BigInteger, + sa.ForeignKey( + wallets.c.wallet_id, + name="fk_payments_methods_to_wallet_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + ), nullable=False, doc="Unique identifier to the wallet owned by the user", index=True, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/payments_transactions.py b/packages/postgres-database/src/simcore_postgres_database/models/payments_transactions.py index 21916b0615b..f44613690b9 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/payments_transactions.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/payments_transactions.py @@ -4,11 +4,13 @@ from ._common import ( NUMERIC_KWARGS, + RefActions, column_created_datetime, column_modified_datetime, register_modified_datetime_auto_update_trigger, ) from .base import metadata +from .wallets import wallets @unique @@ -60,12 +62,24 @@ def is_acknowledged(self) -> bool: sa.Column( "product_name", sa.String, + sa.ForeignKey( + "products.name", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_payments_transactions_to_products_name", + ), nullable=False, doc="Product name from which the transaction took place", ), sa.Column( "user_id", sa.BigInteger, + sa.ForeignKey( + "users.id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + name="fk_payments_transactions_to_user_id", + ), nullable=False, doc="User unique identifier", index=True, @@ -79,6 +93,12 @@ def is_acknowledged(self) -> bool: sa.Column( "wallet_id", sa.BigInteger, + sa.ForeignKey( + wallets.c.wallet_id, + name="fk_payments_transactions_to_wallet_id", + onupdate=RefActions.CASCADE, + ondelete=RefActions.CASCADE, + ), nullable=False, doc="Wallet identifier owned by the user", index=True, diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_comments.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_comments.py deleted file mode 100644 index 919b143bff3..00000000000 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_comments.py +++ /dev/null @@ -1,53 +0,0 @@ -import sqlalchemy as sa - -from ._common import RefActions, column_created_datetime, column_modified_datetime -from .base import metadata -from .projects import projects -from .users import users - -projects_comments = sa.Table( - "projects_comments", - metadata, - sa.Column( - "comment_id", - sa.BigInteger, - nullable=False, - autoincrement=True, - primary_key=True, - doc="Primary key, identifies the comment", - ), - sa.Column( - "project_uuid", - sa.String, - sa.ForeignKey( - projects.c.uuid, - name="fk_projects_comments_project_uuid", - ondelete=RefActions.CASCADE, - onupdate=RefActions.CASCADE, - ), - index=True, - nullable=False, - doc="project reference for this table", - ), - # NOTE: if the user gets deleted, it sets to null which should be interpreted as "unknown" user - sa.Column( - "user_id", - sa.BigInteger, - sa.ForeignKey( - users.c.id, - name="fk_projects_comments_user_id", - ondelete=RefActions.SET_NULL, - ), - doc="user who created the comment", - nullable=True, - ), - sa.Column( - "contents", - sa.String, - nullable=False, - doc="Content of the comment", - ), - column_created_datetime(timezone=True), - column_modified_datetime(timezone=True), - sa.PrimaryKeyConstraint("comment_id", name="projects_comments_pkey"), -) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_service_runs__osparc_io_history_202508.py b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_service_runs__osparc_io_history_202508.py new file mode 100644 index 00000000000..d0e5720efce --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/models/resource_tracker_service_runs__osparc_io_history_202508.py @@ -0,0 +1,228 @@ +"""resource_tracker_service_runs table""" + +import enum + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB + +from ._common import NUMERIC_KWARGS, column_modified_datetime +from .base import metadata + + +class ResourceTrackerServiceTypeOsparcIoHistory(str, enum.Enum): + COMPUTATIONAL_SERVICE = "COMPUTATIONAL_SERVICE" + DYNAMIC_SERVICE = "DYNAMIC_SERVICE" + + +class ResourceTrackerServiceRunStatusOsparcIoHistory(str, enum.Enum): + RUNNING = "RUNNING" + SUCCESS = "SUCCESS" + ERROR = "ERROR" + + +resource_tracker_service_runs__osparc_io_history_202508 = sa.Table( + "resource_tracker_service_runs__osparc_io_history_202508", + metadata, + # Primary keys + sa.Column( + "product_name", sa.String, nullable=False, doc="Product name", primary_key=True + ), + sa.Column( + "service_run_id", + sa.String, + nullable=False, + doc="Refers to the unique service_run_id provided by the director-v2/dynamic-sidecars.", + primary_key=True, + ), + # Wallet fields + sa.Column( + "wallet_id", + sa.BigInteger, + nullable=True, + doc="We want to store the wallet id for tracking/billing purposes and be sure it stays there even when the wallet is deleted (that's also reason why we do not introduce foreign key)", + index=True, + ), + sa.Column( + "wallet_name", + sa.String, + nullable=True, + doc="We want to store the wallet name for tracking/billing purposes and be sure it stays there even when the wallet is deleted (that's also reason why we do not introduce foreign key)", + ), + # Pricing fields + sa.Column( + "pricing_plan_id", + sa.BigInteger, + nullable=True, + doc="Pricing plan id for billing purposes", + ), + sa.Column( + "pricing_unit_id", + sa.BigInteger, + nullable=True, + doc="Pricing unit id for billing purposes", + ), + sa.Column( + "pricing_unit_cost_id", + sa.BigInteger, + nullable=True, + doc="Pricing unit cost id for billing purposes", + ), + sa.Column( + "pricing_unit_cost", + sa.Numeric(**NUMERIC_KWARGS), # type: ignore + nullable=True, + doc="Pricing unit cost used for billing purposes", + ), + # User agent field + sa.Column( + "simcore_user_agent", + sa.String, + nullable=True, + doc="Information about whether it is Puppeteer or not", + ), + # User fields + sa.Column( + "user_id", + sa.BigInteger, + nullable=False, + doc="We want to store the user id for tracking/billing purposes and be sure it stays there even when the user is deleted (that's also reason why we do not introduce foreign key)", + index=True, + ), + sa.Column( + "user_email", + sa.String, + nullable=True, + doc="we want to store the email for tracking/billing purposes and be sure it stays there even when the user is deleted (that's also reason why we do not introduce foreign key)", + ), + # Project fields + sa.Column( + "project_id", # UUID + sa.String, + nullable=False, + doc="We want to store the project id for tracking/billing purposes and be sure it stays there even when the project is deleted (that's also reason why we do not introduce foreign key)", + ), + sa.Column( + "project_name", + sa.String, + nullable=False, + doc="we want to store the project name for tracking/billing purposes and be sure it stays there even when the project is deleted (that's also reason why we do not introduce foreign key)", + ), + # Node fields + sa.Column( + "node_id", # UUID + sa.String, + nullable=False, + doc="We want to store the node id for tracking/billing purposes and be sure it stays there even when the node is deleted (that's also reason why we do not introduce foreign key)", + ), + sa.Column( + "node_name", + sa.String, + nullable=False, + doc="we want to store the node/service name/label for tracking/billing purposes and be sure it stays there even when the node is deleted.", + ), + # Project/Node parent fields + sa.Column( + "parent_project_id", # UUID + sa.String, + nullable=False, + doc="If a user starts computational jobs via a dynamic service, a new project is created in the backend. This newly created project is considered a child project, and the project from which it was created is the parent project. We want to store the parent project ID for tracking and billing purposes, and ensure it remains even when the node is deleted. This is also the reason why we do not introduce a foreign key.", + ), + sa.Column( + "root_parent_project_id", # UUID + sa.String, + nullable=False, + doc="Similar to the parent project concept, we are flexible enough to allow multiple nested computational jobs, which create multiple nested projects. For this reason, we keep the parent project ID, so we know from which project the user started their computation.", + ), + sa.Column( + "root_parent_project_name", + sa.String, + nullable=False, + doc="We want to store the root parent project name for tracking/billing purposes.", + ), + sa.Column( + "parent_node_id", # UUID + sa.String, + nullable=False, + doc="Since each project can have multiple nodes, similar to the parent project concept, we also store the parent node..", + ), + sa.Column( + "root_parent_node_id", # UUID + sa.String, + nullable=False, + doc="Since each project can have multiple nodes, similar to the root parent project concept, we also store the root parent node.", + ), + # Service fields + sa.Column( + "service_key", + sa.String, + nullable=False, + doc="Service Key", + ), + sa.Column( + "service_version", + sa.String, + nullable=False, + doc="Service Version", + ), + sa.Column( + "service_type", + sa.Enum(ResourceTrackerServiceTypeOsparcIoHistory), + nullable=False, + doc="Service type, ex. COMPUTATIONAL, DYNAMIC", + ), + sa.Column( + "service_resources", + JSONB, + nullable=False, + default="'{}'::jsonb", + doc="Service aresources, ex. cpu, gpu, memory, ...", + ), + sa.Column( + "service_additional_metadata", + JSONB, + nullable=False, + default="'{}'::jsonb", + doc="Service additional metadata.", + ), + # Run timestamps + sa.Column( + "started_at", + sa.DateTime(timezone=True), + nullable=False, + doc="Timestamp when the service was started", + index=True, + ), + sa.Column( + "stopped_at", + sa.DateTime(timezone=True), + nullable=True, + doc="Timestamp when the service was stopped", + ), + # Run status + sa.Column( + "service_run_status", # Partial index was defined bellow + sa.Enum(ResourceTrackerServiceRunStatusOsparcIoHistory), + nullable=False, + ), + column_modified_datetime(timezone=True), + # Last Heartbeat + sa.Column( + "last_heartbeat_at", + sa.DateTime(timezone=True), + nullable=False, + doc="Timestamp when was the last heartbeat", + ), + sa.Column( + "service_run_status_msg", + sa.String, + nullable=True, + doc="Custom message/comment, for example to help understand root cause of the error during investigation", + ), + sa.Column( + "missed_heartbeat_counter", + sa.SmallInteger, + nullable=False, + default=0, + doc="How many heartbeat checks have been missed", + ), +) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/tokens.py b/packages/postgres-database/src/simcore_postgres_database/models/tokens.py index 990de23c724..e5ee2ce0419 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/tokens.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/tokens.py @@ -1,6 +1,7 @@ -""" User Tokens table -""" +"""User Tokens table""" + import sqlalchemy as sa +from simcore_postgres_database.models._common import RefActions from .base import metadata from .users import users @@ -10,7 +11,14 @@ "tokens", metadata, sa.Column("token_id", sa.BigInteger, nullable=False, primary_key=True), - sa.Column("user_id", sa.BigInteger, sa.ForeignKey(users.c.id), nullable=False), + sa.Column( + "user_id", + sa.BigInteger, + sa.ForeignKey( + users.c.id, onupdate=RefActions.CASCADE, ondelete=RefActions.CASCADE + ), + nullable=False, + ), sa.Column("token_service", sa.String, nullable=False), sa.Column("token_data", sa.JSON, nullable=False), ) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index bcda438663d..823d889bfe6 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -4712,172 +4712,6 @@ paths: application/json: schema: $ref: '#/components/schemas/Envelope_list_ProjectGroupGet__' - /v0/projects/{project_uuid}/comments: - post: - tags: - - projects - - comments - summary: Create Project Comment - description: Create a new comment for a specific project. The request body should - contain the comment contents and user information. - operationId: create_project_comment - deprecated: true - parameters: - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/_ProjectCommentsBodyParams' - responses: - '201': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_dict_Literal__comment_id____Annotated_int__Gt___' - get: - tags: - - projects - - comments - summary: List Project Comments - description: Retrieve all comments for a specific project. - operationId: list_project_comments - deprecated: true - parameters: - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - - name: limit - in: query - required: false - schema: - type: integer - default: 20 - title: Limit - - name: offset - in: query - required: false - schema: - type: integer - minimum: 0 - default: 0 - title: Offset - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_list_ProjectsCommentsAPI__' - /v0/projects/{project_uuid}/comments/{comment_id}: - put: - tags: - - projects - - comments - summary: Update Project Comment - description: Update the contents of a specific comment for a project. The request - body should contain the updated comment contents. - operationId: update_project_comment - deprecated: true - parameters: - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - - name: comment_id - in: path - required: true - schema: - type: integer - exclusiveMinimum: true - title: Comment Id - minimum: 0 - requestBody: - required: true - content: - application/json: - schema: - $ref: '#/components/schemas/_ProjectCommentsBodyParams' - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_ProjectsCommentsAPI_' - delete: - tags: - - projects - - comments - summary: Delete Project Comment - description: Delete a specific comment associated with a project. - operationId: delete_project_comment - deprecated: true - parameters: - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - - name: comment_id - in: path - required: true - schema: - type: integer - exclusiveMinimum: true - title: Comment Id - minimum: 0 - responses: - '204': - description: Successful Response - get: - tags: - - projects - - comments - summary: Get Project Comment - description: Retrieve a specific comment by its ID within a project. - operationId: get_project_comment - deprecated: true - parameters: - - name: project_uuid - in: path - required: true - schema: - type: string - format: uuid - title: Project Uuid - - name: comment_id - in: path - required: true - schema: - type: integer - exclusiveMinimum: true - title: Comment Id - minimum: 0 - responses: - '200': - description: Successful Response - content: - application/json: - schema: - $ref: '#/components/schemas/Envelope_ProjectsCommentsAPI_' /v0/projects/{project_id}/conversations: post: tags: @@ -10300,19 +10134,6 @@ components: title: Error type: object title: Envelope[ProjectState] - Envelope_ProjectsCommentsAPI_: - properties: - data: - anyOf: - - $ref: '#/components/schemas/ProjectsCommentsAPI' - - type: 'null' - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[ProjectsCommentsAPI] Envelope_RegisterPhoneNextPage_: properties: data: @@ -10607,26 +10428,6 @@ components: title: Error type: object title: Envelope[dict[Annotated[str, StringConstraints], ImageResources]] - Envelope_dict_Literal__comment_id____Annotated_int__Gt___: - properties: - data: - anyOf: - - additionalProperties: - type: integer - exclusiveMinimum: true - minimum: 0 - propertyNames: - const: comment_id - type: object - - type: 'null' - title: Data - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[dict[Literal['comment_id'], Annotated[int, Gt]]] Envelope_dict_UUID__Activity__: properties: data: @@ -10904,22 +10705,6 @@ components: title: Error type: object title: Envelope[list[ProjectMetadataPortGet]] - Envelope_list_ProjectsCommentsAPI__: - properties: - data: - anyOf: - - items: - $ref: '#/components/schemas/ProjectsCommentsAPI' - type: array - - type: 'null' - title: Data - error: - anyOf: - - {} - - type: 'null' - title: Error - type: object - title: Envelope[list[ProjectsCommentsAPI]] Envelope_list_ResourceHit__: properties: data: @@ -14992,49 +14777,6 @@ components: - template - user title: ProjectTypeAPI - ProjectsCommentsAPI: - properties: - comment_id: - type: integer - exclusiveMinimum: true - title: Comment Id - description: Primary key, identifies the comment - minimum: 0 - project_uuid: - type: string - format: uuid - title: Project Uuid - description: project reference for this table - user_id: - type: integer - exclusiveMinimum: true - title: User Id - description: user reference for this table - minimum: 0 - contents: - type: string - title: Contents - description: Contents of the comment - created: - type: string - format: date-time - title: Created - description: Timestamp on creation - modified: - type: string - format: date-time - title: Modified - description: Timestamp with last update - additionalProperties: false - type: object - required: - - comment_id - - project_uuid - - user_id - - contents - - created - - modified - title: ProjectsCommentsAPI ProjectsGroupsBodyParams: properties: read: @@ -17264,16 +17006,6 @@ components: title: Expiration 2Fa type: object title: _PageParams - _ProjectCommentsBodyParams: - properties: - contents: - type: string - title: Contents - additionalProperties: false - type: object - required: - - contents - title: _ProjectCommentsBodyParams _ProjectConversationMessagesCreateBodyParams: properties: content: diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_repository.py b/services/web/server/src/simcore_service_webserver/projects/_comments_repository.py deleted file mode 100644 index 1a871f12ed3..00000000000 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_repository.py +++ /dev/null @@ -1,98 +0,0 @@ -import logging - -from aiopg.sa.result import ResultProxy -from models_library.projects import ProjectID -from models_library.projects_comments import CommentID, ProjectsCommentsDB -from models_library.users import UserID -from pydantic import TypeAdapter -from pydantic.types import PositiveInt -from simcore_postgres_database.models.projects_comments import projects_comments -from sqlalchemy import func, literal_column -from sqlalchemy.sql import select - -_logger = logging.getLogger(__name__) - - -async def create_project_comment( - conn, project_uuid: ProjectID, user_id: UserID, contents: str -) -> CommentID: - project_comment_id: ResultProxy = await conn.execute( - projects_comments.insert() - .values( - project_uuid=project_uuid, - user_id=user_id, - contents=contents, - modified=func.now(), - ) - .returning(projects_comments.c.comment_id) - ) - result: tuple[PositiveInt] = await project_comment_id.first() - return TypeAdapter(CommentID).validate_python(result[0]) - - -async def list_project_comments( - conn, - project_uuid: ProjectID, - offset: PositiveInt, - limit: int, -) -> list[ProjectsCommentsDB]: - result = [] - project_comment_result: ResultProxy = await conn.execute( - projects_comments.select() - .where(projects_comments.c.project_uuid == f"{project_uuid}") - .order_by(projects_comments.c.created.asc()) - .offset(offset) - .limit(limit) - ) - result = [ - ProjectsCommentsDB.model_validate(row) - for row in await project_comment_result.fetchall() - ] - return result - - -async def total_project_comments( - conn, - project_uuid: ProjectID, -) -> PositiveInt: - project_comment_result: ResultProxy = await conn.execute( - select(func.count()) - .select_from(projects_comments) - .where(projects_comments.c.project_uuid == f"{project_uuid}") - ) - result: tuple[PositiveInt] = await project_comment_result.first() - return result[0] - - -async def update_project_comment( - conn, - comment_id: CommentID, - project_uuid: ProjectID, - contents: str, -) -> ProjectsCommentsDB: - project_comment_result = await conn.execute( - projects_comments.update() - .values( - project_uuid=project_uuid, - contents=contents, - modified=func.now(), - ) - .where(projects_comments.c.comment_id == comment_id) - .returning(literal_column("*")) - ) - result = await project_comment_result.first() - return ProjectsCommentsDB.model_validate(result) - - -async def delete_project_comment(conn, comment_id: CommentID) -> None: - await conn.execute( - projects_comments.delete().where(projects_comments.c.comment_id == comment_id) - ) - - -async def get_project_comment(conn, comment_id: CommentID) -> ProjectsCommentsDB: - project_comment_result = await conn.execute( - projects_comments.select().where(projects_comments.c.comment_id == comment_id) - ) - result = await project_comment_result.first() - return ProjectsCommentsDB.model_validate(result) diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_service.py b/services/web/server/src/simcore_service_webserver/projects/_comments_service.py deleted file mode 100644 index 7999d1e591c..00000000000 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_service.py +++ /dev/null @@ -1,96 +0,0 @@ -import logging - -from aiohttp import web -from models_library.projects import ProjectID -from models_library.projects_comments import ( - CommentID, - ProjectsCommentsAPI, - ProjectsCommentsDB, -) -from models_library.users import UserID -from pydantic import PositiveInt - -from ._projects_repository_legacy import APP_PROJECT_DBAPI, ProjectDBAPI - -log = logging.getLogger(__name__) - - -# -# PROJECT COMMENTS ------------------------------------------------------------------- -# - - -async def create_project_comment( - request: web.Request, project_uuid: ProjectID, user_id: UserID, contents: str -) -> CommentID: - db: ProjectDBAPI = request.app[APP_PROJECT_DBAPI] - - comment_id: CommentID = await db.create_project_comment( - project_uuid, user_id, contents - ) - return comment_id - - -async def list_project_comments( - request: web.Request, - project_uuid: ProjectID, - offset: PositiveInt, - limit: int, -) -> list[ProjectsCommentsAPI]: - db: ProjectDBAPI = request.app[APP_PROJECT_DBAPI] - - projects_comments_db_model: list[ProjectsCommentsDB] = ( - await db.list_project_comments(project_uuid, offset, limit) - ) - projects_comments_api_model = [ - ProjectsCommentsAPI(**comment.model_dump()) - for comment in projects_comments_db_model - ] - return projects_comments_api_model - - -async def total_project_comments( - request: web.Request, - project_uuid: ProjectID, -) -> PositiveInt: - db: ProjectDBAPI = request.app[APP_PROJECT_DBAPI] - - project_comments_total: PositiveInt = await db.total_project_comments(project_uuid) - return project_comments_total - - -async def update_project_comment( - request: web.Request, - comment_id: CommentID, - project_uuid: ProjectID, - contents: str, -) -> ProjectsCommentsAPI: - db: ProjectDBAPI = request.app[APP_PROJECT_DBAPI] - - projects_comments_db_model: ProjectsCommentsDB = await db.update_project_comment( - comment_id, project_uuid, contents - ) - projects_comments_api_model = ProjectsCommentsAPI( - **projects_comments_db_model.model_dump() - ) - return projects_comments_api_model - - -async def delete_project_comment(request: web.Request, comment_id: CommentID) -> None: - db: ProjectDBAPI = request.app[APP_PROJECT_DBAPI] - - await db.delete_project_comment(comment_id) - - -async def get_project_comment( - request: web.Request, comment_id: CommentID -) -> ProjectsCommentsAPI: - db: ProjectDBAPI = request.app[APP_PROJECT_DBAPI] - - projects_comments_db_model: ProjectsCommentsDB = await db.get_project_comment( - comment_id - ) - projects_comments_api_model = ProjectsCommentsAPI( - **projects_comments_db_model.model_dump() - ) - return projects_comments_api_model diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/comments_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/comments_rest.py deleted file mode 100644 index 183cf1fa3b6..00000000000 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/comments_rest.py +++ /dev/null @@ -1,228 +0,0 @@ -import logging -from typing import Any - -from aiohttp import web -from models_library.projects import ProjectID -from models_library.projects_comments import CommentID -from models_library.rest_pagination import ( - DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, - MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, - Page, -) -from models_library.rest_pagination_utils import paginate_data -from pydantic import BaseModel, ConfigDict, Field, NonNegativeInt -from servicelib.aiohttp import status -from servicelib.aiohttp.requests_validation import ( - parse_request_body_as, - parse_request_path_parameters_as, - parse_request_query_parameters_as, -) -from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON -from servicelib.rest_constants import RESPONSE_MODEL_POLICY - -from ..._meta import API_VTAG as VTAG -from ...login.decorators import login_required -from ...security.decorators import permission_required -from ...utils_aiohttp import envelope_json_response -from .. import _comments_service, _projects_service -from ._rest_exceptions import handle_plugin_requests_exceptions -from ._rest_schemas import RequestContext - -_logger = logging.getLogger(__name__) - -# -# projects/*/comments COLLECTION ------------------------- -# - -routes = web.RouteTableDef() - - -class _ProjectCommentsPathParams(BaseModel): - project_uuid: ProjectID - model_config = ConfigDict(extra="forbid") - - -class _ProjectCommentsWithCommentPathParams(BaseModel): - project_uuid: ProjectID - comment_id: CommentID - model_config = ConfigDict(extra="forbid") - - -class _ProjectCommentsBodyParams(BaseModel): - contents: str - model_config = ConfigDict(extra="forbid") - - -@routes.post( - f"/{VTAG}/projects/{{project_uuid}}/comments", name="create_project_comment" -) -@login_required -@permission_required("project.read") -@handle_plugin_requests_exceptions -async def create_project_comment(request: web.Request): - req_ctx = RequestContext.model_validate(request) - path_params = parse_request_path_parameters_as(_ProjectCommentsPathParams, request) - body_params = await parse_request_body_as(_ProjectCommentsBodyParams, request) - - # ensure the project exists - await _projects_service.get_project_for_user( - request.app, - project_uuid=f"{path_params.project_uuid}", - user_id=req_ctx.user_id, - include_state=False, - ) - - comment_id = await _comments_service.create_project_comment( - request=request, - project_uuid=path_params.project_uuid, - user_id=req_ctx.user_id, - contents=body_params.contents, - ) - - return envelope_json_response({"comment_id": comment_id}, web.HTTPCreated) - - -class _ListProjectCommentsQueryParams(BaseModel): - limit: int = Field( - default=DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, - description="maximum number of items to return (pagination)", - ge=1, - lt=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, - ) - offset: NonNegativeInt = Field( - default=0, description="index to the first item to return (pagination)" - ) - model_config = ConfigDict(extra="forbid") - - -@routes.get(f"/{VTAG}/projects/{{project_uuid}}/comments", name="list_project_comments") -@login_required -@permission_required("project.read") -@handle_plugin_requests_exceptions -async def list_project_comments(request: web.Request): - req_ctx = RequestContext.model_validate(request) - path_params = parse_request_path_parameters_as(_ProjectCommentsPathParams, request) - query_params: _ListProjectCommentsQueryParams = parse_request_query_parameters_as( - _ListProjectCommentsQueryParams, request - ) - - # ensure the project exists - await _projects_service.get_project_for_user( - request.app, - project_uuid=f"{path_params.project_uuid}", - user_id=req_ctx.user_id, - include_state=False, - ) - - total_project_comments = await _comments_service.total_project_comments( - request=request, - project_uuid=path_params.project_uuid, - ) - - project_comments = await _comments_service.list_project_comments( - request=request, - project_uuid=path_params.project_uuid, - offset=query_params.offset, - limit=query_params.limit, - ) - - page = Page[dict[str, Any]].model_validate( - paginate_data( - chunk=project_comments, - request_url=request.url, - total=total_project_comments, - limit=query_params.limit, - offset=query_params.offset, - ) - ) - return web.Response( - text=page.model_dump_json(**RESPONSE_MODEL_POLICY), - content_type=MIMETYPE_APPLICATION_JSON, - ) - - -@routes.put( - f"/{VTAG}/projects/{{project_uuid}}/comments/{{comment_id}}", - name="update_project_comment", -) -@login_required -@permission_required("project.read") -@handle_plugin_requests_exceptions -async def update_project_comment(request: web.Request): - req_ctx = RequestContext.model_validate(request) - path_params = parse_request_path_parameters_as( - _ProjectCommentsWithCommentPathParams, request - ) - body_params = await parse_request_body_as(_ProjectCommentsBodyParams, request) - - # ensure the project exists - await _projects_service.get_project_for_user( - request.app, - project_uuid=f"{path_params.project_uuid}", - user_id=req_ctx.user_id, - include_state=False, - ) - - updated_comment = await _comments_service.update_project_comment( - request=request, - comment_id=path_params.comment_id, - project_uuid=path_params.project_uuid, - contents=body_params.contents, - ) - return envelope_json_response(updated_comment) - - -@routes.delete( - f"/{VTAG}/projects/{{project_uuid}}/comments/{{comment_id}}", - name="delete_project_comment", -) -@login_required -@permission_required("project.read") -@handle_plugin_requests_exceptions -async def delete_project_comment(request: web.Request): - req_ctx = RequestContext.model_validate(request) - path_params = parse_request_path_parameters_as( - _ProjectCommentsWithCommentPathParams, request - ) - - # ensure the project exists - await _projects_service.get_project_for_user( - request.app, - project_uuid=f"{path_params.project_uuid}", - user_id=req_ctx.user_id, - include_state=False, - ) - - await _comments_service.delete_project_comment( - request=request, - comment_id=path_params.comment_id, - ) - return web.json_response(status=status.HTTP_204_NO_CONTENT) - - -@routes.get( - f"/{VTAG}/projects/{{project_uuid}}/comments/{{comment_id}}", - name="get_project_comment", -) -@login_required -@permission_required("project.read") -@handle_plugin_requests_exceptions -async def get_project_comment(request: web.Request): - req_ctx = RequestContext.model_validate(request) - path_params = parse_request_path_parameters_as( - _ProjectCommentsWithCommentPathParams, request - ) - - # ensure the project exists - await _projects_service.get_project_for_user( - request.app, - project_uuid=f"{path_params.project_uuid}", - user_id=req_ctx.user_id, - include_state=False, - ) - - comment = await _comments_service.get_project_comment( - request=request, - comment_id=path_params.comment_id, - ) - return envelope_json_response(comment) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py index f6df80d866a..1b12ed99f03 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository_legacy.py @@ -20,7 +20,6 @@ from models_library.groups import GroupID from models_library.products import ProductName from models_library.projects import ProjectID, ProjectIDStr -from models_library.projects_comments import CommentID, ProjectsCommentsDB from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.resource_tracker import ( @@ -72,14 +71,6 @@ from tenacity.retry import retry_if_exception_type from ..utils import now_str -from ._comments_repository import ( - create_project_comment, - delete_project_comment, - get_project_comment, - list_project_comments, - total_project_comments, - update_project_comment, -) from ._projects_repository import PROJECT_DB_COLS from ._projects_repository_legacy_utils import ( ANY_USER_ID_SENTINEL, @@ -1337,51 +1328,6 @@ async def remove_tag( project["tags"].remove(tag_id) return convert_to_schema_names(project, user_email) - # - # Project Comments - # - - async def create_project_comment( - self, project_uuid: ProjectID, user_id: UserID, contents: str - ) -> CommentID: - async with self.engine.acquire() as conn: - return await create_project_comment(conn, project_uuid, user_id, contents) - - async def list_project_comments( - self, - project_uuid: ProjectID, - offset: PositiveInt, - limit: int, - ) -> list[ProjectsCommentsDB]: - async with self.engine.acquire() as conn: - return await list_project_comments(conn, project_uuid, offset, limit) - - async def total_project_comments( - self, - project_uuid: ProjectID, - ) -> PositiveInt: - async with self.engine.acquire() as conn: - return await total_project_comments(conn, project_uuid) - - async def update_project_comment( - self, - comment_id: CommentID, - project_uuid: ProjectID, - contents: str, - ) -> ProjectsCommentsDB: - async with self.engine.acquire() as conn: - return await update_project_comment( - conn, comment_id, project_uuid, contents - ) - - async def delete_project_comment(self, comment_id: CommentID) -> None: - async with self.engine.acquire() as conn: - return await delete_project_comment(conn, comment_id) - - async def get_project_comment(self, comment_id: CommentID) -> ProjectsCommentsDB: - async with self.engine.acquire() as conn: - return await get_project_comment(conn, comment_id) - # # Project Wallet # diff --git a/services/web/server/src/simcore_service_webserver/projects/plugin.py b/services/web/server/src/simcore_service_webserver/projects/plugin.py index 5028739d881..51a47294ffb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/plugin.py +++ b/services/web/server/src/simcore_service_webserver/projects/plugin.py @@ -13,7 +13,6 @@ from ..rabbitmq import setup_rabbitmq from ._controller import ( access_rights_rest, - comments_rest, conversations_rest, folders_rest, metadata_rest, @@ -62,7 +61,6 @@ def setup_projects(app: web.Application) -> bool: # setup REST-controllers app.router.add_routes(projects_states_rest.routes) app.router.add_routes(projects_rest.routes) - app.router.add_routes(comments_rest.routes) app.router.add_routes(conversations_rest.routes) app.router.add_routes(access_rights_rest.routes) app.router.add_routes(metadata_rest.routes) From 6dc633803a19793c58959206546719f1962a6fd8 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Wed, 21 May 2025 13:48:53 +0200 Subject: [PATCH 2/2] migration --- api/specs/web-server/_computations.py | 10 +- .../api_schemas_webserver/computations.py | 15 +- .../src/models_library/projects_metadata.py | 17 ++ ...8719855b_add_index_to_projects_metadata.py | 34 +++ .../models/projects_metadata.py | 8 +- .../director_v2/computations.py | 8 +- .../api/rpc/_computations.py | 37 ++- .../modules/db/repositories/comp_runs.py | 8 +- .../db/repositories/comp_tasks/_core.py | 8 +- .../test_api_rpc_computations.py | 4 +- .../api/v0/openapi.yaml | 14 + .../director_v2/_computations_service.py | 72 ++++- .../_computations_service_utils.py | 247 ------------------ .../_controller/computations_rest.py | 18 +- .../projects/_metadata_repository.py | 52 ++++ .../projects/_metadata_service.py | 8 + .../projects/projects_metadata_service.py | 10 +- 17 files changed, 271 insertions(+), 299 deletions(-) create mode 100644 packages/models-library/src/models_library/projects_metadata.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/4e7d8719855b_add_index_to_projects_metadata.py delete mode 100644 services/web/server/src/simcore_service_webserver/director_v2/_computations_service_utils.py diff --git a/api/specs/web-server/_computations.py b/api/specs/web-server/_computations.py index b40c2550a16..fb60fce4175 100644 --- a/api/specs/web-server/_computations.py +++ b/api/specs/web-server/_computations.py @@ -6,9 +6,10 @@ from models_library.api_schemas_webserver.computations import ( ComputationGet, ComputationPathParams, + ComputationRunIterationsLatestListQueryParams, + ComputationRunIterationsListQueryParams, ComputationRunPathParams, ComputationRunRestGet, - ComputationRunWithFiltersListQueryParams, ComputationStart, ComputationStarted, ComputationTaskRestGet, @@ -16,7 +17,6 @@ from models_library.generics import Envelope from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.director_v2._controller.computations_rest import ( - ComputationRunListQueryParams, ComputationTaskListQueryParams, ComputationTaskPathParams, ) @@ -71,7 +71,9 @@ async def stop_computation(_path: Annotated[ComputationPathParams, Depends()]): response_model=Page[ComputationRunRestGet], ) async def list_computations_latest_iteration( - _query: Annotated[as_query(ComputationRunWithFiltersListQueryParams), Depends()], + _query: Annotated[ + as_query(ComputationRunIterationsLatestListQueryParams), Depends() + ], ): ... @@ -80,7 +82,7 @@ async def list_computations_latest_iteration( response_model=Page[ComputationRunRestGet], ) async def list_computation_iterations( - _query: Annotated[as_query(ComputationRunListQueryParams), Depends()], + _query: Annotated[as_query(ComputationRunIterationsListQueryParams), Depends()], _path: Annotated[ComputationRunPathParams, Depends()], ): ... diff --git a/packages/models-library/src/models_library/api_schemas_webserver/computations.py b/packages/models-library/src/models_library/api_schemas_webserver/computations.py index a1f955b20fe..a3de7285621 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/computations.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/computations.py @@ -84,13 +84,20 @@ class ComputationRunListQueryParams( ): ... -class ComputationRunWithFiltersListQueryParams(ComputationRunListQueryParams): +class ComputationRunIterationsLatestListQueryParams(ComputationRunListQueryParams): filter_only_running: bool = Field( default=False, description="If true, only running computations are returned", ) +class ComputationRunIterationsListQueryParams(ComputationRunListQueryParams): + include_children: bool = Field( + default=False, + description="If true, all tasks of the project and its children are returned (Currently supported only for root projects)", + ) + + class ComputationRunRestGet(OutputSchema): project_uuid: ProjectID iteration: int @@ -128,7 +135,11 @@ class ComputationTaskPathParams(BaseModel): class ComputationTaskListQueryParams( PageQueryParameters, ComputationTaskListOrderParams, # type: ignore[misc, valid-type] -): ... +): + include_children: bool = Field( + default=False, + description="If true, all tasks of the project and its children are returned (Currently supported only for root projects)", + ) class ComputationTaskRestGet(OutputSchema): diff --git a/packages/models-library/src/models_library/projects_metadata.py b/packages/models-library/src/models_library/projects_metadata.py new file mode 100644 index 00000000000..0290ff03d0d --- /dev/null +++ b/packages/models-library/src/models_library/projects_metadata.py @@ -0,0 +1,17 @@ +from datetime import datetime +from typing import Any + +from pydantic import BaseModel + +from .projects import ProjectID + + +class ProjectsMetadataDBGet(BaseModel): + project_uuid: ProjectID + custom: dict[str, Any] + created: datetime + modified: datetime + parent_project_uuid: ProjectID + parent_node_id: ProjectID + root_parent_project_uuid: ProjectID + root_parent_node_id: ProjectID diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/4e7d8719855b_add_index_to_projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/4e7d8719855b_add_index_to_projects_metadata.py new file mode 100644 index 00000000000..75ce061eddc --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/4e7d8719855b_add_index_to_projects_metadata.py @@ -0,0 +1,34 @@ +"""add index to projects_metadata + +Revision ID: 4e7d8719855b +Revises: e98c45ff314f +Create Date: 2025-05-21 11:48:34.062860+00:00 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "4e7d8719855b" +down_revision = "e98c45ff314f" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index( + "idx_projects_metadata_root_parent_project_uuid", + "projects_metadata", + ["root_parent_project_uuid"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + "idx_projects_metadata_root_parent_project_uuid", table_name="projects_metadata" + ) + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py index c5379f407d4..3e6ed034a96 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_metadata.py @@ -1,6 +1,6 @@ """ - These tables were designed to be controled by projects-plugin in - the webserver's service +These tables were designed to be controled by projects-plugin in +the webserver's service """ import sqlalchemy as sa @@ -100,6 +100,10 @@ ondelete=RefActions.SET_NULL, name="fk_projects_metadata_root_parent_node_id", ), + ####### + sa.Index( + "idx_projects_metadata_root_parent_project_uuid", "root_parent_project_uuid" + ), ) diff --git a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py index 24720ecd512..a24ed19aba9 100644 --- a/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py +++ b/packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/director_v2/computations.py @@ -64,7 +64,7 @@ async def list_computations_iterations_page( *, product_name: ProductName, user_id: UserID, - project_id: ProjectID, + project_ids: list[ProjectID], # pagination offset: int = 0, limit: int = 20, @@ -76,7 +76,7 @@ async def list_computations_iterations_page( _RPC_METHOD_NAME_ADAPTER.validate_python("list_computations_iterations_page"), product_name=product_name, user_id=user_id, - project_id=project_id, + project_ids=project_ids, offset=offset, limit=limit, order_by=order_by, @@ -92,7 +92,7 @@ async def list_computations_latest_iteration_tasks_page( *, product_name: ProductName, user_id: UserID, - project_id: ProjectID, + project_ids: list[ProjectID], # pagination offset: int = 0, limit: int = 20, @@ -106,7 +106,7 @@ async def list_computations_latest_iteration_tasks_page( ), product_name=product_name, user_id=user_id, - project_id=project_id, + project_ids=project_ids, offset=offset, limit=limit, order_by=order_by, diff --git a/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py b/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py index d0caa15a869..183f2920387 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py +++ b/services/director-v2/src/simcore_service_director_v2/api/rpc/_computations.py @@ -59,7 +59,7 @@ async def list_computations_iterations_page( *, product_name: ProductName, user_id: UserID, - project_id: ProjectID, + project_ids: list[ProjectID], # pagination offset: int = 0, limit: int = 20, @@ -71,7 +71,7 @@ async def list_computations_iterations_page( await comp_runs_repo.list_for_user_and_project_all_iterations( product_name=product_name, user_id=user_id, - project_id=project_id, + project_ids=project_ids, offset=offset, limit=limit, order_by=order_by, @@ -84,12 +84,12 @@ async def list_computations_iterations_page( async def _fetch_task_log( - user_id: UserID, project_id: ProjectID, task: ComputationTaskForRpcDBGet + user_id: UserID, task: ComputationTaskForRpcDBGet ) -> TaskLogFileGet | None: if not task.state.is_running(): return await dask_utils.get_task_log_file( user_id=user_id, - project_id=project_id, + project_id=task.project_uuid, node_id=task.node_id, ) return None @@ -101,7 +101,7 @@ async def list_computations_latest_iteration_tasks_page( *, product_name: ProductName, user_id: UserID, - project_id: ProjectID, + project_ids: list[ProjectID], # pagination offset: int = 0, limit: int = 20, @@ -114,20 +114,30 @@ async def list_computations_latest_iteration_tasks_page( comp_tasks_repo = CompTasksRepository.instance(db_engine=app.state.engine) comp_runs_repo = CompRunsRepository.instance(db_engine=app.state.engine) - comp_latest_run = await comp_runs_repo.get( - user_id=user_id, project_id=project_id, iteration=None # Returns last iteration - ) - total, comp_tasks = await comp_tasks_repo.list_computational_tasks_rpc_domain( - project_id=project_id, + project_ids=project_ids, offset=offset, limit=limit, order_by=order_by, ) + # Get unique set of all project_uuids from comp_tasks + unique_project_uuids = {task.project_uuid for task in comp_tasks} + + # Fetch latest run for each project concurrently + latest_runs = await limited_gather( + *[ + comp_runs_repo.get(user_id=user_id, project_id=project_uuid, iteration=None) + for project_uuid in unique_project_uuids + ], + limit=20, + ) + # Build a dict: project_uuid -> iteration + project_uuid_to_iteration = {run.project_uuid: run.iteration for run in latest_runs} + # Run all log fetches concurrently log_files = await limited_gather( - *[_fetch_task_log(user_id, project_id, task) for task in comp_tasks], + *[_fetch_task_log(user_id, task) for task in comp_tasks], limit=20, ) @@ -142,7 +152,10 @@ async def list_computations_latest_iteration_tasks_page( ended_at=task.ended_at, log_download_link=log_file.download_link if log_file else None, service_run_id=ServiceRunID.get_resource_tracking_run_id_for_computational( - user_id, project_id, task.node_id, comp_latest_run.iteration + user_id, + task.project_uuid, + task.node_id, + project_uuid_to_iteration[task.project_uuid], ), ) for task, log_file in zip(comp_tasks, log_files, strict=True) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py index eb706af4b83..511b53fe1c0 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_runs.py @@ -295,7 +295,7 @@ async def list_for_user_and_project_all_iterations( *, product_name: str, user_id: UserID, - project_id: ProjectID, + project_ids: list[ProjectID], # pagination offset: int, limit: int, @@ -309,7 +309,11 @@ async def list_for_user_and_project_all_iterations( *self._COMPUTATION_RUNS_RPC_GET_COLUMNS, ).where( (comp_runs.c.user_id == user_id) - & (comp_runs.c.project_uuid == f"{project_id}") + & ( + comp_runs.c.project_uuid.in_( + [f"{project_id}" for project_id in project_ids] + ) + ) & ( comp_runs.c.metadata["product_name"].astext == product_name ) # <-- NOTE: We might create a separate column for this for fast retrieval diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_core.py index 8c32f978e2d..67f19db7e0e 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_core.py @@ -78,7 +78,7 @@ async def list_computational_tasks( async def list_computational_tasks_rpc_domain( self, *, - project_id: ProjectID, + project_ids: list[ProjectID], # pagination offset: int = 0, limit: int = 20, @@ -100,7 +100,11 @@ async def list_computational_tasks_rpc_domain( ) .select_from(comp_tasks) .where( - (comp_tasks.c.project_id == f"{project_id}") + ( + comp_tasks.c.project_id.in_( + [f"{project_id}" for project_id in project_ids] + ) + ) & (comp_tasks.c.node_class == NodeClass.COMPUTATIONAL) ) ) diff --git a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_rpc_computations.py b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_rpc_computations.py index 54c91a752eb..910679901e3 100644 --- a/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_rpc_computations.py +++ b/services/director-v2/tests/unit/with_dbs/comp_scheduler/test_api_rpc_computations.py @@ -97,7 +97,7 @@ async def test_rpc_list_computation_runs_and_tasks( # Tasks output = await rpc_computations.list_computations_latest_iteration_tasks_page( - rpc_client, product_name="osparc", user_id=user["id"], project_id=proj.uuid + rpc_client, product_name="osparc", user_id=user["id"], project_ids=[proj.uuid] ) assert output assert output.total == 4 @@ -201,7 +201,7 @@ async def test_rpc_list_computation_runs_history( ) output = await rpc_computations.list_computations_iterations_page( - rpc_client, product_name="osparc", user_id=user["id"], project_id=proj.uuid + rpc_client, product_name="osparc", user_id=user["id"], project_ids=[proj.uuid] ) assert output.total == 3 assert isinstance(output, ComputationRunRpcGetPage) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 19d128e51c5..f2b02a2ea20 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2619,6 +2619,13 @@ paths: type: integer default: 0 title: Offset + - name: include_children + in: query + required: false + schema: + type: boolean + default: false + title: Include Children responses: '200': description: Successful Response @@ -2664,6 +2671,13 @@ paths: type: integer default: 0 title: Offset + - name: include_children + in: query + required: false + schema: + type: boolean + default: false + title: Include Children responses: '200': description: Successful Response diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py index fac04998d1d..d5ca46fbc83 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service.py @@ -32,6 +32,7 @@ ) from ..projects.projects_metadata_service import ( get_project_custom_metadata_or_empty_dict, + get_project_uuids_by_root_parent_project_id, ) from ..rabbitmq import get_rabbitmq_rpc_client @@ -120,9 +121,12 @@ async def list_computations_latest_iteration( async def list_computation_iterations( app: web.Application, + *, product_name: ProductName, user_id: UserID, project_id: ProjectID, + # filters + include_children: bool = False, # pagination offset: int, limit: NonNegativeInt, @@ -130,23 +134,37 @@ async def list_computation_iterations( order_by: OrderBy, ) -> tuple[int, list[ComputationRunWithAttributes]]: """Returns the list of computations for a specific project (all iterations)""" + await check_user_project_permission( + app, project_id=project_id, user_id=user_id, product_name=product_name + ) + + if include_children: + child_projects = await get_project_uuids_by_root_parent_project_id( + app, root_parent_project_uuid=project_id + ) + child_projects_with_root = [*child_projects, project_id] + else: + child_projects_with_root = [project_id] + rpc_client = get_rabbitmq_rpc_client(app) _runs_get = await computations.list_computations_iterations_page( rpc_client, product_name=product_name, user_id=user_id, - project_id=project_id, + project_ids=child_projects_with_root, offset=offset, limit=limit, order_by=order_by, ) + # NOTE: MD: can be improved, many times we ask for the same project # Get projects metadata - _projects_metadata = await _get_projects_metadata(app, project_uuids=[project_id]) - assert len(_projects_metadata) == 1 # nosec + _projects_metadata = await _get_projects_metadata( + app, project_uuids=[item.project_uuid for item in _runs_get.items] + ) # Get Root project names - _projects_root_names = await _get_root_project_names(app, [_runs_get.items[0]]) - assert len(_projects_root_names) == 1 # nosec + root_project_names = await batch_get_project_name(app, projects_uuids=[project_id]) + assert len(root_project_names) == 1 _computational_runs_output = [ ComputationRunWithAttributes( @@ -157,10 +175,12 @@ async def list_computation_iterations( submitted_at=item.submitted_at, started_at=item.started_at, ended_at=item.ended_at, - root_project_name=_projects_root_names[0], - project_custom_metadata=_projects_metadata[0], + root_project_name=root_project_names[0], + project_custom_metadata=project_metadata, + ) + for item, project_metadata in zip( + _runs_get.items, _projects_metadata, strict=True ) - for item in _runs_get.items ] return _runs_get.total, _computational_runs_output @@ -181,9 +201,12 @@ async def _get_credits_or_zero_by_service_run_id( async def list_computations_latest_iteration_tasks( app: web.Application, + *, product_name: ProductName, user_id: UserID, project_id: ProjectID, + # filters + include_children: bool = False, # pagination offset: int, limit: NonNegativeInt, @@ -191,25 +214,44 @@ async def list_computations_latest_iteration_tasks( order_by: OrderBy, ) -> tuple[int, list[ComputationTaskWithAttributes]]: """Returns the list of tasks for the latest iteration of a computation""" - await check_user_project_permission( app, project_id=project_id, user_id=user_id, product_name=product_name ) + if include_children: + child_projects = await get_project_uuids_by_root_parent_project_id( + app, root_parent_project_uuid=project_id + ) + child_projects_with_root = [*child_projects, project_id] + else: + child_projects_with_root = [project_id] + rpc_client = get_rabbitmq_rpc_client(app) _tasks_get = await computations.list_computations_latest_iteration_tasks_page( rpc_client, product_name=product_name, user_id=user_id, - project_id=project_id, + project_ids=child_projects_with_root, offset=offset, limit=limit, order_by=order_by, ) - # Get node names (for all project nodes) - project_dict = await get_project_dict_legacy(app, project_uuid=project_id) - workbench = project_dict["workbench"] + # Get unique set of all project_uuids from comp_tasks + unique_project_uuids = {task.project_uuid for task in _tasks_get.items} + # Fetch projects metadata concurrently + # NOTE: MD: can be improved with a single batch call + project_dicts = await limited_gather( + *[ + get_project_dict_legacy(app, project_uuid=project_uuid) + for project_uuid in unique_project_uuids + ], + limit=20, + ) + # Build a dict: project_uuid -> workbench + project_uuid_to_workbench = { + prj["project_uuid"]: prj["workbench"] for prj in project_dicts + } _service_run_ids = [item.service_run_id for item in _tasks_get.items] _is_product_billable = await is_product_billable(app, product_name=product_name) @@ -239,7 +281,9 @@ async def list_computations_latest_iteration_tasks( started_at=item.started_at, ended_at=item.ended_at, log_download_link=item.log_download_link, - node_name=workbench[f"{item.node_id}"].get("label", ""), + node_name=project_uuid_to_workbench[f"{item.project_uuid}"]["workbench"][ + f"{item.node_id}" + ].get("label", ""), osparc_credits=credits_or_none, ) for item, credits_or_none in zip( diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service_utils.py b/services/web/server/src/simcore_service_webserver/director_v2/_computations_service_utils.py deleted file mode 100644 index f05430a42a8..00000000000 --- a/services/web/server/src/simcore_service_webserver/director_v2/_computations_service_utils.py +++ /dev/null @@ -1,247 +0,0 @@ -from decimal import Decimal - -from aiohttp import web -from models_library.computations import ( - ComputationRunWithAttributes, - ComputationTaskWithAttributes, -) -from models_library.products import ProductName -from models_library.projects import ProjectID -from models_library.rest_ordering import OrderBy -from models_library.users import UserID -from pydantic import NonNegativeInt -from servicelib.rabbitmq.rpc_interfaces.director_v2 import computations -from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker import ( - credit_transactions, -) -from servicelib.utils import limited_gather - -from ..products.products_service import is_product_billable -from ..projects.api import ( - batch_get_project_name, - check_user_project_permission, - get_project_dict_legacy, -) -from ..projects.projects_metadata_service import ( - get_project_custom_metadata_or_empty_dict, -) -from ..rabbitmq import get_rabbitmq_rpc_client - - -async def _fetch_projects_metadata( - app: web.Application, - project_uuids: list[ProjectID], # Using str instead of ProjectID for compatibility -) -> list[dict]: - """Batch fetch project metadata with concurrency control""" - return await limited_gather( - *[ - get_project_custom_metadata_or_empty_dict(app, project_uuid=uuid) - for uuid in project_uuids - ], - limit=20, - ) - - -async def list_computations_latest_iteration( - app: web.Application, - product_name: ProductName, - user_id: UserID, - # filters - filter_only_running: bool, # noqa: FBT001 - # pagination - offset: int, - limit: NonNegativeInt, - # ordering - order_by: OrderBy, -) -> tuple[int, list[ComputationRunWithAttributes]]: - """Returns the list of computations (only latest iterations)""" - rpc_client = get_rabbitmq_rpc_client(app) - _runs_get = await computations.list_computations_latest_iteration_page( - rpc_client, - product_name=product_name, - user_id=user_id, - filter_only_running=filter_only_running, - offset=offset, - limit=limit, - order_by=order_by, - ) - - # Get projects metadata (NOTE: MD: can be improved with a single batch call) - _projects_metadata = await _fetch_projects_metadata( - app, project_uuids=[item.project_uuid for item in _runs_get.items] - ) - - # Get Root project names - _projects_root_uuids: list[ProjectID] = [] - for item in _runs_get.items: - if ( - prj_root_id := item.info.get("project_metadata", {}).get( - "root_parent_project_id", None - ) - ) is not None: - _projects_root_uuids.append(ProjectID(prj_root_id)) - else: - _projects_root_uuids.append(item.project_uuid) - - _projects_root_names = await batch_get_project_name( - app, projects_uuids=_projects_root_uuids - ) - - _computational_runs_output = [ - ComputationRunWithAttributes( - project_uuid=item.project_uuid, - iteration=item.iteration, - state=item.state, - info=item.info, - submitted_at=item.submitted_at, - started_at=item.started_at, - ended_at=item.ended_at, - root_project_name=project_name, - project_custom_metadata=project_metadata, - ) - for item, project_metadata, project_name in zip( - _runs_get.items, _projects_metadata, _projects_root_names, strict=True - ) - ] - - return _runs_get.total, _computational_runs_output - - -async def list_computation_iterations( - app: web.Application, - product_name: ProductName, - user_id: UserID, - project_id: ProjectID, - # pagination - offset: int, - limit: NonNegativeInt, - # ordering - order_by: OrderBy, -) -> tuple[int, list[ComputationRunWithAttributes]]: - """Returns the list of computations (only latest iterations)""" - rpc_client = get_rabbitmq_rpc_client(app) - _runs_get = await computations.list_computations_iterations_page( - rpc_client, - product_name=product_name, - user_id=user_id, - project_id=project_id, - offset=offset, - limit=limit, - order_by=order_by, - ) - - # Get projects metadata (NOTE: MD: can be improved with a single batch call) - _projects_metadata = await limited_gather( - *[ - get_project_custom_metadata_or_empty_dict( - app, project_uuid=item.project_uuid - ) - for item in _runs_get.items - ], - limit=20, - ) - - # Get Root project names - _projects_root_uuids: list[ProjectID] = [] - for item in _runs_get.items: - if ( - prj_root_id := item.info.get("project_metadata", {}).get( - "root_parent_project_id", None - ) - ) is not None: - _projects_root_uuids.append(ProjectID(prj_root_id)) - else: - _projects_root_uuids.append(item.project_uuid) - - _projects_root_names = await batch_get_project_name( - app, projects_uuids=_projects_root_uuids - ) - - _computational_runs_output = [ - ComputationRunWithAttributes( - project_uuid=item.project_uuid, - iteration=item.iteration, - state=item.state, - info=item.info, - submitted_at=item.submitted_at, - started_at=item.started_at, - ended_at=item.ended_at, - root_project_name=project_name, - project_custom_metadata=project_metadata, - ) - for item, project_metadata, project_name in zip( - _runs_get.items, _projects_metadata, _projects_root_names, strict=True - ) - ] - - return _runs_get.total, _computational_runs_output - - -async def list_computations_latest_iteration_tasks( - app: web.Application, - product_name: ProductName, - user_id: UserID, - project_id: ProjectID, - # pagination - offset: int, - limit: NonNegativeInt, - # ordering - order_by: OrderBy, -) -> tuple[int, list[ComputationTaskWithAttributes]]: - """Returns the list of tasks for the latest iteration of a computation""" - - await check_user_project_permission( - app, project_id=project_id, user_id=user_id, product_name=product_name - ) - - rpc_client = get_rabbitmq_rpc_client(app) - _tasks_get = await computations.list_computations_latest_iteration_tasks_page( - rpc_client, - product_name=product_name, - user_id=user_id, - project_id=project_id, - offset=offset, - limit=limit, - order_by=order_by, - ) - - # Get node names (for all project nodes) - project_dict = await get_project_dict_legacy(app, project_uuid=project_id) - workbench = project_dict["workbench"] - - _service_run_ids = [item.service_run_id for item in _tasks_get.items] - _is_product_billable = await is_product_billable(app, product_name=product_name) - _service_run_osparc_credits: list[Decimal | None] - if _is_product_billable: - # NOTE: MD: can be improved with a single batch call - _service_run_osparc_credits = await limited_gather( - *[ - credit_transactions.get_transaction_current_credits_by_service_run_id( - rpc_client, service_run_id=_run_id - ) - for _run_id in _service_run_ids - ], - limit=20, - ) - else: - _service_run_osparc_credits = [None for _ in _service_run_ids] - - # Final output - _tasks_get_output = [ - ComputationTaskWithAttributes( - project_uuid=item.project_uuid, - node_id=item.node_id, - state=item.state, - progress=item.progress, - image=item.image, - started_at=item.started_at, - ended_at=item.ended_at, - log_download_link=item.log_download_link, - node_name=workbench[f"{item.node_id}"].get("label", ""), - osparc_credits=credits_or_none, - ) - for item, credits_or_none in zip( - _tasks_get.items, _service_run_osparc_credits, strict=True - ) - ] - return _tasks_get.total, _tasks_get_output diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py b/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py index 3f13721b34a..96766fe97af 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_controller/computations_rest.py @@ -2,10 +2,10 @@ from aiohttp import web from models_library.api_schemas_webserver.computations import ( - ComputationRunListQueryParams, + ComputationRunIterationsLatestListQueryParams, + ComputationRunIterationsListQueryParams, ComputationRunPathParams, ComputationRunRestGet, - ComputationRunWithFiltersListQueryParams, ComputationTaskListQueryParams, ComputationTaskPathParams, ComputationTaskRestGet, @@ -51,9 +51,9 @@ class ComputationsRequestContext(RequestParameters): async def list_computations_latest_iteration(request: web.Request) -> web.Response: req_ctx = ComputationsRequestContext.model_validate(request) - query_params: ComputationRunWithFiltersListQueryParams = ( + query_params: ComputationRunIterationsLatestListQueryParams = ( parse_request_query_parameters_as( - ComputationRunWithFiltersListQueryParams, request + ComputationRunIterationsLatestListQueryParams, request ) ) @@ -99,8 +99,10 @@ async def list_computations_latest_iteration(request: web.Request) -> web.Respon async def list_computation_iterations(request: web.Request) -> web.Response: req_ctx = ComputationsRequestContext.model_validate(request) - query_params: ComputationRunListQueryParams = parse_request_query_parameters_as( - ComputationRunListQueryParams, request + query_params: ComputationRunIterationsListQueryParams = ( + parse_request_query_parameters_as( + ComputationRunIterationsListQueryParams, request + ) ) path_params = parse_request_path_parameters_as(ComputationRunPathParams, request) @@ -109,6 +111,8 @@ async def list_computation_iterations(request: web.Request) -> web.Response: product_name=req_ctx.product_name, user_id=req_ctx.user_id, project_id=path_params.project_id, + # filters + include_children=query_params.include_children, # pagination offset=query_params.offset, limit=query_params.limit, @@ -157,6 +161,8 @@ async def list_computations_latest_iteration_tasks( product_name=req_ctx.product_name, user_id=req_ctx.user_id, project_id=path_params.project_id, + # filters + include_children=query_params.include_children, # pagination offset=query_params.offset, limit=query_params.limit, diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_repository.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_repository.py index 2c72a395a5a..97c0f95a493 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_repository.py @@ -2,12 +2,16 @@ from collections.abc import Awaitable, Callable from typing import Any, TypeVar +import sqlalchemy as sa +from aiohttp import web from aiopg.sa.engine import Engine from models_library.api_schemas_webserver.projects_metadata import MetadataDict from models_library.projects import ProjectID +from models_library.projects_metadata import ProjectsMetadataDBGet from models_library.projects_nodes_io import NodeID from pydantic import TypeAdapter from simcore_postgres_database import utils_projects_metadata +from simcore_postgres_database.models.projects_metadata import projects_metadata from simcore_postgres_database.utils_projects_metadata import ( DBProjectInvalidAncestorsError, DBProjectInvalidParentNodeError, @@ -19,7 +23,13 @@ ProjectNodesNonUniqueNodeFoundError, ProjectNodesRepo, ) +from simcore_postgres_database.utils_repos import ( + get_columns_from_db_model, + pass_or_acquire_connection, +) +from sqlalchemy.ext.asyncio import AsyncConnection +from ..db.plugin import get_asyncpg_engine from .exceptions import ( NodeNotFoundError, ParentNodeNotFoundError, @@ -31,6 +41,12 @@ F = TypeVar("F", bound=Callable[..., Awaitable[Any]]) +PROJECT_METADATA_DB_COLS = get_columns_from_db_model( + projects_metadata, + ProjectsMetadataDBGet, +) + + def _handle_projects_metadata_exceptions(fct: F) -> F: """Transforms project errors -> http errors""" @@ -145,3 +161,39 @@ async def set_project_ancestors( parent_project_uuid=parent_project_uuid, parent_node_id=parent_node_id, ) + + +async def get( + app: web.Application, + connection: AsyncConnection | None = None, + *, + project_uuid: ProjectID, +) -> ProjectsMetadataDBGet: + query = sa.select(*PROJECT_METADATA_DB_COLS).where( + projects_metadata.c.project_uuid == f"{project_uuid}" + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute(query) + row = result.one_or_none() + if row is None: + raise ProjectNotFoundError(project_uuid=project_uuid) + return ProjectsMetadataDBGet.model_validate(row) + + +async def get_project_uuids_by_root_parent_project_id( + app: web.Application, + connection: AsyncConnection | None = None, + *, + root_parent_project_uuid: ProjectID, +) -> list[ProjectID]: + stmt = sa.select(projects_metadata.c.project_uuid).where( + projects_metadata.c.root_parent_project_uuid == f"{root_parent_project_uuid}" + ) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.stream(stmt) + output: list[ProjectID] = [ + ProjectID(row["project_uuid"]) async for row in result + ] + return output diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py index 8c6efe8d808..5d890a6c374 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_service.py @@ -116,3 +116,11 @@ async def set_project_ancestors( parent_project_uuid=parent_project_uuid, parent_node_id=parent_node_id, ) + + +async def get_project_uuids_by_root_parent_project_id( + app: web.Application, root_parent_project_uuid: ProjectID +) -> list[ProjectID]: + return await _metadata_repository.get_project_uuids_by_root_parent_project_id( + app=app, root_parent_project_uuid=root_parent_project_uuid + ) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_metadata_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_metadata_service.py index 84934a4edd3..6a82d53ae49 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_metadata_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_metadata_service.py @@ -1,6 +1,12 @@ -from ._metadata_service import get_project_custom_metadata_or_empty_dict +from ._metadata_service import ( + get_project_custom_metadata_or_empty_dict, + get_project_uuids_by_root_parent_project_id, +) -__all__: tuple[str, ...] = ("get_project_custom_metadata_or_empty_dict",) +__all__: tuple[str, ...] = ( + "get_project_custom_metadata_or_empty_dict", + "get_project_uuids_by_root_parent_project_id", +) # nopycln: file