From 5807f81181003b08ee7f6ac331ca62fcfef9f51c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 7 Jan 2025 11:08:42 +0100 Subject: [PATCH 01/55] extract first columns --- .../models/projects_nodes.py | 114 +++++++++++++++++- 1 file changed, 111 insertions(+), 3 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index f4b569270c4b..3644ad775b85 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -38,25 +38,133 @@ ), nullable=False, index=True, - doc="The project unique identifier", + doc="Project unique identifier", ), sa.Column( "node_id", sa.String, nullable=False, index=True, - doc="The node unique identifier", + doc="Node unique identifier", ), sa.Column( "required_resources", JSONB, nullable=False, server_default=sa.text("'{}'::jsonb"), - doc="The node required resources", + doc="Node required resources", ), # TIME STAMPS ---- column_created_datetime(timezone=True), column_modified_datetime(timezone=True), + sa.Column( + "key", + sa.String, + nullable=False, + doc="Distinctive name for the node based on the docker registry path", + ), + sa.Column( + "version", + sa.String, + nullable=False, + doc="Semantic version number of the node", + ), + sa.Column( + "label", + sa.String, + nullable=False, + doc="Short name of the node", + ), + sa.Column( + "progress", + sa.Numeric, + nullable=True, + doc="Node progress value", + ), + sa.Column( + "thumbnail", + sa.String, + nullable=False, + doc="Url of the latest screenshot of the node", + ), + sa.Column( + "inputs", + JSONB, + nullable=True, + doc="Values of input properties", + ), + sa.Column( + "outputs", + JSONB, + nullable=True, + doc="Values of output properties", + ), + sa.Column( + "run_hash", + sa.String, + nullable=True, + doc="HEX digest of the resolved inputs + outputs hash at the time when the last outputs were generated", + ), + sa.Column( + "state", + JSONB, + nullable=True, + doc="Node state", + ), + # sa.Column( + # "inputs_units", + # JSONB, + # nullable=False, + # server_default=sa.text("'{}'::jsonb"), + # doc="Values of input unit", + # ), + # sa.Column( + # "input_access", + # JSONB, + # nullable=False, + # server_default=sa.text("'{}'::jsonb"), + # doc="Map with key - access level pairs", + # ), + # sa.Column( + # "input_nodes", + # JSONB, # <-- ARRAY + # nullable=False, + # server_default=sa.text("'[]'::jsonb"), + # doc="Node IDs of where the node is connected to", + # ), + # sa.Column( + # "output_node", + # sa.BOOLEAN, + # nullable=False, + # doc="Deprecated", + # ), + # sa.Column( + # "output_nodes", + # JSONB, # <-- ARRAY + # nullable=False, + # server_default=sa.text("'[]'::jsonb"), + # doc="Used in group-nodes. Node IDs of those connected to the output", + # ), + # sa.Column( + # "parent", + # sa.String, + # nullable=True, + # doc="Parent's (group-nodes) node IDs.", + # ), + # sa.Column( + # "position", + # JSONB, + # nullable=True, + # server_default=sa.text("'{}'::jsonb"), + # doc="Deprecated", + # ), + # sa.Column( + # "boot_options", + # JSONB, + # nullable=True, + # server_default=sa.text("'{}'::jsonb"), + # doc="Some services provide alternative parameters to be injected at boot time. The user selection should be stored here, and it will overwrite the services's defaults.", + # ), sa.UniqueConstraint("project_uuid", "node_id"), ) From 45732175eb49207314fd64296df779e9cb14d8e1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 7 Jan 2025 11:18:48 +0100 Subject: [PATCH 02/55] add migration script --- .../e11f9d1e3f44_migrate_workbench.py | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/e11f9d1e3f44_migrate_workbench.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e11f9d1e3f44_migrate_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e11f9d1e3f44_migrate_workbench.py new file mode 100644 index 000000000000..c4edf27e8910 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/e11f9d1e3f44_migrate_workbench.py @@ -0,0 +1,53 @@ +"""Migrate workbench + +Revision ID: e11f9d1e3f44 +Revises: 307017ee1a49 +Create Date: 2025-01-07 10:16:43.305789+00:00 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "e11f9d1e3f44" +down_revision = "307017ee1a49" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("projects_nodes", sa.Column("key", sa.String(), nullable=False)) + op.add_column("projects_nodes", sa.Column("version", sa.String(), nullable=False)) + op.add_column("projects_nodes", sa.Column("label", sa.String(), nullable=False)) + op.add_column("projects_nodes", sa.Column("progress", sa.Numeric(), nullable=True)) + op.add_column("projects_nodes", sa.Column("thumbnail", sa.String(), nullable=False)) + op.add_column( + "projects_nodes", + sa.Column("inputs", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + op.add_column( + "projects_nodes", + sa.Column("outputs", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + op.add_column("projects_nodes", sa.Column("run_hash", sa.String(), nullable=True)) + op.add_column( + "projects_nodes", + sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("projects_nodes", "state") + op.drop_column("projects_nodes", "run_hash") + op.drop_column("projects_nodes", "outputs") + op.drop_column("projects_nodes", "inputs") + op.drop_column("projects_nodes", "thumbnail") + op.drop_column("projects_nodes", "progress") + op.drop_column("projects_nodes", "label") + op.drop_column("projects_nodes", "version") + op.drop_column("projects_nodes", "key") + # ### end Alembic commands ### From 05f619be5a642b79582e0aa53692c675c8beaddb Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 7 Jan 2025 11:28:27 +0100 Subject: [PATCH 03/55] update thumbnail --- ...ench.py => 028ada3e3ec9_move_projects_workbench.py} | 10 +++++----- .../simcore_postgres_database/models/projects_nodes.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/migration/versions/{e11f9d1e3f44_migrate_workbench.py => 028ada3e3ec9_move_projects_workbench.py} (92%) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e11f9d1e3f44_migrate_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/028ada3e3ec9_move_projects_workbench.py similarity index 92% rename from packages/postgres-database/src/simcore_postgres_database/migration/versions/e11f9d1e3f44_migrate_workbench.py rename to packages/postgres-database/src/simcore_postgres_database/migration/versions/028ada3e3ec9_move_projects_workbench.py index c4edf27e8910..56d07dbd21ea 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/e11f9d1e3f44_migrate_workbench.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/028ada3e3ec9_move_projects_workbench.py @@ -1,8 +1,8 @@ -"""Migrate workbench +"""Move projects workbench -Revision ID: e11f9d1e3f44 +Revision ID: 028ada3e3ec9 Revises: 307017ee1a49 -Create Date: 2025-01-07 10:16:43.305789+00:00 +Create Date: 2025-01-07 10:25:54.137095+00:00 """ import sqlalchemy as sa @@ -10,7 +10,7 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = "e11f9d1e3f44" +revision = "028ada3e3ec9" down_revision = "307017ee1a49" branch_labels = None depends_on = None @@ -22,7 +22,7 @@ def upgrade(): op.add_column("projects_nodes", sa.Column("version", sa.String(), nullable=False)) op.add_column("projects_nodes", sa.Column("label", sa.String(), nullable=False)) op.add_column("projects_nodes", sa.Column("progress", sa.Numeric(), nullable=True)) - op.add_column("projects_nodes", sa.Column("thumbnail", sa.String(), nullable=False)) + op.add_column("projects_nodes", sa.Column("thumbnail", sa.String(), nullable=True)) op.add_column( "projects_nodes", sa.Column("inputs", postgresql.JSONB(astext_type=sa.Text()), nullable=True), diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index 3644ad775b85..d9774f5e95af 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -84,7 +84,7 @@ sa.Column( "thumbnail", sa.String, - nullable=False, + nullable=True, doc="Url of the latest screenshot of the node", ), sa.Column( From 4c566adb11ca3824fb9b2e2ea989cba8904d2611 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 7 Jan 2025 13:21:14 +0100 Subject: [PATCH 04/55] update script --- ...> 1d74a6de5e36_move_projects_workbench.py} | 37 ++++++++++++++++--- 1 file changed, 31 insertions(+), 6 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/migration/versions/{028ada3e3ec9_move_projects_workbench.py => 1d74a6de5e36_move_projects_workbench.py} (66%) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/028ada3e3ec9_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py similarity index 66% rename from packages/postgres-database/src/simcore_postgres_database/migration/versions/028ada3e3ec9_move_projects_workbench.py rename to packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py index 56d07dbd21ea..2461703d6984 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/028ada3e3ec9_move_projects_workbench.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py @@ -1,8 +1,8 @@ """Move projects workbench -Revision ID: 028ada3e3ec9 +Revision ID: 1d74a6de5e36 Revises: 307017ee1a49 -Create Date: 2025-01-07 10:25:54.137095+00:00 +Create Date: 2025-01-07 11:54:32.232909+00:00 """ import sqlalchemy as sa @@ -10,7 +10,7 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = "028ada3e3ec9" +revision = "1d74a6de5e36" down_revision = "307017ee1a49" branch_labels = None depends_on = None @@ -18,9 +18,10 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column("projects_nodes", sa.Column("key", sa.String(), nullable=False)) - op.add_column("projects_nodes", sa.Column("version", sa.String(), nullable=False)) - op.add_column("projects_nodes", sa.Column("label", sa.String(), nullable=False)) + op.add_column("projects_nodes", sa.Column("key", sa.String(), nullable=True)) + op.add_column("projects_nodes", sa.Column("version", sa.String(), nullable=True)) + op.add_column("projects_nodes", sa.Column("label", sa.String(), nullable=True)) + op.add_column("projects_nodes", sa.Column("progress", sa.Numeric(), nullable=True)) op.add_column("projects_nodes", sa.Column("thumbnail", sa.String(), nullable=True)) op.add_column( @@ -38,6 +39,30 @@ def upgrade(): ) # ### end Alembic commands ### + op.execute( + """ +UPDATE projects_nodes +SET key = subquery.key, + version = subquery.version, + label = subquery.label +FROM ( + SELECT + projects.uuid AS project_id, + js.key AS node_id, + js.value::jsonb ->> 'key' AS key, + js.value::jsonb ->> 'label' AS label, + js.value::jsonb ->> 'version' AS version + FROM projects, + json_each(projects.workbench) AS js +) AS subquery +WHERE projects_nodes.project_uuid = subquery.project_id +AND projects_nodes.node_id = subquery.node_id; +""" + ) + op.alter_column("projects_nodes", "key", nullable=False) + op.alter_column("projects_nodes", "version", nullable=False) + op.alter_column("projects_nodes", "label", nullable=False) + def downgrade(): # ### commands auto generated by Alembic - please adjust! ### From 9494ab0aa90322626e81c7d26ca2786d8b405271 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 7 Jan 2025 13:54:26 +0100 Subject: [PATCH 05/55] add more data --- .../1d74a6de5e36_move_projects_workbench.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py index 2461703d6984..16df97503c71 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py @@ -39,19 +39,32 @@ def upgrade(): ) # ### end Alembic commands ### + # Populate the new columns with data from the workbench op.execute( """ UPDATE projects_nodes SET key = subquery.key, version = subquery.version, - label = subquery.label + label = subquery.label, + progress = subquery.progress::numeric, + thumbnail = subquery.thumbnail, + inputs = subquery.inputs::jsonb, + outputs = subquery.outputs::jsonb, + run_hash = subquery.run_hash, + state = subquery.state::jsonb FROM ( SELECT projects.uuid AS project_id, js.key AS node_id, js.value::jsonb ->> 'key' AS key, js.value::jsonb ->> 'label' AS label, - js.value::jsonb ->> 'version' AS version + js.value::jsonb ->> 'version' AS version, + (js.value::jsonb ->> 'progress')::numeric AS progress, + js.value::jsonb ->> 'thumbnail' AS thumbnail, + js.value::jsonb ->> 'inputs' AS inputs, + js.value::jsonb ->> 'outputs' AS outputs, + js.value::jsonb ->> 'runHash' AS run_hash, + js.value::jsonb ->> 'state' AS state FROM projects, json_each(projects.workbench) AS js ) AS subquery From bc54acc43b022e331f86ef60034606b6f3adbbb8 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 7 Jan 2025 15:29:22 +0100 Subject: [PATCH 06/55] fix test --- .../utils_projects_nodes.py | 5 ++++- .../pytest_simcore/helpers/webserver_projects.py | 5 ++++- .../src/simcore_service_webserver/projects/db.py | 16 ++++++++++++---- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 42b40c778dc5..f225f13b829f 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -40,6 +40,9 @@ class ProjectNodesDuplicateNodeError(BaseProjectNodesError): class ProjectNodeCreate(BaseModel): node_id: uuid.UUID required_resources: dict[str, Any] = Field(default_factory=dict) + key: str + version: str + label: str @classmethod def get_field_names(cls, *, exclude: set[str]) -> set[str]: @@ -65,7 +68,7 @@ async def add( *, nodes: list[ProjectNodeCreate], ) -> list[ProjectNode]: - """creates a new entry in *projects_nodes* and *projects_to_projects_nodes* tables + """Creates a new entry in *projects_nodes* table NOTE: Do not use this in an asyncio.gather call as this will fail! diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index 092ab82d6556..341138aeb80e 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -81,8 +81,11 @@ async def create_project( required_resources=ServiceResourcesDictHelpers.model_config[ "json_schema_extra" ]["examples"][0], + key=node_info.get("key"), + version=node_info.get("version"), + label=node_info.get("label"), ) - for node_id in project_data.get("workbench", {}) + for node_id, node_info in project_data.get("workbench", {}).items() }, ) diff --git a/services/web/server/src/simcore_service_webserver/projects/db.py b/services/web/server/src/simcore_service_webserver/projects/db.py index b0fc7c5551a6..5bd1b34e2998 100644 --- a/services/web/server/src/simcore_service_webserver/projects/db.py +++ b/services/web/server/src/simcore_service_webserver/projects/db.py @@ -216,19 +216,27 @@ def _reraise_if_not_unique_uuid_error(err: UniqueViolation): if project_nodes is None: project_nodes = { NodeID(node_id): ProjectNodeCreate( - node_id=NodeID(node_id), required_resources={} + node_id=NodeID(node_id), + required_resources={}, + key=node_info.get("key"), + version=node_info.get("version"), + label=node_info.get("label"), ) - for node_id in selected_values["workbench"] + for node_id, node_info in selected_values["workbench"].items() } nodes = [ project_nodes.get( NodeID(node_id), ProjectNodeCreate( - node_id=NodeID(node_id), required_resources={} + node_id=NodeID(node_id), + required_resources={}, + key=node_info.get("key"), + version=node_info.get("version"), + label=node_info.get("label"), ), ) - for node_id in selected_values["workbench"] + for node_id, node_info in selected_values["workbench"].items() ] await project_nodes_repo.add(conn, nodes=nodes) return selected_values From e97a1cf6fefe5aa35085d95c086fd591a04693c7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 7 Jan 2025 16:43:21 +0100 Subject: [PATCH 07/55] fix node create --- .../projects/_crud_api_create.py | 4 ++++ .../simcore_service_webserver/projects/projects_api.py | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 9953914f5d04..352e0c036d42 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -2,6 +2,7 @@ import logging from collections.abc import Coroutine from contextlib import AsyncExitStack +from platform import node from typing import Any, TypeAlias from aiohttp import web @@ -215,6 +216,9 @@ async def _compose_project_data( app, user_id, node_data["key"], node_data["version"] ) ), + key=node_data.get("key"), + version=node_data.get("version"), + label=node_data.get("label"), ) for node_id, node_data in predefined_project.get("workbench", {}).items() } diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 3edd4c50e391..9168db3fc268 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -790,13 +790,17 @@ async def add_project_node( default_resources = await catalog_client.get_service_resources( request.app, user_id, service_key, service_version ) - db: ProjectDBAPI = request.app[APP_PROJECT_DBAPI] + db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(request.app) assert db # nosec await db.add_project_node( user_id, ProjectID(project["uuid"]), ProjectNodeCreate( - node_id=node_uuid, required_resources=jsonable_encoder(default_resources) + node_id=node_uuid, + required_resources=jsonable_encoder(default_resources), + key=service_key, + version=service_version, + label=service_key.split("/")[-1] ), Node.model_validate( { From bd9c666c1db76520d9c5f61b677bceb5a2725576 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 8 Jan 2025 07:34:16 +0100 Subject: [PATCH 08/55] fix node creation mock --- packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py index 67e8ec1722dc..dc696093ec37 100644 --- a/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py +++ b/packages/pytest-simcore/src/pytest_simcore/db_entries_mocks.py @@ -96,7 +96,7 @@ async def creator( inserted_project = ProjectAtDB.model_validate(await result.first()) project_nodes_repo = ProjectNodesRepo(project_uuid=project_uuid) # NOTE: currently no resources is passed until it becomes necessary - default_node_config = {"required_resources": {}} + default_node_config = {"required_resources": {}, "key": faker.pystr(), "version": faker.pystr(), "label": faker.pystr()} if project_nodes_overrides: default_node_config.update(project_nodes_overrides) await project_nodes_repo.add( From 11455b24f5242ed56712d5aaa10f5eddf3b3cb8f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 8 Jan 2025 08:28:49 +0100 Subject: [PATCH 09/55] fix fixture --- packages/postgres-database/tests/conftest.py | 3 +++ .../src/simcore_service_webserver/projects/_crud_api_create.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/tests/conftest.py b/packages/postgres-database/tests/conftest.py index feb8bfaae97a..f9125cc2fcee 100644 --- a/packages/postgres-database/tests/conftest.py +++ b/packages/postgres-database/tests/conftest.py @@ -361,6 +361,9 @@ async def _creator(project_uuid: uuid.UUID) -> ProjectNode: fake_node = ProjectNodeCreate( node_id=uuid.uuid4(), required_resources=faker.pydict(allowed_types=(str,)), + key=faker.pystr(), + version=faker.pystr(), + label=faker.pystr(), ) repo = ProjectNodesRepo(project_uuid=project_uuid) created_nodes = await repo.add(connection, nodes=[fake_node]) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 352e0c036d42..a8c23e025bc9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -2,7 +2,6 @@ import logging from collections.abc import Coroutine from contextlib import AsyncExitStack -from platform import node from typing import Any, TypeAlias from aiohttp import web @@ -134,7 +133,7 @@ async def _copy_project_nodes_from_source_project( db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(app) def _mapped_node_id(node: ProjectNode) -> NodeID: - return NodeID(nodes_map[NodeIDStr(f"{node.node_id}")]) + return NodeID(nodes_map[TypeAdapter(NodeIDStr).validate_python(f"{node.node_id}")]) return { _mapped_node_id(node): ProjectNodeCreate( From 031d947a23b007fe5789a8194f2a06b27284065b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 8 Jan 2025 08:30:53 +0100 Subject: [PATCH 10/55] fix fixture --- packages/postgres-database/tests/test_utils_projects_nodes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/postgres-database/tests/test_utils_projects_nodes.py b/packages/postgres-database/tests/test_utils_projects_nodes.py index 21c130bcc7d9..45362854b42c 100644 --- a/packages/postgres-database/tests/test_utils_projects_nodes.py +++ b/packages/postgres-database/tests/test_utils_projects_nodes.py @@ -79,6 +79,9 @@ def _creator() -> ProjectNodeCreate: node = ProjectNodeCreate( node_id=uuid.uuid4(), required_resources=faker.pydict(allowed_types=(str,)), + key=faker.pystr(), + version=faker.pystr(), + label=faker.pystr(), ) assert node return node From 49415a5069bd0e73df0a7b0b87fa87a283eb1cc6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 8 Jan 2025 11:48:27 +0100 Subject: [PATCH 11/55] add columns --- .../1d74a6de5e36_move_projects_workbench.py | 91 --------------- .../876bae5ff8da_move_projects_workbench.py | 104 ++++++++++++++++++ .../models/projects_nodes.py | 95 ++++++++-------- 3 files changed, 149 insertions(+), 141 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py deleted file mode 100644 index 16df97503c71..000000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/1d74a6de5e36_move_projects_workbench.py +++ /dev/null @@ -1,91 +0,0 @@ -"""Move projects workbench - -Revision ID: 1d74a6de5e36 -Revises: 307017ee1a49 -Create Date: 2025-01-07 11:54:32.232909+00:00 - -""" -import sqlalchemy as sa -from alembic import op -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = "1d74a6de5e36" -down_revision = "307017ee1a49" -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column("projects_nodes", sa.Column("key", sa.String(), nullable=True)) - op.add_column("projects_nodes", sa.Column("version", sa.String(), nullable=True)) - op.add_column("projects_nodes", sa.Column("label", sa.String(), nullable=True)) - - op.add_column("projects_nodes", sa.Column("progress", sa.Numeric(), nullable=True)) - op.add_column("projects_nodes", sa.Column("thumbnail", sa.String(), nullable=True)) - op.add_column( - "projects_nodes", - sa.Column("inputs", postgresql.JSONB(astext_type=sa.Text()), nullable=True), - ) - op.add_column( - "projects_nodes", - sa.Column("outputs", postgresql.JSONB(astext_type=sa.Text()), nullable=True), - ) - op.add_column("projects_nodes", sa.Column("run_hash", sa.String(), nullable=True)) - op.add_column( - "projects_nodes", - sa.Column("state", postgresql.JSONB(astext_type=sa.Text()), nullable=True), - ) - # ### end Alembic commands ### - - # Populate the new columns with data from the workbench - op.execute( - """ -UPDATE projects_nodes -SET key = subquery.key, - version = subquery.version, - label = subquery.label, - progress = subquery.progress::numeric, - thumbnail = subquery.thumbnail, - inputs = subquery.inputs::jsonb, - outputs = subquery.outputs::jsonb, - run_hash = subquery.run_hash, - state = subquery.state::jsonb -FROM ( - SELECT - projects.uuid AS project_id, - js.key AS node_id, - js.value::jsonb ->> 'key' AS key, - js.value::jsonb ->> 'label' AS label, - js.value::jsonb ->> 'version' AS version, - (js.value::jsonb ->> 'progress')::numeric AS progress, - js.value::jsonb ->> 'thumbnail' AS thumbnail, - js.value::jsonb ->> 'inputs' AS inputs, - js.value::jsonb ->> 'outputs' AS outputs, - js.value::jsonb ->> 'runHash' AS run_hash, - js.value::jsonb ->> 'state' AS state - FROM projects, - json_each(projects.workbench) AS js -) AS subquery -WHERE projects_nodes.project_uuid = subquery.project_id -AND projects_nodes.node_id = subquery.node_id; -""" - ) - op.alter_column("projects_nodes", "key", nullable=False) - op.alter_column("projects_nodes", "version", nullable=False) - op.alter_column("projects_nodes", "label", nullable=False) - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("projects_nodes", "state") - op.drop_column("projects_nodes", "run_hash") - op.drop_column("projects_nodes", "outputs") - op.drop_column("projects_nodes", "inputs") - op.drop_column("projects_nodes", "thumbnail") - op.drop_column("projects_nodes", "progress") - op.drop_column("projects_nodes", "label") - op.drop_column("projects_nodes", "version") - op.drop_column("projects_nodes", "key") - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py new file mode 100644 index 000000000000..0bf3179ab94f --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py @@ -0,0 +1,104 @@ +"""Move projects workbench + +Revision ID: 876bae5ff8da +Revises: 307017ee1a49 +Create Date: 2025-01-08 10:43:50.901038+00:00 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '876bae5ff8da' +down_revision = '307017ee1a49' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('projects_nodes', sa.Column('key', sa.String(), nullable=False)) + op.add_column('projects_nodes', sa.Column('version', sa.String(), nullable=False)) + op.add_column('projects_nodes', sa.Column('label', sa.String(), nullable=False)) + op.add_column('projects_nodes', sa.Column('progress', sa.Numeric(), nullable=True)) + op.add_column('projects_nodes', sa.Column('thumbnail', sa.String(), nullable=True)) + op.add_column('projects_nodes', sa.Column('input_access', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('projects_nodes', sa.Column('input_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('projects_nodes', sa.Column('inputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('projects_nodes', sa.Column('inputs_units', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('projects_nodes', sa.Column('output_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('projects_nodes', sa.Column('outputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('projects_nodes', sa.Column('run_hash', sa.String(), nullable=True)) + op.add_column('projects_nodes', sa.Column('state', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('projects_nodes', sa.Column('parent', sa.String(), nullable=True)) + op.add_column('projects_nodes', sa.Column('boot_options', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + # ### end Alembic commands ### + +# Populate the new columns with data from the workbench + op.execute( + """ +UPDATE projects_nodes +SET key = subquery.key, + version = subquery.version, + label = subquery.label, + progress = subquery.progress::numeric, + thumbnail = subquery.thumbnail, + input_access = subquery.input_access::jsonb, + input_nodes = subquery.input_nodes::jsonb, + inputs = subquery.inputs::jsonb, + inputs_units = subquery.inputs_units::jsonb, + outpuy_nodes = subquery.output_nodes::jsonb, + outputs = subquery.outputs::jsonb, + run_hash = subquery.run_hash, + state = subquery.state::jsonb + parent = subquery.parent, + boot_options = subquery.boot_options::jsonb +FROM ( + SELECT + projects.uuid AS project_id, + js.key AS node_id, + js.value::jsonb ->> 'key' AS key, + js.value::jsonb ->> 'label' AS label, + js.value::jsonb ->> 'version' AS version, + (js.value::jsonb ->> 'progress')::numeric AS progress, + js.value::jsonb ->> 'thumbnail' AS thumbnail, + js.value::jsonb ->> 'inputAccess' AS input_access, + js.value::jsonb ->> 'inputNodes' AS input_nodes, + js.value::jsonb ->> 'inputs' AS inputs, + js.value::jsonb ->> 'inputsUnits' AS inputs_units, + js.value::jsonb ->> 'outputNodes' AS output_nodes, + js.value::jsonb ->> 'outputs' AS outputs, + js.value::jsonb ->> 'runHash' AS run_hash, + js.value::jsonb ->> 'state' AS state + js.value::jsonb ->> 'parent' AS parent, + js.value::jsonb ->> 'bootOptions' AS boot_options + FROM projects, + json_each(projects.workbench) AS js +) AS subquery +WHERE projects_nodes.project_uuid = subquery.project_id +AND projects_nodes.node_id = subquery.node_id; +""" + ) + op.alter_column("projects_nodes", "key", nullable=False) + op.alter_column("projects_nodes", "version", nullable=False) + op.alter_column("projects_nodes", "label", nullable=False) + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('projects_nodes', 'boot_options') + op.drop_column('projects_nodes', 'parent') + op.drop_column('projects_nodes', 'state') + op.drop_column('projects_nodes', 'run_hash') + op.drop_column('projects_nodes', 'outputs') + op.drop_column('projects_nodes', 'output_nodes') + op.drop_column('projects_nodes', 'inputs_units') + op.drop_column('projects_nodes', 'inputs') + op.drop_column('projects_nodes', 'input_nodes') + op.drop_column('projects_nodes', 'input_access') + op.drop_column('projects_nodes', 'thumbnail') + op.drop_column('projects_nodes', 'progress') + op.drop_column('projects_nodes', 'label') + op.drop_column('projects_nodes', 'version') + op.drop_column('projects_nodes', 'key') + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index d9774f5e95af..5179392ec308 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -87,18 +87,48 @@ nullable=True, doc="Url of the latest screenshot of the node", ), + sa.Column( + "input_access", + JSONB, + nullable=True, + doc="Map with key - access level pairs", + ), + sa.Column( + "input_nodes", + JSONB, # Array + nullable=True, + doc="Node IDs of where the node is connected to", + ), sa.Column( "inputs", JSONB, nullable=True, - doc="Values of input properties", + doc="Input properties values", + ), + sa.Column( + "inputs_units", + JSONB, + nullable=True, + doc="Input unit values", + ), + sa.Column( + "output_nodes", + JSONB, # Array + nullable=True, + doc="Node IDs of those connected to the output", ), sa.Column( "outputs", JSONB, nullable=True, - doc="Values of output properties", + doc="Output properties values", ), + # sa.Column( + # "output_node", + # sa.BOOLEAN, + # nullable=True, + # doc="Deprecated", + # ), sa.Column( "run_hash", sa.String, @@ -111,60 +141,25 @@ nullable=True, doc="Node state", ), - # sa.Column( - # "inputs_units", - # JSONB, - # nullable=False, - # server_default=sa.text("'{}'::jsonb"), - # doc="Values of input unit", - # ), - # sa.Column( - # "input_access", - # JSONB, - # nullable=False, - # server_default=sa.text("'{}'::jsonb"), - # doc="Map with key - access level pairs", - # ), - # sa.Column( - # "input_nodes", - # JSONB, # <-- ARRAY - # nullable=False, - # server_default=sa.text("'[]'::jsonb"), - # doc="Node IDs of where the node is connected to", - # ), - # sa.Column( - # "output_node", - # sa.BOOLEAN, - # nullable=False, - # doc="Deprecated", - # ), - # sa.Column( - # "output_nodes", - # JSONB, # <-- ARRAY - # nullable=False, - # server_default=sa.text("'[]'::jsonb"), - # doc="Used in group-nodes. Node IDs of those connected to the output", - # ), - # sa.Column( - # "parent", - # sa.String, - # nullable=True, - # doc="Parent's (group-nodes) node IDs.", - # ), + sa.Column( + "parent", + sa.String, + nullable=True, + doc="Parent's (group-nodes) node ID", + ), # sa.Column( # "position", # JSONB, # nullable=True, - # server_default=sa.text("'{}'::jsonb"), # doc="Deprecated", # ), - # sa.Column( - # "boot_options", - # JSONB, - # nullable=True, - # server_default=sa.text("'{}'::jsonb"), - # doc="Some services provide alternative parameters to be injected at boot time. The user selection should be stored here, and it will overwrite the services's defaults.", - # ), + sa.Column( + "boot_options", + JSONB, + nullable=True, + doc="Some services provide alternative parameters to be injected at boot time." + "The user selection should be stored here, and it will overwrite the services's defaults", + ), sa.UniqueConstraint("project_uuid", "node_id"), ) From 0b701a7789662f8d06b131fe40ca0720aff22367 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 8 Jan 2025 11:59:48 +0100 Subject: [PATCH 12/55] fix sql --- .../versions/876bae5ff8da_move_projects_workbench.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py index 0bf3179ab94f..e3b55231b310 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py @@ -51,7 +51,7 @@ def upgrade(): outpuy_nodes = subquery.output_nodes::jsonb, outputs = subquery.outputs::jsonb, run_hash = subquery.run_hash, - state = subquery.state::jsonb + state = subquery.state::jsonb, parent = subquery.parent, boot_options = subquery.boot_options::jsonb FROM ( @@ -70,7 +70,7 @@ def upgrade(): js.value::jsonb ->> 'outputNodes' AS output_nodes, js.value::jsonb ->> 'outputs' AS outputs, js.value::jsonb ->> 'runHash' AS run_hash, - js.value::jsonb ->> 'state' AS state + js.value::jsonb ->> 'state' AS state, js.value::jsonb ->> 'parent' AS parent, js.value::jsonb ->> 'bootOptions' AS boot_options FROM projects, From 4b91286c68f86189fd4c621b8e3bb40dfb0a5568 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 8 Jan 2025 12:10:32 +0100 Subject: [PATCH 13/55] fix column name --- .../migration/versions/876bae5ff8da_move_projects_workbench.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py index e3b55231b310..b34b79857f34 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py @@ -48,7 +48,7 @@ def upgrade(): input_nodes = subquery.input_nodes::jsonb, inputs = subquery.inputs::jsonb, inputs_units = subquery.inputs_units::jsonb, - outpuy_nodes = subquery.output_nodes::jsonb, + output_nodes = subquery.output_nodes::jsonb, outputs = subquery.outputs::jsonb, run_hash = subquery.run_hash, state = subquery.state::jsonb, From ebd80bba4407f2a85c1a331fa3766ab45bb62f48 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 8 Jan 2025 15:35:32 +0100 Subject: [PATCH 14/55] add fields --- .../utils_projects_nodes.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index f225f13b829f..40dc3bc8d69e 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -1,4 +1,6 @@ import datetime +from re import S +from tkinter import N import uuid from dataclasses import dataclass from typing import Any @@ -43,10 +45,22 @@ class ProjectNodeCreate(BaseModel): key: str version: str label: str + progress: float | None = None + thumbnail: str | None = None + input_access: dict[str, Any] | None = None + input_nodes: list[dict[str, Any]] | None = None + inputs: dict[str, Any] | None = None + inputs_units: dict[str, Any] | None = None + output_nodes: list[dict[str, Any]] | None = None + outputs: dict[str, Any] | None = None + run_hash: str | None = None + state: dict[str, Any] | None = None + parent: str | None = None + boot_options: dict[str, Any] | None = None @classmethod def get_field_names(cls, *, exclude: set[str]) -> set[str]: - return {name for name in cls.model_fields.keys() if name not in exclude} + return cls.model_fields.keys() - exclude model_config = ConfigDict(frozen=True) From 3bbd5025abcac60e55dbeb462d9f82425d8ba221 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 9 Jan 2025 11:21:57 +0100 Subject: [PATCH 15/55] fix imports --- .../src/simcore_postgres_database/utils_projects_nodes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 40dc3bc8d69e..0c01ca41789c 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -1,6 +1,4 @@ import datetime -from re import S -from tkinter import N import uuid from dataclasses import dataclass from typing import Any From c7845fca9efebbc062ea67347ea9ffcf940e5610 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 10 Jan 2025 15:15:28 +0100 Subject: [PATCH 16/55] add new repo --- .../api_schemas_webserver/_base.py | 10 ++---- .../webserver_models.py | 2 ++ .../projects/_nodes_handlers.py | 3 +- .../projects/_projects_nodes_repository.py | 35 +++++++++++++++++++ .../projects/models.py | 35 ++++++++++++++++--- .../projects/projects_api.py | 14 +++++--- 6 files changed, 82 insertions(+), 17 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py diff --git a/packages/models-library/src/models_library/api_schemas_webserver/_base.py b/packages/models-library/src/models_library/api_schemas_webserver/_base.py index a5eaa42c0066..44edd74e5096 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/_base.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/_base.py @@ -22,9 +22,8 @@ class InputSchemaWithoutCamelCase(BaseModel): ) -class InputSchema(BaseModel): +class InputSchema(InputSchemaWithoutCamelCase): model_config = ConfigDict( - **InputSchemaWithoutCamelCase.model_config, alias_generator=snake_to_camel, ) @@ -32,17 +31,14 @@ class InputSchema(BaseModel): class OutputSchemaWithoutCamelCase(BaseModel): model_config = ConfigDict( populate_by_name=True, - extra="ignore", + extra="ignore", # Used to prune extra fields from internal data frozen=True, ) -class OutputSchema(BaseModel): +class OutputSchema(OutputSchemaWithoutCamelCase): model_config = ConfigDict( alias_generator=snake_to_camel, - populate_by_name=True, - extra="ignore", # Used to prune extra fields from internal data - frozen=True, ) def data( diff --git a/packages/postgres-database/src/simcore_postgres_database/webserver_models.py b/packages/postgres-database/src/simcore_postgres_database/webserver_models.py index 571db047cfbe..b62a2fd83fe6 100644 --- a/packages/postgres-database/src/simcore_postgres_database/webserver_models.py +++ b/packages/postgres-database/src/simcore_postgres_database/webserver_models.py @@ -12,6 +12,7 @@ from .models.groups import GroupType, groups, user_to_groups from .models.products import products from .models.projects import ProjectType, projects +from .models.projects_nodes import projects_nodes from .models.projects_tags import projects_tags from .models.projects_to_wallet import projects_to_wallet from .models.scicrunch_resources import scicrunch_resources @@ -32,6 +33,7 @@ "NodeClass", "products", "projects", + "projects_nodes", "ProjectType", "scicrunch_resources", "StateType", diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 9ddd88c0df1f..405fa4f04420 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -27,6 +27,7 @@ ) from models_library.groups import EVERYONE_GROUP_ID, Group, GroupID, GroupType from models_library.projects import Project, ProjectID +from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.services import ServiceKeyVersion from models_library.services_resources import ServiceResourcesDict @@ -234,7 +235,7 @@ async def patch_project_node(request: web.Request) -> web.Response: user_id=req_ctx.user_id, project_id=path_params.project_id, node_id=path_params.node_id, - node_patch=node_patch, + node_patch=Node.model_construct(**node_patch.model_dump()), ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py new file mode 100644 index 000000000000..8d40306448e9 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -0,0 +1,35 @@ +import logging + +import sqlalchemy as sa +from aiohttp import web +from models_library.projects import ProjectID +from models_library.projects_nodes import Node, NodeID +from simcore_postgres_database.utils_repos import transaction_context +from simcore_postgres_database.webserver_models import projects_nodes +from sqlalchemy.ext.asyncio import AsyncConnection + +from ..db.plugin import get_asyncpg_engine +from .models import NodeDB + +_logger = logging.getLogger(__name__) + + +async def update( + app: web.Application, + connection: AsyncConnection | None = None, + *, + project_id: ProjectID, + node_id: NodeID, + node: Node, +) -> None: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + await conn.stream( + projects_nodes.update() + .values(**NodeDB.model_validate(node.model_dump()).model_dump()) + .where( + sa.and_( + projects_nodes.c.project_uuid == f"{project_id}", + projects_nodes.c.node_id == f"{node_id}", + ) + ) + ) diff --git a/services/web/server/src/simcore_service_webserver/projects/models.py b/services/web/server/src/simcore_service_webserver/projects/models.py index dca631ba39a1..dea26b15b7ee 100644 --- a/services/web/server/src/simcore_service_webserver/projects/models.py +++ b/services/web/server/src/simcore_service_webserver/projects/models.py @@ -1,4 +1,4 @@ -from datetime import datetime +import datetime as dt from enum import Enum from typing import Any, TypeAlias @@ -6,6 +6,7 @@ from models_library.api_schemas_webserver.projects import ProjectPatch from models_library.folders import FolderID from models_library.projects import ClassifierID, ProjectID +from models_library.projects_nodes import NodeID from models_library.projects_ui import StudyUI from models_library.users import UserID from models_library.utils.common_validators import ( @@ -42,8 +43,8 @@ class ProjectDB(BaseModel): description: str thumbnail: HttpUrl | None prj_owner: UserID - creation_date: datetime - last_change_date: datetime + creation_date: dt.datetime + last_change_date: dt.datetime ui: StudyUI | None classifiers: list[ClassifierID] dev: dict | None @@ -51,7 +52,7 @@ class ProjectDB(BaseModel): published: bool hidden: bool workspace_id: WorkspaceID | None - trashed_at: datetime | None + trashed_at: dt.datetime | None trashed_explicitly: bool = False model_config = ConfigDict(from_attributes=True, arbitrary_types_allowed=True) @@ -95,12 +96,36 @@ class UserProjectAccessRightsWithWorkspace(BaseModel): class ProjectPatchExtended(ProjectPatch): # Only used internally - trashed_at: datetime | None + trashed_at: dt.datetime | None trashed_explicitly: bool model_config = ConfigDict(populate_by_name=True, extra="forbid") +class NodeDB(BaseModel): + node_id: NodeID + required_resources: dict[str, Any] + created: dt.datetime + modified: dt.datetime + project_uuid: ProjectID + project_node_id: int + key: str + version: str + label: str + progress: float | None + thumbnail: HttpUrl | None + input_access: dict[str, Any] + input_nodes: list[NodeID] + inputs: dict[str, Any] + inouts_units: dict[str, Any] + output_nodes: list[NodeID] + outputs: dict[str, Any] + run_hash: str | None + state: dict[str, Any] | None + parent: str | None + boot_options: dict[str, Any] | None + + __all__: tuple[str, ...] = ( "ProjectDict", "ProjectProxy", diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 9168db3fc268..ecd5cfe0fa0c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -32,7 +32,6 @@ DynamicServiceStop, ) from models_library.api_schemas_webserver.projects import ProjectPatch -from models_library.api_schemas_webserver.projects_nodes import NodePatch from models_library.basic_types import KeyIDStr from models_library.errors import ErrorDict from models_library.groups import GroupID @@ -121,7 +120,7 @@ from ..wallets import api as wallets_api from ..wallets.errors import WalletNotEnoughCreditsError from ..workspaces import _workspaces_db as workspaces_db -from . import _crud_api_delete, _nodes_api, _projects_db +from . import _crud_api_delete, _nodes_api, _projects_db, _projects_nodes_repository from ._access_rights_api import ( check_user_project_permission, has_user_project_access_rights, @@ -800,7 +799,7 @@ async def add_project_node( required_resources=jsonable_encoder(default_resources), key=service_key, version=service_version, - label=service_key.split("/")[-1] + label=service_key.split("/")[-1], ), Node.model_validate( { @@ -1001,7 +1000,7 @@ async def patch_project_node( user_id: UserID, project_id: ProjectID, node_id: NodeID, - node_patch: NodePatch, + node_patch: Node, ) -> None: _node_patch_exclude_unset: dict[str, Any] = jsonable_encoder( node_patch, exclude_unset=True, by_alias=True @@ -1044,6 +1043,13 @@ async def patch_project_node( new_node_data=_node_patch_exclude_unset, ) + await _projects_nodes_repository.update( + app, + project_id=project_id, + node_id=node_id, + node=node_patch, + ) + # 4. Make calls to director-v2 to keep data in sync (ex. comp_tasks DB table) await director_v2_api.create_or_update_pipeline( app, user_id, project_id, product_name=product_name From 7d00d5d8e75da2898ece21ffaa527be59838a2a7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Jan 2025 10:24:52 +0100 Subject: [PATCH 17/55] fix import --- .../simcore_service_webserver/projects/_nodes_handlers.py | 2 +- .../projects/_projects_nodes_repository.py | 5 +++-- .../server/src/simcore_service_webserver/projects/models.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 405fa4f04420..80fdca97ea2e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -235,7 +235,7 @@ async def patch_project_node(request: web.Request) -> web.Response: user_id=req_ctx.user_id, project_id=path_params.project_id, node_id=path_params.node_id, - node_patch=Node.model_construct(**node_patch.model_dump()), + node_patch=Node.model_validate(node_patch.model_dump()), ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 8d40306448e9..b6ca59b23ccd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -3,7 +3,8 @@ import sqlalchemy as sa from aiohttp import web from models_library.projects import ProjectID -from models_library.projects_nodes import Node, NodeID +from models_library.projects_nodes import Node +from models_library.projects_nodes_io import NodeID from simcore_postgres_database.utils_repos import transaction_context from simcore_postgres_database.webserver_models import projects_nodes from sqlalchemy.ext.asyncio import AsyncConnection @@ -25,7 +26,7 @@ async def update( async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.stream( projects_nodes.update() - .values(**NodeDB.model_validate(node.model_dump()).model_dump()) + .values(**NodeDB.model_construct(**node.model_dump()).model_dump(exclude_none=True)) .where( sa.and_( projects_nodes.c.project_uuid == f"{project_id}", diff --git a/services/web/server/src/simcore_service_webserver/projects/models.py b/services/web/server/src/simcore_service_webserver/projects/models.py index dea26b15b7ee..6fc3c360d42d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/models.py +++ b/services/web/server/src/simcore_service_webserver/projects/models.py @@ -6,7 +6,7 @@ from models_library.api_schemas_webserver.projects import ProjectPatch from models_library.folders import FolderID from models_library.projects import ClassifierID, ProjectID -from models_library.projects_nodes import NodeID +from models_library.projects_nodes_io import NodeID from models_library.projects_ui import StudyUI from models_library.users import UserID from models_library.utils.common_validators import ( From 48145b4666cbc22dc97ad66586a4e693cbb3c39d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Jan 2025 10:43:23 +0100 Subject: [PATCH 18/55] fix mapping --- .../api_schemas_webserver/projects_nodes.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 45e3c87643b4..8178cbbc4677 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -3,6 +3,9 @@ from pydantic import ConfigDict, Field +from common_library.dict_tools import remap_keys +from models_library.projects_nodes import Node + from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut from ..basic_types import PortInt from ..projects_nodes import InputID, InputsDict @@ -47,6 +50,19 @@ class NodePatch(InputSchemaWithoutCamelCase): str, Any ] | None = None # NOTE: it is used by frontend for File Picker + def to_model(self) -> Node: + data = remap_keys( + self.model_dump( + mode="json", + exclude_unset=True, + ), + rename={ + "service_key": "key", + "service_version": "version", + }, + ) + return Node(**data) + class NodeCreated(OutputSchema): node_id: NodeID From f71c1c6b5d67e0d1b442ee2f8d31c150382d7dd4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Jan 2025 11:36:25 +0100 Subject: [PATCH 19/55] fix model --- .../api_schemas_webserver/projects_nodes.py | 28 +++++++++++++------ .../src/models_library/projects_nodes.py | 5 ++++ .../projects/_nodes_handlers.py | 4 +-- .../projects/projects_api.py | 2 +- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 8178cbbc4677..1aa6b58ec04e 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -8,7 +8,7 @@ from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut from ..basic_types import PortInt -from ..projects_nodes import InputID, InputsDict +from ..projects_nodes import InputID, InputsDict, PartialNode from ..projects_nodes_io import NodeID from ..services import ServiceKey, ServicePortKey, ServiceVersion from ..services_enums import ServiceState @@ -29,14 +29,26 @@ class NodeCreate(InputSchemaWithoutCamelCase): class NodePatch(InputSchemaWithoutCamelCase): - service_key: ServiceKey | None = Field(default=None, alias="key") - service_version: ServiceVersion | None = Field(default=None, alias="version") - label: str | None = Field(default=None) + service_key: Annotated[ + ServiceKey | None, + Field(alias="key"), + ] = None + service_version: Annotated[ + ServiceVersion | None, + Field(alias="version"), + ] = None + label: str | None = None inputs: Annotated[ InputsDict, Field(default_factory=dict, json_schema_extra={"default": {}}) ] - inputs_required: list[InputID] | None = Field(default=None, alias="inputsRequired") - input_nodes: list[NodeID] | None = Field(default=None, alias="inputNodes") + inputs_required: Annotated[ + list[InputID] | None, + Field(alias="inputsRequired"), + ] = None + input_nodes: Annotated[ + list[NodeID] | None, + Field(alias="inputNodes"), + ] = None progress: Annotated[ float | None, Field( @@ -50,7 +62,7 @@ class NodePatch(InputSchemaWithoutCamelCase): str, Any ] | None = None # NOTE: it is used by frontend for File Picker - def to_model(self) -> Node: + def to_model(self) -> PartialNode: data = remap_keys( self.model_dump( mode="json", @@ -61,7 +73,7 @@ def to_model(self) -> Node: "service_version": "version", }, ) - return Node(**data) + return PartialNode(**data) class NodeCreated(OutputSchema): diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index f7db56b1ded3..c1b8374d2577 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -274,3 +274,8 @@ def _convert_from_enum(cls, v): extra="forbid", populate_by_name=True, ) + + +class PartialNode(Node): + key: Annotated[ServiceKey | None, Field(default=None)] + version: Annotated[ServiceVersion | None, Field(default=None)] diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 80fdca97ea2e..55db0aefa86c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -229,13 +229,13 @@ async def patch_project_node(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(NodePathParams, request) node_patch = await parse_request_body_as(NodePatch, request) - await projects_api.patch_project_node( + await projects_api.update_project_node( request.app, product_name=req_ctx.product_name, user_id=req_ctx.user_id, project_id=path_params.project_id, node_id=path_params.node_id, - node_patch=Node.model_validate(node_patch.model_dump()), + node_patch=node_patch.to_model(), ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index ecd5cfe0fa0c..1bdfe3057e4c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -993,7 +993,7 @@ async def is_project_hidden(app: web.Application, project_id: ProjectID) -> bool return await db.is_hidden(project_id) -async def patch_project_node( +async def update_project_node( app: web.Application, *, product_name: ProductName, From 56099a8cb8e553ae704c2313cd04c5aa8113a989 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Jan 2025 11:58:47 +0100 Subject: [PATCH 20/55] fix model --- .../api_schemas_webserver/projects_nodes.py | 14 ++++---------- .../projects/_projects_nodes_repository.py | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 1aa6b58ec04e..25c53615b28f 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -63,17 +63,11 @@ class NodePatch(InputSchemaWithoutCamelCase): ] | None = None # NOTE: it is used by frontend for File Picker def to_model(self) -> PartialNode: - data = remap_keys( - self.model_dump( - mode="json", - exclude_unset=True, - ), - rename={ - "service_key": "key", - "service_version": "version", - }, + data = self.model_dump( + exclude_unset=True, + by_alias=True, ) - return PartialNode(**data) + return PartialNode.model_construct(**data) class NodeCreated(OutputSchema): diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index b6ca59b23ccd..6111ada0ee14 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -26,7 +26,7 @@ async def update( async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.stream( projects_nodes.update() - .values(**NodeDB.model_construct(**node.model_dump()).model_dump(exclude_none=True)) + .values(**NodeDB.model_construct(**node.model_dump()).model_dump(mode="json", exclude_none=True)) .where( sa.and_( projects_nodes.c.project_uuid == f"{project_id}", From 9136b340aea1dc12c842054e5dec4f1f51cb3bca Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Jan 2025 13:04:08 +0100 Subject: [PATCH 21/55] fix field name --- .../projects/_projects_nodes_repository.py | 6 +++--- .../server/src/simcore_service_webserver/projects/models.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 6111ada0ee14..ebcec072a088 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -3,7 +3,7 @@ import sqlalchemy as sa from aiohttp import web from models_library.projects import ProjectID -from models_library.projects_nodes import Node +from models_library.projects_nodes import PartialNode from models_library.projects_nodes_io import NodeID from simcore_postgres_database.utils_repos import transaction_context from simcore_postgres_database.webserver_models import projects_nodes @@ -21,12 +21,12 @@ async def update( *, project_id: ProjectID, node_id: NodeID, - node: Node, + node: PartialNode, ) -> None: async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.stream( projects_nodes.update() - .values(**NodeDB.model_construct(**node.model_dump()).model_dump(mode="json", exclude_none=True)) + .values(**NodeDB.model_construct(**node.model_dump()).model_dump(mode="json", exclude_unset=True, exclude_none=True)) .where( sa.and_( projects_nodes.c.project_uuid == f"{project_id}", diff --git a/services/web/server/src/simcore_service_webserver/projects/models.py b/services/web/server/src/simcore_service_webserver/projects/models.py index 6fc3c360d42d..3681101f2960 100644 --- a/services/web/server/src/simcore_service_webserver/projects/models.py +++ b/services/web/server/src/simcore_service_webserver/projects/models.py @@ -117,7 +117,7 @@ class NodeDB(BaseModel): input_access: dict[str, Any] input_nodes: list[NodeID] inputs: dict[str, Any] - inouts_units: dict[str, Any] + inputs_units: dict[str, Any] output_nodes: list[NodeID] outputs: dict[str, Any] run_hash: str | None From 279e538ace966e7c1894a6c0a5272ddac619aad6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Jan 2025 13:18:59 +0100 Subject: [PATCH 22/55] add output update --- .../src/models_library/projects_nodes.py | 1 + .../simcore_service_webserver/projects/projects_api.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index c1b8374d2577..d95c42b032cd 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -279,3 +279,4 @@ def _convert_from_enum(cls, v): class PartialNode(Node): key: Annotated[ServiceKey | None, Field(default=None)] version: Annotated[ServiceVersion | None, Field(default=None)] + label: Annotated[str | None, Field(default=None)] diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 1bdfe3057e4c..d309b807c9f1 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -38,7 +38,7 @@ from models_library.products import ProductName from models_library.projects import Project, ProjectID, ProjectIDStr from models_library.projects_access import Owner -from models_library.projects_nodes import Node +from models_library.projects_nodes import Node, PartialNode from models_library.projects_nodes_io import NodeID, NodeIDStr, PortLink from models_library.projects_state import ( ProjectLocked, @@ -109,6 +109,7 @@ send_message_to_standard_group, send_message_to_user, ) +from . import _projects_nodes_repository from ..storage import api as storage_api from ..users.api import FullNameDict, get_user, get_user_fullname, get_user_role from ..users.exceptions import UserNotFoundError @@ -1111,6 +1112,13 @@ async def update_project_node_outputs( new_node_data={"outputs": new_outputs, "runHash": new_run_hash}, ) + await _projects_nodes_repository.update( + app, + project_id=project_id, + node_id=node_id, + node=PartialNode.model_construct(outputs=new_outputs, run_hash=new_run_hash), + ) + log.debug( "patched project %s, following entries changed: %s", project_id, From d45aaceefe47f053af98e9fbf2393c071bb320e3 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Jan 2025 14:17:35 +0100 Subject: [PATCH 23/55] rename service --- .../utils_projects_nodes.py | 4 +-- .../_db_comp_tasks_listening_task.py | 8 ++--- .../exporter/_formatter/_sds.py | 2 +- .../exporter/_handlers.py | 2 +- .../folders/_folders_api.py | 2 +- .../garbage_collector/_core_disconnected.py | 2 +- .../garbage_collector/_core_guests.py | 2 +- .../garbage_collector/_core_orphans.py | 2 +- .../_rabbitmq_exclusive_queue_consumers.py | 4 +-- .../projects/_comments_handlers.py | 12 +++---- .../projects/_crud_api_create.py | 12 +++---- .../projects/_crud_api_read.py | 4 +-- .../projects/_crud_handlers.py | 14 ++++---- .../projects/_nodes_handlers.py | 36 +++++++++---------- .../projects/_observer.py | 2 +- .../projects/_ports_handlers.py | 4 +-- .../_projects_nodes_pricing_unit_handlers.py | 8 ++--- .../projects/_projects_nodes_repository.py | 2 +- .../projects/_states_handlers.py | 26 +++++++------- .../projects/_trash_api.py | 8 ++--- .../projects/_wallets_handlers.py | 6 ++-- .../projects/nodes_utils.py | 10 +++--- .../{projects_api.py => projects_service.py} | 6 ++-- .../studies_dispatcher/_projects.py | 2 +- .../02/test_projects_crud_handlers__delete.py | 2 +- .../tests/unit/with_dbs/03/test_project_db.py | 2 +- .../test_version_control_core.py | 4 +-- .../test_version_control_handlers.py | 4 +-- .../test_resource_manager.py | 2 +- .../test_studies_dispatcher_projects.py | 2 +- .../test_studies_dispatcher_studies_access.py | 2 +- 31 files changed, 99 insertions(+), 99 deletions(-) rename services/web/server/src/simcore_service_webserver/projects/{projects_api.py => projects_service.py} (99%) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index 0c01ca41789c..b71008f6ba2c 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -46,10 +46,10 @@ class ProjectNodeCreate(BaseModel): progress: float | None = None thumbnail: str | None = None input_access: dict[str, Any] | None = None - input_nodes: list[dict[str, Any]] | None = None + input_nodes: list[str] | None = None inputs: dict[str, Any] | None = None inputs_units: dict[str, Any] | None = None - output_nodes: list[dict[str, Any]] | None = None + output_nodes: list[str] | None = None outputs: dict[str, Any] | None = None run_hash: str | None = None state: dict[str, Any] | None = None diff --git a/services/web/server/src/simcore_service_webserver/db_listener/_db_comp_tasks_listening_task.py b/services/web/server/src/simcore_service_webserver/db_listener/_db_comp_tasks_listening_task.py index f97f214d4a92..2777fe57e496 100644 --- a/services/web/server/src/simcore_service_webserver/db_listener/_db_comp_tasks_listening_task.py +++ b/services/web/server/src/simcore_service_webserver/db_listener/_db_comp_tasks_listening_task.py @@ -22,7 +22,7 @@ from sqlalchemy.sql import select from ..db.plugin import get_database_engine -from ..projects import exceptions, projects_api +from ..projects import exceptions, projects_service from ..projects.nodes_utils import update_node_outputs from ._utils import convert_state_from_db @@ -47,12 +47,12 @@ async def _update_project_state( new_state: RunningState, node_errors: list[ErrorDict] | None, ) -> None: - project = await projects_api.update_project_node_state( + project = await projects_service.update_project_node_state( app, user_id, project_uuid, node_uuid, new_state ) - await projects_api.notify_project_node_update(app, project, node_uuid, node_errors) - await projects_api.notify_project_state_update(app, project) + await projects_service.notify_project_node_update(app, project, node_uuid, node_errors) + await projects_service.notify_project_state_update(app, project) @dataclass(frozen=True) diff --git a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py index 487522963db4..62f02f2b1d18 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_formatter/_sds.py @@ -10,7 +10,7 @@ from ...catalog.client import get_service from ...projects.exceptions import BaseProjectError from ...projects.models import ProjectDict -from ...projects.projects_api import get_project_for_user +from ...projects.projects_service import get_project_for_user from ...scicrunch.db import ResearchResourceRepository from ..exceptions import SDSException from .template_json import write_template_json diff --git a/services/web/server/src/simcore_service_webserver/exporter/_handlers.py b/services/web/server/src/simcore_service_webserver/exporter/_handlers.py index 97749637f54e..a605c7ad35f1 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/_handlers.py +++ b/services/web/server/src/simcore_service_webserver/exporter/_handlers.py @@ -12,7 +12,7 @@ from .._meta import API_VTAG from ..login.decorators import login_required from ..projects.lock import lock_project -from ..projects.projects_api import retrieve_and_notify_project_locked_state +from ..projects.projects_service import retrieve_and_notify_project_locked_state from ..security.decorators import permission_required from ..users.api import get_user_fullname from ._formatter.archive import get_sds_archive_path diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_api.py b/services/web/server/src/simcore_service_webserver/folders/_folders_api.py index 6cd65316b057..521302165072 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_api.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_api.py @@ -17,7 +17,7 @@ from servicelib.utils import fire_and_forget_task from ..folders.errors import FolderValueNotPermittedError -from ..projects.projects_api import submit_delete_project_task +from ..projects.projects_service import submit_delete_project_task from ..users.api import get_user from ..workspaces.api import check_user_workspace_access from ..workspaces.errors import ( diff --git a/services/web/server/src/simcore_service_webserver/garbage_collector/_core_disconnected.py b/services/web/server/src/simcore_service_webserver/garbage_collector/_core_disconnected.py index 2acdbed94470..68a7c6b55bf8 100644 --- a/services/web/server/src/simcore_service_webserver/garbage_collector/_core_disconnected.py +++ b/services/web/server/src/simcore_service_webserver/garbage_collector/_core_disconnected.py @@ -6,7 +6,7 @@ from servicelib.utils import logged_gather from ..projects.exceptions import ProjectLockError, ProjectNotFoundError -from ..projects.projects_api import remove_project_dynamic_services +from ..projects.projects_service import remove_project_dynamic_services from ..redis import get_redis_lock_manager_client from ..resource_manager.registry import ( RedisResourceRegistry, diff --git a/services/web/server/src/simcore_service_webserver/garbage_collector/_core_guests.py b/services/web/server/src/simcore_service_webserver/garbage_collector/_core_guests.py index 8649d2e2451a..f89278ead785 100644 --- a/services/web/server/src/simcore_service_webserver/garbage_collector/_core_guests.py +++ b/services/web/server/src/simcore_service_webserver/garbage_collector/_core_guests.py @@ -13,7 +13,7 @@ from ..projects.db import ProjectDBAPI from ..projects.exceptions import ProjectDeleteError, ProjectNotFoundError -from ..projects.projects_api import get_project_for_user, submit_delete_project_task +from ..projects.projects_service import get_project_for_user, submit_delete_project_task from ..redis import get_redis_lock_manager_client from ..resource_manager.registry import RedisResourceRegistry from ..users import exceptions diff --git a/services/web/server/src/simcore_service_webserver/garbage_collector/_core_orphans.py b/services/web/server/src/simcore_service_webserver/garbage_collector/_core_orphans.py index 0920aecd1688..bcb8af72dfa9 100644 --- a/services/web/server/src/simcore_service_webserver/garbage_collector/_core_orphans.py +++ b/services/web/server/src/simcore_service_webserver/garbage_collector/_core_orphans.py @@ -15,7 +15,7 @@ from ..dynamic_scheduler import api as dynamic_scheduler_api from ..projects.api import has_user_project_access_rights -from ..projects.projects_api import ( +from ..projects.projects_service import ( is_node_id_present_in_any_project_workbench, list_node_ids_in_project, ) diff --git a/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_exclusive_queue_consumers.py b/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_exclusive_queue_consumers.py index 1ba51262d84f..4193c6fce7fd 100644 --- a/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_exclusive_queue_consumers.py +++ b/services/web/server/src/simcore_service_webserver/notifications/_rabbitmq_exclusive_queue_consumers.py @@ -18,7 +18,7 @@ from servicelib.rabbitmq import RabbitMQClient from servicelib.utils import logged_gather -from ..projects import projects_api +from ..projects import projects_service from ..projects.exceptions import ProjectNotFoundError from ..rabbitmq import get_rabbitmq_client from ..socketio.messages import ( @@ -42,7 +42,7 @@ async def _convert_to_node_update_event( app: web.Application, message: ProgressRabbitMessageNode ) -> SocketMessageDict | None: try: - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( app, f"{message.project_id}", message.user_id ) if f"{message.node_id}" in project["workbench"]: diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py index 6ad8b290ba08..a8596b7407f5 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py @@ -30,7 +30,7 @@ from ..login.decorators import login_required from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _comments_api, projects_api +from . import _comments_api, projects_service from ._common_models import RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError @@ -86,7 +86,7 @@ async def create_project_comment(request: web.Request): body_params = await parse_request_body_as(_ProjectCommentsBodyParams, request) # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_uuid}", user_id=req_ctx.user_id, @@ -128,7 +128,7 @@ async def list_project_comments(request: web.Request): ) # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_uuid}", user_id=req_ctx.user_id, @@ -176,7 +176,7 @@ async def update_project_comment(request: web.Request): body_params = await parse_request_body_as(_ProjectCommentsBodyParams, request) # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_uuid}", user_id=req_ctx.user_id, @@ -205,7 +205,7 @@ async def delete_project_comment(request: web.Request): ) # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_uuid}", user_id=req_ctx.user_id, @@ -233,7 +233,7 @@ async def get_project_comment(request: web.Request): ) # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_uuid}", user_id=req_ctx.user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index a8c23e025bc9..dd3cb5e21156 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -38,7 +38,7 @@ from ..workspaces.api import check_user_workspace_access from ..workspaces.errors import WorkspaceAccessForbiddenError from . import _folders_db as project_to_folders_db -from . import projects_api +from . import projects_service from ._metadata_api import set_project_ancestors from ._permalink_api import update_or_pop_permalink_in_project from .db import ProjectDBAPI @@ -75,7 +75,7 @@ async def _prepare_project_copy( deep_copy: bool, task_progress: TaskProgress, ) -> tuple[ProjectDict, CopyProjectNodesCoro | None, CopyFileCoro | None]: - source_project = await projects_api.get_project_for_user( + source_project = await projects_service.get_project_for_user( app, project_uuid=f"{src_project_uuid}", user_id=user_id, @@ -167,7 +167,7 @@ async def _copy_files_from_source_project( async with AsyncExitStack() as stack: if needs_lock_source_project: await stack.enter_async_context( - projects_api.lock_with_notification( + projects_service.lock_with_notification( app, source_project["uuid"], ProjectStatus.CLONING, @@ -392,7 +392,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche # get the latest state of the project (lastChangeDate for instance) new_project, _ = await db.get_project(project_uuid=new_project["uuid"]) # Appends state - new_project = await projects_api.add_project_states_for_user( + new_project = await projects_service.add_project_states_for_user( user_id=user_id, project=new_project, is_template=as_template, @@ -444,7 +444,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche except (ParentProjectNotFoundError, ParentNodeNotFoundError) as exc: if project_uuid := new_project.get("uuid"): - await projects_api.submit_delete_project_task( + await projects_service.submit_delete_project_task( app=request.app, project_uuid=project_uuid, user_id=user_id, @@ -458,7 +458,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche f"{user_id=}", ) if project_uuid := new_project.get("uuid"): - await projects_api.submit_delete_project_task( + await projects_service.submit_delete_project_task( app=request.app, project_uuid=project_uuid, user_id=user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index 55a9b7c6429b..033906eacff7 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -21,7 +21,7 @@ from ..catalog.client import get_services_for_user_in_product from ..folders import _folders_db as folders_db from ..workspaces._workspaces_api import check_user_workspace_access -from . import projects_api +from . import projects_service from ._permalink_api import update_or_pop_permalink_in_project from .db import ProjectDBAPI from .models import ProjectDict, ProjectTypeAPI @@ -36,7 +36,7 @@ async def _append_fields( model_schema_cls: type[OutputSchema], ): # state - await projects_api.add_project_states_for_user( + await projects_service.add_project_states_for_user( user_id=user_id, project=project, is_template=is_template, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index 8173a7a2db6d..0c6a7e509eef 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -47,7 +47,7 @@ from ..security.decorators import permission_required from ..users.api import get_user_fullname from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError -from . import _crud_api_create, _crud_api_read, projects_api +from . import _crud_api_create, _crud_api_read, projects_service from ._common_models import ProjectPathParams, RequestContext from ._crud_handlers_models import ( ProjectActiveQueryParams, @@ -295,7 +295,7 @@ async def get_active_project(request: web.Request) -> web.Response: data = None if user_active_projects: - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( request.app, project_uuid=user_active_projects[0], user_id=req_ctx.user_id, @@ -335,7 +335,7 @@ async def get_project(request: web.Request): ) try: - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -384,7 +384,7 @@ async def get_project(request: web.Request): async def get_project_inactivity(request: web.Request): path_params = parse_request_path_parameters_as(ProjectPathParams, request) - project_inactivity = await projects_api.get_project_inactivity( + project_inactivity = await projects_service.get_project_inactivity( app=request.app, project_id=path_params.project_id ) return web.json_response(Envelope(data=project_inactivity), dumps=json_dumps) @@ -403,7 +403,7 @@ async def patch_project(request: web.Request): path_params = parse_request_path_parameters_as(ProjectPathParams, request) project_patch = await parse_request_body_as(ProjectPatch, request) - await projects_api.patch_project( + await projects_service.patch_project( request.app, user_id=req_ctx.user_id, project_uuid=path_params.project_id, @@ -436,7 +436,7 @@ async def delete_project(request: web.Request): path_params = parse_request_path_parameters_as(ProjectPathParams, request) try: - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -473,7 +473,7 @@ async def delete_project(request: web.Request): reason=f"Project {path_params.project_id} is locked: {project_locked_state=}" ) - await projects_api.submit_delete_project_task( + await projects_service.submit_delete_project_task( request.app, path_params.project_id, req_ctx.user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 55db0aefa86c..4330d9cdbb9d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -74,7 +74,7 @@ from ..users.exceptions import UserDefaultWalletNotFoundError from ..utils_aiohttp import envelope_json_response from ..wallets.errors import WalletAccessForbiddenError, WalletNotEnoughCreditsError -from . import nodes_utils, projects_api +from . import nodes_utils, projects_service from ._common_models import ProjectPathParams, RequestContext from ._nodes_api import NodeScreenshot, get_node_screenshots from .exceptions import ( @@ -148,7 +148,7 @@ async def create_node(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(ProjectPathParams, request) body = await parse_request_body_as(NodeCreate, request) - if await projects_api.is_service_deprecated( + if await projects_service.is_service_deprecated( request.app, req_ctx.user_id, body.service_key, @@ -160,13 +160,13 @@ async def create_node(request: web.Request) -> web.Response: ) # ensure the project exists - project_data = await projects_api.get_project_for_user( + project_data = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, ) data = { - "node_id": await projects_api.add_project_node( + "node_id": await projects_service.add_project_node( request, project_data, req_ctx.user_id, @@ -191,13 +191,13 @@ async def get_node(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(NodePathParams, request) # ensure the project exists - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, ) - if await projects_api.is_project_node_deprecated( + if await projects_service.is_project_node_deprecated( request.app, req_ctx.user_id, project, @@ -229,13 +229,13 @@ async def patch_project_node(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(NodePathParams, request) node_patch = await parse_request_body_as(NodePatch, request) - await projects_api.update_project_node( + await projects_service.update_project_node( request.app, product_name=req_ctx.product_name, user_id=req_ctx.user_id, project_id=path_params.project_id, node_id=path_params.node_id, - node_patch=node_patch.to_model(), + partial_node=node_patch.to_model(), ) return web.json_response(status=status.HTTP_204_NO_CONTENT) @@ -250,12 +250,12 @@ async def delete_node(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(NodePathParams, request) # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, ) - await projects_api.delete_project_node( + await projects_service.delete_project_node( request, path_params.project_id, req_ctx.user_id, @@ -325,7 +325,7 @@ async def start_node(request: web.Request) -> web.Response: req_ctx = RequestContext.model_validate(request) path_params = parse_request_path_parameters_as(NodePathParams, request) - await projects_api.start_project_node( + await projects_service.start_project_node( request, product_name=req_ctx.product_name, user_id=req_ctx.user_id, @@ -435,7 +435,7 @@ async def get_node_resources(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(NodePathParams, request) # ensure the project exists - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -445,7 +445,7 @@ async def get_node_resources(request: web.Request) -> web.Response: node_id = f"{path_params.node_id}" raise NodeNotFoundError(project_uuid=project_uuid, node_uuid=node_id) - resources: ServiceResourcesDict = await projects_api.get_project_node_resources( + resources: ServiceResourcesDict = await projects_service.get_project_node_resources( request.app, user_id=req_ctx.user_id, project_id=path_params.project_id, @@ -469,7 +469,7 @@ async def replace_node_resources(request: web.Request) -> web.Response: body = await parse_request_body_as(ServiceResourcesDict, request) # ensure the project exists - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -479,7 +479,7 @@ async def replace_node_resources(request: web.Request) -> web.Response: project_uuid=f"{path_params.project_id}", node_uuid=f"{path_params.node_id}" ) try: - new_node_resources = await projects_api.update_project_node_resources( + new_node_resources = await projects_service.update_project_node_resources( request.app, user_id=req_ctx.user_id, project_id=path_params.project_id, @@ -531,7 +531,7 @@ async def get_project_services_access_for_gid( _ServicesAccessQuery, request ) - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -648,7 +648,7 @@ async def list_project_nodes_previews(request: web.Request) -> web.Response: assert req_ctx # nosec nodes_previews: list[_ProjectNodePreview] = [] - project_data = await projects_api.get_project_for_user( + project_data = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -687,7 +687,7 @@ async def get_project_node_preview(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(NodePathParams, request) assert req_ctx # nosec - project_data = await projects_api.get_project_for_user( + project_data = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_observer.py b/services/web/server/src/simcore_service_webserver/projects/_observer.py index e6267305e469..f830ae40f6f3 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_observer.py +++ b/services/web/server/src/simcore_service_webserver/projects/_observer.py @@ -17,7 +17,7 @@ from ..notifications import project_logs from ..resource_manager.user_sessions import PROJECT_ID_KEY, managed_resource -from .projects_api import retrieve_and_notify_project_locked_state +from .projects_service import retrieve_and_notify_project_locked_state _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py index eaacd9c1aa3f..388021a247fa 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py @@ -32,7 +32,7 @@ from ..login.decorators import login_required from ..projects._access_rights_api import check_user_project_permission from ..security.decorators import permission_required -from . import _ports_api, projects_api +from . import _ports_api, projects_service from ._common_models import ProjectPathParams, RequestContext from .db import ProjectDBAPI from .exceptions import ( @@ -81,7 +81,7 @@ async def wrapper(request: web.Request) -> web.Response: async def _get_validated_workbench_model( app: web.Application, project_id: ProjectID, user_id: UserID ) -> dict[NodeID, Node]: - project: ProjectDict = await projects_api.get_project_for_user( + project: ProjectDict = await projects_service.get_project_for_user( app, project_uuid=f"{project_id}", user_id=user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py index 05bb2f8e7676..2115f08bafd5 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py @@ -20,7 +20,7 @@ from ..resource_usage import api as rut_api from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import projects_api +from . import projects_service from ._common_models import RequestContext from ._nodes_handlers import NodePathParams from .db import ProjectDBAPI @@ -68,7 +68,7 @@ async def get_project_node_pricing_unit(request: web.Request): path_params = parse_request_path_parameters_as(NodePathParams, request) # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -117,7 +117,7 @@ async def connect_pricing_unit_to_project_node(request: web.Request): ) # ensure the project exists - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -143,7 +143,7 @@ async def connect_pricing_unit_to_project_node(request: web.Request): node_data = project["workbench"][NodeIDStr(f"{path_params.node_id}")] - await projects_api.update_project_node_resources_from_hardware_info( + await projects_service.update_project_node_resources_from_hardware_info( request.app, user_id=req_ctx.user_id, project_id=path_params.project_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index ebcec072a088..ec6c14c59207 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -26,7 +26,7 @@ async def update( async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.stream( projects_nodes.update() - .values(**NodeDB.model_construct(**node.model_dump()).model_dump(mode="json", exclude_unset=True, exclude_none=True)) + .values(**NodeDB.model_construct(**node.model_dump()).model_dump(mode="json", exclude_defaults=True, exclude_unset=True, exclude_none=True)) .where( sa.and_( projects_nodes.c.project_uuid == f"{project_id}", diff --git a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py index 8ec0400238cb..ef43b9b0766f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py @@ -35,7 +35,7 @@ from ..users.exceptions import UserDefaultWalletNotFoundError from ..utils_aiohttp import envelope_json_response from ..wallets.errors import WalletNotEnoughCreditsError -from . import projects_api +from . import projects_service from ._common_models import ProjectPathParams, RequestContext from .exceptions import ( DefaultPricingUnitNotFoundError, @@ -106,7 +106,7 @@ async def open_project(request: web.Request) -> web.Response: raise web.HTTPBadRequest(reason="Invalid request body") from exc try: - project_type: ProjectType = await projects_api.get_project_type( + project_type: ProjectType = await projects_service.get_project_type( request.app, path_params.project_id ) user_role: UserRole = await api.get_user_role( @@ -116,7 +116,7 @@ async def open_project(request: web.Request) -> web.Response: # only USERS/TESTERS can do that raise web.HTTPForbidden(reason="Wrong user role to open/edit a template") - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -128,7 +128,7 @@ async def open_project(request: web.Request) -> web.Response: product: Product = get_current_product(request) - if not await projects_api.try_open_project_for_user( + if not await projects_service.try_open_project_for_user( req_ctx.user_id, project_uuid=f"{path_params.project_id}", client_session_id=client_session_id, @@ -138,7 +138,7 @@ async def open_project(request: web.Request) -> web.Response: raise HTTPLockedError(reason="Project is locked, try later") # the project can be opened, let's update its product links - await projects_api.update_project_linked_product( + await projects_service.update_project_linked_product( request.app, path_params.project_id, req_ctx.product_name ) @@ -151,30 +151,30 @@ async def open_project(request: web.Request) -> web.Response: # NOTE: this method raises that exception when the number of dynamic # services in the project is highter than the maximum allowed per project # the project shall still open though. - await projects_api.run_project_dynamic_services( + await projects_service.run_project_dynamic_services( request, project, req_ctx.user_id, req_ctx.product_name ) # and let's update the project last change timestamp - await projects_api.update_project_last_change_timestamp( + await projects_service.update_project_last_change_timestamp( request.app, path_params.project_id ) # notify users that project is now opened - project = await projects_api.add_project_states_for_user( + project = await projects_service.add_project_states_for_user( user_id=req_ctx.user_id, project=project, is_template=False, app=request.app, ) - await projects_api.notify_project_state_update(request.app, project) + await projects_service.notify_project_state_update(request.app, project) return envelope_json_response(project) except DirectorServiceError as exc: # there was an issue while accessing the director-v2/director-v0 # ensure the project is closed again - await projects_api.try_close_project_for_user( + await projects_service.try_close_project_for_user( user_id=req_ctx.user_id, project_uuid=f"{path_params.project_id}", client_session_id=client_session_id, @@ -208,13 +208,13 @@ async def close_project(request: web.Request) -> web.Response: raise web.HTTPBadRequest(reason="Invalid request body") from exc # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, include_state=False, ) - await projects_api.try_close_project_for_user( + await projects_service.try_close_project_for_user( req_ctx.user_id, f"{path_params.project_id}", client_session_id, @@ -240,7 +240,7 @@ async def get_project_state(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(ProjectPathParams, request) # check that project exists and queries state - validated_project = await projects_api.get_project_for_user( + validated_project = await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_api.py b/services/web/server/src/simcore_service_webserver/projects/_trash_api.py index e15a98423c79..a8f1d23bb473 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_api.py @@ -13,7 +13,7 @@ from ..director_v2 import api as director_v2_api from ..dynamic_scheduler import api as dynamic_scheduler_api -from . import projects_api +from . import projects_service from ._access_rights_api import check_user_project_permission from .exceptions import ProjectRunningConflictError from .models import ProjectPatchExtended @@ -93,7 +93,7 @@ async def _schedule(): director_v2_api.stop_pipeline( app, user_id=user_id, project_id=project_id ), - projects_api.remove_project_dynamic_services( + projects_service.remove_project_dynamic_services( user_id=user_id, project_uuid=f"{project_id}", app=app, @@ -115,7 +115,7 @@ async def _schedule(): product_name=product_name, ) - await projects_api.patch_project( + await projects_service.patch_project( app, user_id=user_id, product_name=product_name, @@ -134,7 +134,7 @@ async def untrash_project( project_id: ProjectID, ): # NOTE: check_user_project_permission is inside projects_api.patch_project - await projects_api.patch_project( + await projects_service.patch_project( app, user_id=user_id, product_name=product_name, diff --git a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py index 56e7136d299d..2e104b0d7b1a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py @@ -19,7 +19,7 @@ from ..security.decorators import permission_required from ..wallets.errors import WalletAccessForbiddenError from . import _wallets_api as wallets_api -from . import projects_api +from . import projects_service from ._common_models import ProjectPathParams, RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError @@ -53,7 +53,7 @@ async def get_project_wallet(request: web.Request): path_params = parse_request_path_parameters_as(ProjectPathParams, request) # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, @@ -84,7 +84,7 @@ async def connect_wallet_to_project(request: web.Request): path_params = parse_request_path_parameters_as(_ProjectWalletPathParams, request) # ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/nodes_utils.py b/services/web/server/src/simcore_service_webserver/projects/nodes_utils.py index 41925557a20b..32531914163a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/nodes_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/nodes_utils.py @@ -12,7 +12,7 @@ from servicelib.logging_utils import log_decorator from servicelib.utils import fire_and_forget_task, logged_gather -from . import projects_api +from . import projects_service from .utils import get_frontend_node_outputs_changes log = logging.getLogger(__name__) @@ -46,7 +46,7 @@ async def update_node_outputs( ui_changed_keys: set[str] | None, ) -> None: # the new outputs might be {}, or {key_name: payload} - project, keys_changed = await projects_api.update_project_node_outputs( + project, keys_changed = await projects_service.update_project_node_outputs( app, user_id, project_uuid, @@ -55,14 +55,14 @@ async def update_node_outputs( new_run_hash=run_hash, ) - await projects_api.notify_project_node_update( + await projects_service.notify_project_node_update( app, project, node_uuid, errors=node_errors ) # get depending node and notify for these ones as well depending_node_uuids = await project_get_depending_nodes(project, node_uuid) await logged_gather( *[ - projects_api.notify_project_node_update(app, project, nid, errors=None) + projects_service.notify_project_node_update(app, project, nid, errors=None) for nid in depending_node_uuids ] ) @@ -86,7 +86,7 @@ async def update_node_outputs( ) # fire&forget to notify connected nodes to retrieve its inputs **if necessary** - await projects_api.post_trigger_connected_service_retrieve( + await projects_service.post_trigger_connected_service_retrieve( app=app, project=project, updated_node_uuid=f"{node_uuid}", changed_keys=keys ) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_service.py similarity index 99% rename from services/web/server/src/simcore_service_webserver/projects/projects_api.py rename to services/web/server/src/simcore_service_webserver/projects/projects_service.py index d309b807c9f1..2d2b474e88ed 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_service.py @@ -1001,10 +1001,10 @@ async def update_project_node( user_id: UserID, project_id: ProjectID, node_id: NodeID, - node_patch: Node, + partial_node: PartialNode, ) -> None: _node_patch_exclude_unset: dict[str, Any] = jsonable_encoder( - node_patch, exclude_unset=True, by_alias=True + partial_node, exclude_unset=True, by_alias=True ) db: ProjectDBAPI = app[APP_PROJECT_DBAPI] @@ -1048,7 +1048,7 @@ async def update_project_node( app, project_id=project_id, node_id=node_id, - node=node_patch, + node=partial_node, ) # 4. Make calls to director-v2 to keep data in sync (ex. comp_tasks DB table) diff --git a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py index 3c80c8322462..788ca886593f 100644 --- a/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py +++ b/services/web/server/src/simcore_service_webserver/studies_dispatcher/_projects.py @@ -23,7 +23,7 @@ from ..projects.db import ProjectDBAPI from ..projects.exceptions import ProjectInvalidRightsError, ProjectNotFoundError -from ..projects.projects_api import get_project_for_user +from ..projects.projects_service import get_project_for_user from ..utils import now_str from ._core import compose_uuid_from from ._models import FileParams, ServiceInfo, ViewerInfo diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__delete.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__delete.py index 80f7f4a7bd3b..0267820b7408 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__delete.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__delete.py @@ -36,7 +36,7 @@ from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects import _crud_api_delete from simcore_service_webserver.projects.models import ProjectDict -from simcore_service_webserver.projects.projects_api import lock_with_notification +from simcore_service_webserver.projects.projects_service import lock_with_notification from socketio.exceptions import ConnectionError as SocketConnectionError diff --git a/services/web/server/tests/unit/with_dbs/03/test_project_db.py b/services/web/server/tests/unit/with_dbs/03/test_project_db.py index 1ab6ca802f3d..eb265b2fe356 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_project_db.py +++ b/services/web/server/tests/unit/with_dbs/03/test_project_db.py @@ -39,7 +39,7 @@ ProjectNotFoundError, ) from simcore_service_webserver.projects.models import ProjectDict -from simcore_service_webserver.projects.projects_api import ( +from simcore_service_webserver.projects.projects_service import ( _check_project_node_has_all_required_inputs, ) from simcore_service_webserver.users.exceptions import UserNotFoundError diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py index fa05653b87ca..b409b18f1cd8 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py @@ -10,7 +10,7 @@ from aiohttp.test_utils import TestClient, make_mocked_request from faker import Faker from simcore_service_webserver._constants import RQT_USERID_KEY -from simcore_service_webserver.projects import projects_api +from simcore_service_webserver.projects import projects_service from simcore_service_webserver.projects.models import ProjectDict from simcore_service_webserver.version_control._core import ( checkout_checkpoint, @@ -81,7 +81,7 @@ async def test_workflow( checkpoint_co = await checkout_checkpoint(vc_repo, project_uuid, checkpoint1.id) assert checkpoint1 == checkpoint_co - project = await projects_api.get_project_for_user( + project = await projects_service.get_project_for_user( aiohttp_mocked_request.app, str(project_uuid), user_id ) assert project["workbench"] == user_project["workbench"] diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py index 325eb63e353e..1c87a9a55312 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py @@ -238,9 +238,9 @@ async def test_delete_project_and_repo( # TMP fix here waits ------------ # FIXME: mark as deleted, still gets entrypoints!! - from simcore_service_webserver.projects import projects_api + from simcore_service_webserver.projects import projects_service - delete_task = projects_api.get_delete_project_task(project_uuid, user_id) + delete_task = projects_service.get_delete_project_task(project_uuid, user_id) assert delete_task await delete_task # -------------------------------- diff --git a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py index 18cba7971775..99d0533031c2 100644 --- a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py @@ -47,7 +47,7 @@ from simcore_service_webserver.products.plugin import setup_products from simcore_service_webserver.projects.exceptions import ProjectNotFoundError from simcore_service_webserver.projects.plugin import setup_projects -from simcore_service_webserver.projects.projects_api import ( +from simcore_service_webserver.projects.projects_service import ( remove_project_dynamic_services, submit_delete_project_task, ) diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py index cd9bc502089e..639a28b186ed 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py @@ -18,7 +18,7 @@ from pytest_simcore.helpers.webserver_login import NewUser from pytest_simcore.helpers.webserver_projects import delete_all_projects from simcore_service_webserver.groups.api import auto_add_user_to_groups -from simcore_service_webserver.projects.projects_api import get_project_for_user +from simcore_service_webserver.projects.projects_service import get_project_for_user from simcore_service_webserver.studies_dispatcher._models import ServiceInfo from simcore_service_webserver.studies_dispatcher._projects import ( UserInfo, diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py index 366f10dba16f..ebf99e9e27c8 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -32,7 +32,7 @@ from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from settings_library.utils_session import DEFAULT_SESSION_COOKIE_NAME from simcore_service_webserver.projects.models import ProjectDict -from simcore_service_webserver.projects.projects_api import submit_delete_project_task +from simcore_service_webserver.projects.projects_service import submit_delete_project_task from simcore_service_webserver.users.api import ( delete_user_without_projects, get_user_role, From 9572145996d5f7233d58e9f2afe0218d141d3232 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Jan 2025 14:47:04 +0100 Subject: [PATCH 24/55] add missing column --- ...> 5c69e5661f45_move_projects_workbench.py} | 56 ++----------------- .../models/projects_nodes.py | 6 ++ .../projects/_projects_nodes_repository.py | 6 +- .../projects/projects_service.py | 4 +- 4 files changed, 16 insertions(+), 56 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/migration/versions/{876bae5ff8da_move_projects_workbench.py => 5c69e5661f45_move_projects_workbench.py} (57%) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py similarity index 57% rename from packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py rename to packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py index b34b79857f34..fd644a15599c 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/876bae5ff8da_move_projects_workbench.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py @@ -1,8 +1,8 @@ """Move projects workbench -Revision ID: 876bae5ff8da +Revision ID: 5c69e5661f45 Revises: 307017ee1a49 -Create Date: 2025-01-08 10:43:50.901038+00:00 +Create Date: 2025-01-13 13:45:47.583609+00:00 """ from alembic import op @@ -10,7 +10,7 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '876bae5ff8da' +revision = '5c69e5661f45' down_revision = '307017ee1a49' branch_labels = None depends_on = None @@ -26,6 +26,7 @@ def upgrade(): op.add_column('projects_nodes', sa.Column('input_access', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) op.add_column('projects_nodes', sa.Column('input_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) op.add_column('projects_nodes', sa.Column('inputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('projects_nodes', sa.Column('inputs_required', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) op.add_column('projects_nodes', sa.Column('inputs_units', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) op.add_column('projects_nodes', sa.Column('output_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) op.add_column('projects_nodes', sa.Column('outputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) @@ -35,54 +36,6 @@ def upgrade(): op.add_column('projects_nodes', sa.Column('boot_options', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) # ### end Alembic commands ### -# Populate the new columns with data from the workbench - op.execute( - """ -UPDATE projects_nodes -SET key = subquery.key, - version = subquery.version, - label = subquery.label, - progress = subquery.progress::numeric, - thumbnail = subquery.thumbnail, - input_access = subquery.input_access::jsonb, - input_nodes = subquery.input_nodes::jsonb, - inputs = subquery.inputs::jsonb, - inputs_units = subquery.inputs_units::jsonb, - output_nodes = subquery.output_nodes::jsonb, - outputs = subquery.outputs::jsonb, - run_hash = subquery.run_hash, - state = subquery.state::jsonb, - parent = subquery.parent, - boot_options = subquery.boot_options::jsonb -FROM ( - SELECT - projects.uuid AS project_id, - js.key AS node_id, - js.value::jsonb ->> 'key' AS key, - js.value::jsonb ->> 'label' AS label, - js.value::jsonb ->> 'version' AS version, - (js.value::jsonb ->> 'progress')::numeric AS progress, - js.value::jsonb ->> 'thumbnail' AS thumbnail, - js.value::jsonb ->> 'inputAccess' AS input_access, - js.value::jsonb ->> 'inputNodes' AS input_nodes, - js.value::jsonb ->> 'inputs' AS inputs, - js.value::jsonb ->> 'inputsUnits' AS inputs_units, - js.value::jsonb ->> 'outputNodes' AS output_nodes, - js.value::jsonb ->> 'outputs' AS outputs, - js.value::jsonb ->> 'runHash' AS run_hash, - js.value::jsonb ->> 'state' AS state, - js.value::jsonb ->> 'parent' AS parent, - js.value::jsonb ->> 'bootOptions' AS boot_options - FROM projects, - json_each(projects.workbench) AS js -) AS subquery -WHERE projects_nodes.project_uuid = subquery.project_id -AND projects_nodes.node_id = subquery.node_id; -""" - ) - op.alter_column("projects_nodes", "key", nullable=False) - op.alter_column("projects_nodes", "version", nullable=False) - op.alter_column("projects_nodes", "label", nullable=False) def downgrade(): # ### commands auto generated by Alembic - please adjust! ### @@ -93,6 +46,7 @@ def downgrade(): op.drop_column('projects_nodes', 'outputs') op.drop_column('projects_nodes', 'output_nodes') op.drop_column('projects_nodes', 'inputs_units') + op.drop_column('projects_nodes', 'inputs_required') op.drop_column('projects_nodes', 'inputs') op.drop_column('projects_nodes', 'input_nodes') op.drop_column('projects_nodes', 'input_access') diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index 5179392ec308..1bc377a6c8e8 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -105,6 +105,12 @@ nullable=True, doc="Input properties values", ), + sa.Column( + "inputs_required", + JSONB, # Array + nullable=True, + doc="Input IDs that are required", + ), sa.Column( "inputs_units", JSONB, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index ec6c14c59207..b68081cc51ee 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -10,7 +10,6 @@ from sqlalchemy.ext.asyncio import AsyncConnection from ..db.plugin import get_asyncpg_engine -from .models import NodeDB _logger = logging.getLogger(__name__) @@ -21,12 +20,13 @@ async def update( *, project_id: ProjectID, node_id: NodeID, - node: PartialNode, + partial_node: PartialNode, ) -> None: + values = partial_node.model_dump(mode="json", exclude_unset=True) async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.stream( projects_nodes.update() - .values(**NodeDB.model_construct(**node.model_dump()).model_dump(mode="json", exclude_defaults=True, exclude_unset=True, exclude_none=True)) + .values(**values) .where( sa.and_( projects_nodes.c.project_uuid == f"{project_id}", diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_service.py index 2d2b474e88ed..0ee9f0d024f7 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_service.py @@ -1048,7 +1048,7 @@ async def update_project_node( app, project_id=project_id, node_id=node_id, - node=partial_node, + partial_node=partial_node, ) # 4. Make calls to director-v2 to keep data in sync (ex. comp_tasks DB table) @@ -1116,7 +1116,7 @@ async def update_project_node_outputs( app, project_id=project_id, node_id=node_id, - node=PartialNode.model_construct(outputs=new_outputs, run_hash=new_run_hash), + partial_node=PartialNode.model_construct(outputs=new_outputs, run_hash=new_run_hash), ) log.debug( From 851c5bedc88a7c7a0c6e76959e40845f6af39761 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 13 Jan 2025 15:42:29 +0100 Subject: [PATCH 25/55] fix tests --- services/web/server/tests/unit/with_dbs/02/conftest.py | 6 +++--- .../with_dbs/02/test_projects_nodes_handlers__patch.py | 8 ++++---- .../02/test_projects_nodes_pricing_unit_handlers.py | 2 +- .../tests/unit/with_dbs/03/version_control/conftest.py | 4 ++-- .../server/tests/unit/with_dbs/04/folders/test_folders.py | 2 +- .../04/garbage_collector/test_resource_manager.py | 2 +- .../test_studies_dispatcher_handlers.py | 2 +- .../test_studies_dispatcher_studies_access.py | 2 +- .../test_workspaces__folders_and_projects_crud.py | 2 +- services/web/server/tests/unit/with_dbs/conftest.py | 2 +- 10 files changed, 16 insertions(+), 16 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/02/conftest.py b/services/web/server/tests/unit/with_dbs/02/conftest.py index 148d6315d8df..25be7db87c8c 100644 --- a/services/web/server/tests/unit/with_dbs/02/conftest.py +++ b/services/web/server/tests/unit/with_dbs/02/conftest.py @@ -92,12 +92,12 @@ def mock_catalog_api( ) -> dict[str, mock.Mock]: return { "get_service_resources": mocker.patch( - "simcore_service_webserver.projects.projects_api.catalog_client.get_service_resources", + "simcore_service_webserver.projects.projects_service.catalog_client.get_service_resources", return_value=mock_service_resources, autospec=True, ), "get_service": mocker.patch( - "simcore_service_webserver.projects.projects_api.catalog_client.get_service", + "simcore_service_webserver.projects.projects_service.catalog_client.get_service", return_value=mock_service, autospec=True, ), @@ -374,7 +374,7 @@ def mock_get_total_project_dynamic_nodes_creation_interval( ) -> None: _VERY_LONG_LOCK_TIMEOUT_S: Final[float] = 300 mocker.patch( - "simcore_service_webserver.projects.projects_api._nodes_api" + "simcore_service_webserver.projects.projects_service._nodes_api" ".get_total_project_dynamic_nodes_creation_interval", return_value=_VERY_LONG_LOCK_TIMEOUT_S, ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py index f9af27e93981..1f6f18cc00a3 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py @@ -47,7 +47,7 @@ def mock_project_uses_available_services(mocker: MockerFixture): @pytest.fixture def mock_catalog_rpc_check_for_service(mocker: MockerFixture): mocker.patch( - "simcore_service_webserver.projects.projects_api.catalog_rpc.check_for_service", + "simcore_service_webserver.projects.projects_service.catalog_rpc.check_for_service", spec=True, return_value=True, ) @@ -56,7 +56,7 @@ def mock_catalog_rpc_check_for_service(mocker: MockerFixture): @pytest.fixture def mocked_notify_project_node_update(mocker: MockerFixture): return mocker.patch( - "simcore_service_webserver.projects.projects_api.notify_project_node_update", + "simcore_service_webserver.projects.projects_service.notify_project_node_update", ) @@ -362,14 +362,14 @@ async def test_patch_project_node_service_key_with_error( _patch_version = {"version": "2.0.9"} with mocker.patch( - "simcore_service_webserver.projects.projects_api.catalog_rpc.check_for_service", + "simcore_service_webserver.projects.projects_service.catalog_rpc.check_for_service", side_effect=CatalogForbiddenError(name="test"), ): resp = await client.patch(f"{base_url}", json=_patch_version) assert resp.status == status.HTTP_403_FORBIDDEN with mocker.patch( - "simcore_service_webserver.projects.projects_api.catalog_rpc.check_for_service", + "simcore_service_webserver.projects.projects_service.catalog_rpc.check_for_service", side_effect=CatalogItemNotFoundError(name="test"), ): resp = await client.patch(f"{base_url}", json=_patch_version) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py index 5812190c354e..be88694b3074 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py @@ -136,7 +136,7 @@ def _fake_instance_type_details( ] return mocker.patch( - "simcore_service_webserver.projects.projects_api.get_instance_type_details", + "simcore_service_webserver.projects.projects_service.get_instance_type_details", side_effect=_fake_instance_type_details, ) diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index e9a6244886c7..81668ae2bc93 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -254,11 +254,11 @@ async def request_delete_project( mocker: MockerFixture, ) -> AsyncIterator[Callable[[TestClient, UUID], Awaitable]]: director_v2_api_delete_pipeline: mock.AsyncMock = mocker.patch( - "simcore_service_webserver.projects.projects_api.director_v2_api.delete_pipeline", + "simcore_service_webserver.projects.projects_service.director_v2_api.delete_pipeline", autospec=True, ) dynamic_scheduler_api_stop_dynamic_services_in_project: mock.AsyncMock = mocker.patch( - "simcore_service_webserver.projects.projects_api.dynamic_scheduler_api.stop_dynamic_services_in_project", + "simcore_service_webserver.projects.projects_service.dynamic_scheduler_api.stop_dynamic_services_in_project", autospec=True, ) fire_and_forget_call_to_storage: mock.Mock = mocker.patch( diff --git a/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py b/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py index 03e30daedc40..f4b2df540ae8 100644 --- a/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py +++ b/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py @@ -387,7 +387,7 @@ def mock_storage_delete_data_folders(mocker: MockerFixture) -> mock.Mock: autospec=True, ) mocker.patch( - "simcore_service_webserver.projects.projects_api.remove_project_dynamic_services", + "simcore_service_webserver.projects.projects_service.remove_project_dynamic_services", autospec=True, ) mocker.patch( diff --git a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py index 99d0533031c2..044d8b4270e6 100644 --- a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py @@ -657,7 +657,7 @@ async def test_interactive_services_remain_after_websocket_reconnection_from_2_t async def mocked_notification_system(mocker): mocks = {} mocked_notification_system = mocker.patch( - "simcore_service_webserver.projects.projects_api.retrieve_and_notify_project_locked_state", + "simcore_service_webserver.projects.projects_service.retrieve_and_notify_project_locked_state", return_value=Future(), ) mocked_notification_system.return_value.set_result("") diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py index 86ed849075fa..22fa00d1b3f5 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py @@ -302,7 +302,7 @@ def mocks_on_projects_api(mocker) -> None: All projects in this module are UNLOCKED """ mocker.patch( - "simcore_service_webserver.projects.projects_api._get_project_lock_state", + "simcore_service_webserver.projects.projects_service._get_project_lock_state", return_value=ProjectLocked(value=False, status=ProjectStatus.CLOSED), ) diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py index ebf99e9e27c8..2df1faaccbd3 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -132,7 +132,7 @@ def mocks_on_projects_api(mocker: MockerFixture) -> None: All projects in this module are UNLOCKED """ mocker.patch( - "simcore_service_webserver.projects.projects_api._get_project_lock_state", + "simcore_service_webserver.projects.projects_service._get_project_lock_state", return_value=ProjectLocked(value=False, status=ProjectStatus.CLOSED), ) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py index 3d5c2d7991a7..1084efacde32 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py @@ -259,7 +259,7 @@ def mock_storage_delete_data_folders(mocker: MockerFixture) -> mock.Mock: autospec=True, ) mocker.patch( - "simcore_service_webserver.projects.projects_api.remove_project_dynamic_services", + "simcore_service_webserver.projects.projects_service.remove_project_dynamic_services", autospec=True, ) mocker.patch( diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 6661af40d5e3..765f7559b048 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -369,7 +369,7 @@ async def _mock_result(): ) mock2 = mocker.patch( - "simcore_service_webserver.projects.projects_api.storage_api.delete_data_folders_of_project_node", + "simcore_service_webserver.projects.projects_service.storage_api.delete_data_folders_of_project_node", autospec=True, return_value=None, ) From d2fd817655870843e9a57f9091e98b8cbc72892c Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Jan 2025 13:20:08 +0100 Subject: [PATCH 26/55] store only set values --- .../src/simcore_postgres_database/utils_projects_nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py index b71008f6ba2c..9cab49d27fa9 100644 --- a/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/utils_projects_nodes.py @@ -98,7 +98,7 @@ async def add( [ { "project_uuid": f"{self.project_uuid}", - **node.model_dump(), + **node.model_dump(exclude_unset=True), } for node in nodes ] From 55b60bc970efbd03a8550e71185fb46f2ae6026a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Jan 2025 14:33:25 +0100 Subject: [PATCH 27/55] fix update node state --- .../projects/_crud_handlers.py | 2 +- .../projects/projects_service.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index 0c6a7e509eef..0f0e2c3a99bc 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -48,7 +48,7 @@ from ..users.api import get_user_fullname from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError from . import _crud_api_create, _crud_api_read, projects_service -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from ._crud_handlers_models import ( ProjectActiveQueryParams, ProjectCreateHeaders, diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_service.py index 0ee9f0d024f7..444edced883a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_service.py @@ -38,7 +38,7 @@ from models_library.products import ProductName from models_library.projects import Project, ProjectID, ProjectIDStr from models_library.projects_access import Owner -from models_library.projects_nodes import Node, PartialNode +from models_library.projects_nodes import Node, NodeState, PartialNode from models_library.projects_nodes_io import NodeID, NodeIDStr, PortLink from models_library.projects_state import ( ProjectLocked, @@ -977,6 +977,7 @@ async def update_project_node_state( permission="write", # NOTE: MD: before only read was sufficient, double check this ) + # TODO: delete this once workbench is removed from the projects table updated_project, _ = await db.update_project_node_data( user_id=user_id, project_uuid=project_id, @@ -984,6 +985,14 @@ async def update_project_node_state( product_name=None, new_node_data={"state": {"currentStatus": new_state}}, ) + + await _projects_nodes_repository.update( + app, + project_id=project_id, + node_id=node_id, + partial_node=PartialNode.model_construct(state=NodeState(currentStatus=RunningState(new_state))), + ) + return await add_project_states_for_user( user_id=user_id, project=updated_project, is_template=False, app=app ) From d73fed1493ad3b2ea17ea8bb1461db22c56aefcc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Jan 2025 14:38:50 +0100 Subject: [PATCH 28/55] fix packages --- .../simcore_service_webserver/projects/_comments_handlers.py | 2 +- .../src/simcore_service_webserver/projects/_nodes_handlers.py | 2 +- .../src/simcore_service_webserver/projects/_ports_handlers.py | 2 +- .../projects/_projects_nodes_pricing_unit_handlers.py | 2 +- .../src/simcore_service_webserver/projects/_states_handlers.py | 2 +- .../src/simcore_service_webserver/projects/_wallets_handlers.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py index a8596b7407f5..04ac3d5ca35a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py @@ -31,7 +31,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _comments_api, projects_service -from ._common_models import RequestContext +from ._common.models import RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 4330d9cdbb9d..902b207968d2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -75,7 +75,7 @@ from ..utils_aiohttp import envelope_json_response from ..wallets.errors import WalletAccessForbiddenError, WalletNotEnoughCreditsError from . import nodes_utils, projects_service -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from ._nodes_api import NodeScreenshot, get_node_screenshots from .exceptions import ( ClustersKeeperNotAvailableError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py index 388021a247fa..b134929a8afb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py @@ -33,7 +33,7 @@ from ..projects._access_rights_api import check_user_project_permission from ..security.decorators import permission_required from . import _ports_api, projects_service -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .db import ProjectDBAPI from .exceptions import ( NodeNotFoundError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py index 2115f08bafd5..a2797993c9a2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py @@ -21,7 +21,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import projects_service -from ._common_models import RequestContext +from ._common.models import RequestContext from ._nodes_handlers import NodePathParams from .db import ProjectDBAPI from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError diff --git a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py index ef43b9b0766f..319868c115e9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py @@ -36,7 +36,7 @@ from ..utils_aiohttp import envelope_json_response from ..wallets.errors import WalletNotEnoughCreditsError from . import projects_service -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .exceptions import ( DefaultPricingUnitNotFoundError, ProjectInvalidRightsError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py index 2e104b0d7b1a..9f820e284053 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py @@ -20,7 +20,7 @@ from ..wallets.errors import WalletAccessForbiddenError from . import _wallets_api as wallets_api from . import projects_service -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError _logger = logging.getLogger(__name__) From ac8c9a6a36d39a6090113680134e057fb3f47177 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Jan 2025 14:57:39 +0100 Subject: [PATCH 29/55] update sql --- .../projects/_projects_nodes_repository.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index b68081cc51ee..2b2007a53438 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -1,6 +1,5 @@ import logging -import sqlalchemy as sa from aiohttp import web from models_library.projects import ProjectID from models_library.projects_nodes import PartialNode @@ -28,9 +27,7 @@ async def update( projects_nodes.update() .values(**values) .where( - sa.and_( - projects_nodes.c.project_uuid == f"{project_id}", - projects_nodes.c.node_id == f"{node_id}", - ) + (projects_nodes.c.project_uuid == f"{project_id}") + & (projects_nodes.c.node_id == f"{node_id}") ) ) From 6a865f3322102644aa3e6902656594dbdf9048a5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Jan 2025 15:47:58 +0100 Subject: [PATCH 30/55] fix name --- .../tests/unit/with_dbs/02/test_projects_states_handlers.py | 2 +- services/web/server/tests/unit/with_dbs/03/test_trash.py | 2 +- .../server/tests/unit/with_dbs/03/version_control/conftest.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index 6fcd177f37b2..de6f73d5cbfd 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -1000,7 +1000,7 @@ async def test_project_node_lifetime( # noqa: PLR0915 create_dynamic_service_mock: Callable[..., Awaitable[DynamicServiceGet]], ): mock_storage_api_delete_data_folders_of_project_node = mocker.patch( - "simcore_service_webserver.projects._crud_handlers.projects_api.storage_api.delete_data_folders_of_project_node", + "simcore_service_webserver.projects._crud_handlers.projects_service.storage_api.delete_data_folders_of_project_node", return_value="", ) assert client.app diff --git a/services/web/server/tests/unit/with_dbs/03/test_trash.py b/services/web/server/tests/unit/with_dbs/03/test_trash.py index 9080eb74fd82..867f0a9736d2 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_trash.py +++ b/services/web/server/tests/unit/with_dbs/03/test_trash.py @@ -76,7 +76,7 @@ async def test_trash_projects( # noqa: PLR0915 # this test should have no errors stopping services mock_remove_dynamic_services = mocker.patch( - "simcore_service_webserver.projects._trash_api.projects_api.remove_project_dynamic_services", + "simcore_service_webserver.projects._trash_api.projects_service.remove_project_dynamic_services", autospec=True, ) mock_stop_pipeline = mocker.patch( diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index bde4f51bf90d..92484c96b3bd 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -140,12 +140,12 @@ def request_update_project( mocker: MockerFixture, ) -> Callable[[TestClient, UUID], Awaitable]: mocker.patch( - "simcore_service_webserver.projects._nodes_handlers.projects_api.is_service_deprecated", + "simcore_service_webserver.projects._nodes_handlers.projects_service.is_service_deprecated", autospec=True, return_value=False, ) mocker.patch( - "simcore_service_webserver.projects._nodes_handlers.projects_api.catalog_client.get_service_resources", + "simcore_service_webserver.projects._nodes_handlers.projects_service.catalog_client.get_service_resources", autospec=True, return_value=ServiceResourcesDict(), ) From 459d56795ce3b15f191e60eabda557f85944e29a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 14 Jan 2025 15:49:07 +0100 Subject: [PATCH 31/55] fix pylint --- .../src/simcore_service_webserver/projects/_nodes_handlers.py | 1 - .../src/simcore_service_webserver/projects/projects_service.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 902b207968d2..9dc630f22685 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -27,7 +27,6 @@ ) from models_library.groups import EVERYONE_GROUP_ID, Group, GroupID, GroupType from models_library.projects import Project, ProjectID -from models_library.projects_nodes import Node from models_library.projects_nodes_io import NodeID, NodeIDStr from models_library.services import ServiceKeyVersion from models_library.services_resources import ServiceResourcesDict diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_service.py index 444edced883a..43b7662ed8e6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_service.py @@ -121,7 +121,7 @@ from ..wallets import api as wallets_api from ..wallets.errors import WalletNotEnoughCreditsError from ..workspaces import _workspaces_db as workspaces_db -from . import _crud_api_delete, _nodes_api, _projects_db, _projects_nodes_repository +from . import _crud_api_delete, _nodes_api, _projects_db from ._access_rights_api import ( check_user_project_permission, has_user_project_access_rights, From be92b63ff7f3a7768f445af422071b786b188cb7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 13:02:36 +0100 Subject: [PATCH 32/55] revert _base.py --- .../src/models_library/api_schemas_webserver/_base.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/_base.py b/packages/models-library/src/models_library/api_schemas_webserver/_base.py index 44edd74e5096..a5eaa42c0066 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/_base.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/_base.py @@ -22,8 +22,9 @@ class InputSchemaWithoutCamelCase(BaseModel): ) -class InputSchema(InputSchemaWithoutCamelCase): +class InputSchema(BaseModel): model_config = ConfigDict( + **InputSchemaWithoutCamelCase.model_config, alias_generator=snake_to_camel, ) @@ -31,14 +32,17 @@ class InputSchema(InputSchemaWithoutCamelCase): class OutputSchemaWithoutCamelCase(BaseModel): model_config = ConfigDict( populate_by_name=True, - extra="ignore", # Used to prune extra fields from internal data + extra="ignore", frozen=True, ) -class OutputSchema(OutputSchemaWithoutCamelCase): +class OutputSchema(BaseModel): model_config = ConfigDict( alias_generator=snake_to_camel, + populate_by_name=True, + extra="ignore", # Used to prune extra fields from internal data + frozen=True, ) def data( From be56df7de8102f170df754a3ddc4d826bd585083 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 13:20:52 +0100 Subject: [PATCH 33/55] add migration SQL cmd --- .../5c69e5661f45_move_projects_workbench.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py index fd644a15599c..9818e9464dad 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py @@ -36,6 +36,54 @@ def upgrade(): op.add_column('projects_nodes', sa.Column('boot_options', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) # ### end Alembic commands ### + # Populate the new columns with data from the workbench + op.execute( +""" +UPDATE projects_nodes +SET key = subquery.key, + version = subquery.version, + label = subquery.label, + progress = subquery.progress::numeric, + thumbnail = subquery.thumbnail, + input_access = subquery.input_access::jsonb, + input_nodes = subquery.input_nodes::jsonb, + inputs = subquery.inputs::jsonb, + inputs_required = subquery.inputs_required::jsonb, + inputs_units = subquery.inputs_units::jsonb, + output_nodes = subquery.output_nodes::jsonb, + outputs = subquery.outputs::jsonb, + run_hash = subquery.run_hash, + state = subquery.state::jsonb, + parent = subquery.parent, + boot_options = subquery.boot_options::jsonb +FROM ( + SELECT + projects.uuid AS project_id, + js.key AS node_id, + js.value::jsonb ->> 'key' AS key, + js.value::jsonb ->> 'label' AS label, + js.value::jsonb ->> 'version' AS version, + (js.value::jsonb ->> 'progress')::numeric AS progress, + js.value::jsonb ->> 'thumbnail' AS thumbnail, + js.value::jsonb ->> 'inputAccess' AS input_access, + js.value::jsonb ->> 'inputNodes' AS input_nodes, + js.value::jsonb ->> 'inputs' AS inputs, + js.value::jsonb ->> 'inputsRequired' AS inputs_required, + js.value::jsonb ->> 'inputsUnits' AS inputs_units, + js.value::jsonb ->> 'outputNodes' AS output_nodes, + js.value::jsonb ->> 'outputs' AS outputs, + js.value::jsonb ->> 'runHash' AS run_hash, + js.value::jsonb ->> 'state' AS state, + js.value::jsonb ->> 'parent' AS parent, + js.value::jsonb ->> 'bootOptions' AS boot_options + FROM projects, + json_each(projects.workbench) AS js +) AS subquery +WHERE projects_nodes.project_uuid = subquery.project_id +AND projects_nodes.node_id = subquery.node_id; +""" +) + def downgrade(): # ### commands auto generated by Alembic - please adjust! ### From a79bc7e2f141507d590b704e7584ec9fc7c3d336 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 13:43:24 +0100 Subject: [PATCH 34/55] update script --- ...> 50af59b6c37a_move_projects_workbench.py} | 47 +++++++++---------- .../models/projects_nodes.py | 43 +++++++++-------- 2 files changed, 43 insertions(+), 47 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/migration/versions/{5c69e5661f45_move_projects_workbench.py => 50af59b6c37a_move_projects_workbench.py} (69%) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/50af59b6c37a_move_projects_workbench.py similarity index 69% rename from packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py rename to packages/postgres-database/src/simcore_postgres_database/migration/versions/50af59b6c37a_move_projects_workbench.py index 9818e9464dad..88bddcb0c1da 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/5c69e5661f45_move_projects_workbench.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/50af59b6c37a_move_projects_workbench.py @@ -1,8 +1,8 @@ """Move projects workbench -Revision ID: 5c69e5661f45 +Revision ID: 50af59b6c37a Revises: 307017ee1a49 -Create Date: 2025-01-13 13:45:47.583609+00:00 +Create Date: 2025-01-15 12:39:46.219516+00:00 """ from alembic import op @@ -10,33 +10,30 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '5c69e5661f45' +revision = '50af59b6c37a' down_revision = '307017ee1a49' branch_labels = None depends_on = None def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('projects_nodes', sa.Column('key', sa.String(), nullable=False)) - op.add_column('projects_nodes', sa.Column('version', sa.String(), nullable=False)) - op.add_column('projects_nodes', sa.Column('label', sa.String(), nullable=False)) - op.add_column('projects_nodes', sa.Column('progress', sa.Numeric(), nullable=True)) - op.add_column('projects_nodes', sa.Column('thumbnail', sa.String(), nullable=True)) - op.add_column('projects_nodes', sa.Column('input_access', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) - op.add_column('projects_nodes', sa.Column('input_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) - op.add_column('projects_nodes', sa.Column('inputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) - op.add_column('projects_nodes', sa.Column('inputs_required', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) - op.add_column('projects_nodes', sa.Column('inputs_units', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) - op.add_column('projects_nodes', sa.Column('output_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) - op.add_column('projects_nodes', sa.Column('outputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) - op.add_column('projects_nodes', sa.Column('run_hash', sa.String(), nullable=True)) - op.add_column('projects_nodes', sa.Column('state', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) - op.add_column('projects_nodes', sa.Column('parent', sa.String(), nullable=True)) - op.add_column('projects_nodes', sa.Column('boot_options', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) - # ### end Alembic commands ### + op.add_column('projects_nodes', sa.Column('key', sa.String(), nullable=False, comment='Distinctive name (based on the Docker registry path)')) + op.add_column('projects_nodes', sa.Column('version', sa.String(), nullable=False, comment='Semantic version number')) + op.add_column('projects_nodes', sa.Column('label', sa.String(), nullable=False, comment='Short name ')) + op.add_column('projects_nodes', sa.Column('progress', sa.Numeric(), nullable=True, comment='Progress value')) + op.add_column('projects_nodes', sa.Column('thumbnail', sa.String(), nullable=True, comment='Url of the latest screenshot')) + op.add_column('projects_nodes', sa.Column('input_access', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Map with key - access level pairs')) + op.add_column('projects_nodes', sa.Column('input_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='IDs of the nodes where is connected to')) + op.add_column('projects_nodes', sa.Column('inputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Input properties values')) + op.add_column('projects_nodes', sa.Column('inputs_required', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Required input IDs')) + op.add_column('projects_nodes', sa.Column('inputs_units', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Input units')) + op.add_column('projects_nodes', sa.Column('output_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Node IDs of those connected to the output')) + op.add_column('projects_nodes', sa.Column('outputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Output properties values')) + op.add_column('projects_nodes', sa.Column('run_hash', sa.String(), nullable=True, comment='HEX digest of the resolved inputs + outputs hash at the time when the last outputs were generated')) + op.add_column('projects_nodes', sa.Column('state', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='State')) + op.add_column('projects_nodes', sa.Column('parent', sa.String(), nullable=True, comment="Parent's (group-nodes) node ID")) + op.add_column('projects_nodes', sa.Column('boot_options', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment="Some services provide alternative parameters to be injected at boot time.The user selection should be stored here, and it will overwrite the services's defaults")) - # Populate the new columns with data from the workbench op.execute( """ UPDATE projects_nodes @@ -83,10 +80,11 @@ def upgrade(): AND projects_nodes.node_id = subquery.node_id; """ ) - + op.alter_column("projects_nodes", "key", nullable=False) + op.alter_column("projects_nodes", "version", nullable=False) + op.alter_column("projects_nodes", "label", nullable=False) def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### op.drop_column('projects_nodes', 'boot_options') op.drop_column('projects_nodes', 'parent') op.drop_column('projects_nodes', 'state') @@ -103,4 +101,3 @@ def downgrade(): op.drop_column('projects_nodes', 'label') op.drop_column('projects_nodes', 'version') op.drop_column('projects_nodes', 'key') - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index 1bc377a6c8e8..993315077da8 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -25,7 +25,7 @@ nullable=False, autoincrement=True, primary_key=True, - doc="Project node index", + comment="Index of the project node", ), sa.Column( "project_uuid", @@ -38,21 +38,21 @@ ), nullable=False, index=True, - doc="Project unique identifier", + comment="Unique identifier of the project", ), sa.Column( "node_id", sa.String, nullable=False, index=True, - doc="Node unique identifier", + comment="Unique identifier of the node", ), sa.Column( "required_resources", JSONB, nullable=False, server_default=sa.text("'{}'::jsonb"), - doc="Node required resources", + comment="Required resources", ), # TIME STAMPS ---- column_created_datetime(timezone=True), @@ -61,97 +61,97 @@ "key", sa.String, nullable=False, - doc="Distinctive name for the node based on the docker registry path", + comment="Distinctive name (based on the Docker registry path)", ), sa.Column( "version", sa.String, nullable=False, - doc="Semantic version number of the node", + comment="Semantic version number", ), sa.Column( "label", sa.String, nullable=False, - doc="Short name of the node", + comment="Short name ", ), sa.Column( "progress", sa.Numeric, nullable=True, - doc="Node progress value", + comment="Progress value", ), sa.Column( "thumbnail", sa.String, nullable=True, - doc="Url of the latest screenshot of the node", + comment="Url of the latest screenshot", ), sa.Column( "input_access", JSONB, nullable=True, - doc="Map with key - access level pairs", + comment="Map with key - access level pairs", ), sa.Column( "input_nodes", JSONB, # Array nullable=True, - doc="Node IDs of where the node is connected to", + comment="IDs of the nodes where is connected to", ), sa.Column( "inputs", JSONB, nullable=True, - doc="Input properties values", + comment="Input properties values", ), sa.Column( "inputs_required", JSONB, # Array nullable=True, - doc="Input IDs that are required", + comment="Required input IDs", ), sa.Column( "inputs_units", JSONB, nullable=True, - doc="Input unit values", + comment="Input units", ), sa.Column( "output_nodes", JSONB, # Array nullable=True, - doc="Node IDs of those connected to the output", + comment="Node IDs of those connected to the output", ), sa.Column( "outputs", JSONB, nullable=True, - doc="Output properties values", + comment="Output properties values", ), # sa.Column( # "output_node", # sa.BOOLEAN, # nullable=True, - # doc="Deprecated", + # comment="Deprecated", # ), sa.Column( "run_hash", sa.String, nullable=True, - doc="HEX digest of the resolved inputs + outputs hash at the time when the last outputs were generated", + comment="HEX digest of the resolved inputs + outputs hash at the time when the last outputs were generated", ), sa.Column( "state", JSONB, nullable=True, - doc="Node state", + comment="State", ), sa.Column( "parent", sa.String, nullable=True, - doc="Parent's (group-nodes) node ID", + comment="Parent's (group-nodes) node ID", ), # sa.Column( # "position", @@ -163,11 +163,10 @@ "boot_options", JSONB, nullable=True, - doc="Some services provide alternative parameters to be injected at boot time." + comment="Some services provide alternative parameters to be injected at boot time." "The user selection should be stored here, and it will overwrite the services's defaults", ), sa.UniqueConstraint("project_uuid", "node_id"), ) - register_modified_datetime_auto_update_trigger(projects_nodes) From c48529deaf1d87d535761637291a6acd0e06231d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 14:09:13 +0100 Subject: [PATCH 35/55] fix pylint --- .../src/models_library/api_schemas_webserver/projects_nodes.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 25c53615b28f..676a9bf4e1e2 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -3,8 +3,6 @@ from pydantic import ConfigDict, Field -from common_library.dict_tools import remap_keys -from models_library.projects_nodes import Node from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut from ..basic_types import PortInt From 5d92523a56f987012ab2a6aace8b9188e97f8b8e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 14:29:12 +0100 Subject: [PATCH 36/55] fix model --- .../models-library/src/models_library/projects_nodes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/projects_nodes.py b/packages/models-library/src/models_library/projects_nodes.py index d95c42b032cd..3fec1406c579 100644 --- a/packages/models-library/src/models_library/projects_nodes.py +++ b/packages/models-library/src/models_library/projects_nodes.py @@ -277,6 +277,6 @@ def _convert_from_enum(cls, v): class PartialNode(Node): - key: Annotated[ServiceKey | None, Field(default=None)] - version: Annotated[ServiceVersion | None, Field(default=None)] - label: Annotated[str | None, Field(default=None)] + key: Annotated[ServiceKey, Field(default=None)] + version: Annotated[ServiceVersion, Field(default=None)] + label: Annotated[str, Field(default=None)] From 733055a5351936fa9664815b1c4d166b46dbb4e7 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 14:51:20 +0100 Subject: [PATCH 37/55] blank --- .../src/models_library/api_schemas_webserver/projects_nodes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 676a9bf4e1e2..4138e706b971 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -3,7 +3,6 @@ from pydantic import ConfigDict, Field - from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut from ..basic_types import PortInt from ..projects_nodes import InputID, InputsDict, PartialNode From e93491fcfcd18c54a7a3fd9a5bf9c67731bda3cc Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 15:58:07 +0100 Subject: [PATCH 38/55] add get --- .../projects/_projects_nodes_repository.py | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 2b2007a53438..3c33acd63426 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -1,18 +1,60 @@ import logging +import sqlalchemy as sa + from aiohttp import web from models_library.projects import ProjectID -from models_library.projects_nodes import PartialNode +from models_library.projects_nodes import Node, PartialNode from models_library.projects_nodes_io import NodeID from simcore_postgres_database.utils_repos import transaction_context from simcore_postgres_database.webserver_models import projects_nodes from sqlalchemy.ext.asyncio import AsyncConnection +from .exceptions import NodeNotFoundError from ..db.plugin import get_asyncpg_engine _logger = logging.getLogger(__name__) +_SELECTION_PROJECTS_NODES_DB_ARGS = [ + projects_nodes.c.id, + projects_nodes.c.key, + projects_nodes.c.version, + projects_nodes.c.label, + projects_nodes.c.thumbnail, + projects_nodes.c.inputs, + projects_nodes.c.outputs, +] + + +async def get( + app: web.Application, + connection: AsyncConnection | None = None, + *, + project_id: ProjectID, + node_id: NodeID, +) -> Node: + async with transaction_context(get_asyncpg_engine(app), connection) as conn: + get_stmt = sa.select( + *_SELECTION_PROJECTS_NODES_DB_ARGS + ).where( + (projects_nodes.c.project_uuid == f"{project_id}") + & (projects_nodes.c.node_id == f"{node_id}") + ) + + result = await conn.stream(get_stmt) + assert result # nosec + + row = await result.first() + if row is None: + raise NodeNotFoundError( + project_uuid=f"{project_id}", + node_uuid=f"{node_id}" + ) + assert row # nosec + return Node.model_validate(row, from_attributes=True) + + async def update( app: web.Application, connection: AsyncConnection | None = None, @@ -22,6 +64,7 @@ async def update( partial_node: PartialNode, ) -> None: values = partial_node.model_dump(mode="json", exclude_unset=True) + async with transaction_context(get_asyncpg_engine(app), connection) as conn: await conn.stream( projects_nodes.update() From 31c4f5061de1d92c2f825dfecbf8f3e870015e51 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 16:50:17 +0100 Subject: [PATCH 39/55] fix column --- .../projects/_projects_nodes_repository.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 3c33acd63426..a433e53328b8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -17,7 +17,6 @@ _SELECTION_PROJECTS_NODES_DB_ARGS = [ - projects_nodes.c.id, projects_nodes.c.key, projects_nodes.c.version, projects_nodes.c.label, From 8f44d30a6bdf2d378bece0af7594f07fc68bc026 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 17:10:05 +0100 Subject: [PATCH 40/55] add more columns --- .../projects/_projects_nodes_repository.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index a433e53328b8..427d95046b9f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -20,12 +20,23 @@ projects_nodes.c.key, projects_nodes.c.version, projects_nodes.c.label, + projects_nodes.c.progress, projects_nodes.c.thumbnail, + projects_nodes.c.input_access, + projects_nodes.c.input_nodes, projects_nodes.c.inputs, + projects_nodes.c.inputs_required, + projects_nodes.c.inputs_units, + projects_nodes.c.output_nodes, projects_nodes.c.outputs, + projects_nodes.c.run_hash, + projects_nodes.c.state, + projects_nodes.c.parent, + projects_nodes.c.boot_options, ] + async def get( app: web.Application, connection: AsyncConnection | None = None, From 2d45d5cbce8de9a364146e49c4a4ca1d270ecd0a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 15 Jan 2025 17:13:53 +0100 Subject: [PATCH 41/55] blank --- .../projects/_projects_nodes_repository.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py index 427d95046b9f..cd6880732e0a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_repository.py @@ -36,7 +36,6 @@ ] - async def get( app: web.Application, connection: AsyncConnection | None = None, From 623f926027c9e1995d3512d012185397355c631f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Thu, 16 Jan 2025 09:52:39 +0100 Subject: [PATCH 42/55] add issue reference --- .../src/simcore_service_webserver/projects/projects_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_service.py index 4c63e588f274..0504855c3667 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_service.py @@ -975,7 +975,8 @@ async def update_project_node_state( permission="write", # NOTE: MD: before only read was sufficient, double check this ) - # TODO: delete this once workbench is removed from the projects table + # TODO: https://github.com/ITISFoundation/osparc-simcore/issues/7046 + # delete this once workbench is removed from the projects table updated_project, _ = await db.update_project_node_data( user_id=user_id, project_uuid=project_id, From 2abb1e2d827a41492462d4c8a6eb2e686d3ae1a1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 20 Jan 2025 14:24:05 +0100 Subject: [PATCH 43/55] fixes --- .../src/models_library/api_schemas_webserver/projects_nodes.py | 2 +- .../src/simcore_service_webserver/projects/_nodes_handlers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py index 4138e706b971..69aafba0962a 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py @@ -59,7 +59,7 @@ class NodePatch(InputSchemaWithoutCamelCase): str, Any ] | None = None # NOTE: it is used by frontend for File Picker - def to_model(self) -> PartialNode: + def to_domain_model(self) -> PartialNode: data = self.model_dump( exclude_unset=True, by_alias=True, diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 9dc630f22685..43e38a62a005 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -234,7 +234,7 @@ async def patch_project_node(request: web.Request) -> web.Response: user_id=req_ctx.user_id, project_id=path_params.project_id, node_id=path_params.node_id, - partial_node=node_patch.to_model(), + partial_node=node_patch.to_domain_model(), ) return web.json_response(status=status.HTTP_204_NO_CONTENT) From 61c13e959ade0c8a47f1150b994ebe5cdf950335 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 20 Jan 2025 14:24:45 +0100 Subject: [PATCH 44/55] requild script --- ...py => 37858a94d3de_move_workbench_column.py} | 17 +++++++++++------ .../models/projects_nodes.py | 8 ++++---- 2 files changed, 15 insertions(+), 10 deletions(-) rename packages/postgres-database/src/simcore_postgres_database/migration/versions/{50af59b6c37a_move_projects_workbench.py => 37858a94d3de_move_workbench_column.py} (93%) diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/50af59b6c37a_move_projects_workbench.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/37858a94d3de_move_workbench_column.py similarity index 93% rename from packages/postgres-database/src/simcore_postgres_database/migration/versions/50af59b6c37a_move_projects_workbench.py rename to packages/postgres-database/src/simcore_postgres_database/migration/versions/37858a94d3de_move_workbench_column.py index 88bddcb0c1da..6a6848e12758 100644 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/50af59b6c37a_move_projects_workbench.py +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/37858a94d3de_move_workbench_column.py @@ -1,8 +1,8 @@ -"""Move projects workbench +"""move workbench column -Revision ID: 50af59b6c37a -Revises: 307017ee1a49 -Create Date: 2025-01-15 12:39:46.219516+00:00 +Revision ID: 37858a94d3de +Revises: f19905923355 +Create Date: 2025-01-20 13:23:14.397622+00:00 """ from alembic import op @@ -10,13 +10,14 @@ from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '50af59b6c37a' -down_revision = '307017ee1a49' +revision = '37858a94d3de' +down_revision = 'f19905923355' branch_labels = None depends_on = None def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### op.add_column('projects_nodes', sa.Column('key', sa.String(), nullable=False, comment='Distinctive name (based on the Docker registry path)')) op.add_column('projects_nodes', sa.Column('version', sa.String(), nullable=False, comment='Semantic version number')) op.add_column('projects_nodes', sa.Column('label', sa.String(), nullable=False, comment='Short name ')) @@ -33,6 +34,7 @@ def upgrade(): op.add_column('projects_nodes', sa.Column('state', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='State')) op.add_column('projects_nodes', sa.Column('parent', sa.String(), nullable=True, comment="Parent's (group-nodes) node ID")) op.add_column('projects_nodes', sa.Column('boot_options', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment="Some services provide alternative parameters to be injected at boot time.The user selection should be stored here, and it will overwrite the services's defaults")) + # ### end Alembic commands ### op.execute( """ @@ -84,7 +86,9 @@ def upgrade(): op.alter_column("projects_nodes", "version", nullable=False) op.alter_column("projects_nodes", "label", nullable=False) + def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### op.drop_column('projects_nodes', 'boot_options') op.drop_column('projects_nodes', 'parent') op.drop_column('projects_nodes', 'state') @@ -101,3 +105,4 @@ def downgrade(): op.drop_column('projects_nodes', 'label') op.drop_column('projects_nodes', 'version') op.drop_column('projects_nodes', 'key') + # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index 993315077da8..7298ad79a162 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -25,7 +25,7 @@ nullable=False, autoincrement=True, primary_key=True, - comment="Index of the project node", + doc="Index of the project node", ), sa.Column( "project_uuid", @@ -38,21 +38,21 @@ ), nullable=False, index=True, - comment="Unique identifier of the project", + doc="Unique identifier of the project", ), sa.Column( "node_id", sa.String, nullable=False, index=True, - comment="Unique identifier of the node", + doc="Unique identifier of the node", ), sa.Column( "required_resources", JSONB, nullable=False, server_default=sa.text("'{}'::jsonb"), - comment="Required resources", + doc="Required resources", ), # TIME STAMPS ---- column_created_datetime(timezone=True), From 9d80eddf2b480ba12289f2a30d406aa50db1b7ff Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 20 Jan 2025 14:33:14 +0100 Subject: [PATCH 45/55] remove validation --- .../src/simcore_service_webserver/projects/_crud_api_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 0414c5b2d9d3..f4ae4554388b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -132,7 +132,7 @@ async def _copy_project_nodes_from_source_project( db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(app) def _mapped_node_id(node: ProjectNode) -> NodeID: - return NodeID(nodes_map[TypeAdapter(NodeIDStr).validate_python(f"{node.node_id}")]) + return NodeID(nodes_map[f"{node.node_id}"]) return { _mapped_node_id(node): ProjectNodeCreate( From 931bbecf8ae0731dcbf62c126b79a0744b094110 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 20 Jan 2025 15:02:18 +0100 Subject: [PATCH 46/55] rename method --- .../projects/_nodes_handlers.py | 2 +- .../projects/projects_service.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 43e38a62a005..dab67c9aaa8e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -228,7 +228,7 @@ async def patch_project_node(request: web.Request) -> web.Response: path_params = parse_request_path_parameters_as(NodePathParams, request) node_patch = await parse_request_body_as(NodePatch, request) - await projects_service.update_project_node( + await projects_service.patch_project_node( request.app, product_name=req_ctx.product_name, user_id=req_ctx.user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_service.py index b7fdde702f3f..6b4d760f7a2f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_service.py @@ -1000,7 +1000,7 @@ async def is_project_hidden(app: web.Application, project_id: ProjectID) -> bool return await db.is_hidden(project_id) -async def update_project_node( +async def patch_project_node( app: web.Application, *, product_name: ProductName, @@ -1009,10 +1009,9 @@ async def update_project_node( node_id: NodeID, partial_node: PartialNode, ) -> None: - _node_patch_exclude_unset: dict[str, Any] = jsonable_encoder( - partial_node, exclude_unset=True, by_alias=True - ) - db: ProjectDBAPI = app[APP_PROJECT_DBAPI] + _node_patch_exclude_unset: dict[str, Any] = partial_node.model_dump(mode='json', exclude_unset=True, by_alias=True) + + _projects_repository = ProjectDBAPI.get_from_app_context(app) # 1. Check user permissions await check_user_project_permission( @@ -1025,7 +1024,7 @@ async def update_project_node( # 2. If patching service key or version make sure it's valid if _node_patch_exclude_unset.get("key") or _node_patch_exclude_unset.get("version"): - _project, _ = await db.get_project(project_uuid=f"{project_id}") + _project, _ = await _projects_repository.get_project(project_uuid=f"{project_id}") _project_node_data = _project["workbench"][f"{node_id}"] _service_key = _node_patch_exclude_unset.get("key", _project_node_data["key"]) @@ -1042,7 +1041,7 @@ async def update_project_node( ) # 3. Patch the project node - updated_project, _ = await db.update_project_node_data( + updated_project, _ = await _projects_repository.update_project_node_data( user_id=user_id, project_uuid=project_id, node_id=node_id, From cdd0e959fd7b3f9235a2b32032c800b816e6bc8e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 20 Jan 2025 15:05:55 +0100 Subject: [PATCH 47/55] pylint --- .../projects/_crud_api_create.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index f4ae4554388b..2656b0a0f9c3 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -10,7 +10,7 @@ from models_library.api_schemas_long_running_tasks.base import ProgressPercent from models_library.api_schemas_webserver.projects import ProjectGet from models_library.projects import ProjectID -from models_library.projects_nodes_io import NodeID, NodeIDStr +from models_library.projects_nodes_io import NodeID from models_library.projects_state import ProjectStatus from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder @@ -155,9 +155,10 @@ async def _copy_files_from_source_project( user_id: UserID, task_progress: TaskProgress, ): - db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(app) + _projects_repository = ProjectDBAPI.get_from_app_context(app) + needs_lock_source_project: bool = ( - await db.get_project_type( + await _projects_repository.get_project_type( TypeAdapter(ProjectID).validate_python(source_project["uuid"]) ) != ProjectTypeDB.TEMPLATE @@ -263,7 +264,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche """ assert request.app # nosec - db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(request.app) + _projects_repository = ProjectDBAPI.get_from_app_context(request.app) new_project: ProjectDict = {} copy_file_coro = None @@ -340,7 +341,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche ) # 3.1 save new project in DB - new_project = await db.insert_project( + new_project = await _projects_repository.insert_project( project=jsonable_encoder(new_project), user_id=user_id, product_name=product_name, @@ -376,7 +377,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche # 5. unhide the project if needed since it is now complete if not new_project_was_hidden_before_data_was_copied: - await db.set_hidden_flag(new_project["uuid"], hidden=False) + await _projects_repository.set_hidden_flag(new_project["uuid"], hidden=False) # update the network information in director-v2 await dynamic_scheduler_api.update_projects_networks( @@ -389,7 +390,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche request.app, user_id, new_project["uuid"], product_name ) # get the latest state of the project (lastChangeDate for instance) - new_project, _ = await db.get_project(project_uuid=new_project["uuid"]) + new_project, _ = await _projects_repository.get_project(project_uuid=new_project["uuid"]) # Appends state new_project = await projects_service.add_project_states_for_user( user_id=user_id, @@ -403,7 +404,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche await update_or_pop_permalink_in_project(request, new_project) # Adds folderId - user_specific_project_data_db = await db.get_user_specific_project_data_db( + user_specific_project_data_db = await _projects_repository.get_user_specific_project_data_db( project_uuid=new_project["uuid"], private_workspace_user_id_or_none=user_id if workspace_id is None else None, ) From 904d6e9eed11fa42eae1c43a8051788a8f3e883a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 20 Jan 2025 15:40:51 +0100 Subject: [PATCH 48/55] update comment --- .../src/simcore_postgres_database/models/projects_nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index 7298ad79a162..5a1e71fcddc2 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -73,7 +73,7 @@ "label", sa.String, nullable=False, - comment="Short name ", + comment="Short name used for display", ), sa.Column( "progress", From 3f7f275d8f83a58939d2b0806a33c48dd844ab01 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 20 Jan 2025 19:07:27 +0100 Subject: [PATCH 49/55] remove deprecated --- .../models/projects_nodes.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index 5a1e71fcddc2..e6228c076311 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -129,12 +129,6 @@ nullable=True, comment="Output properties values", ), - # sa.Column( - # "output_node", - # sa.BOOLEAN, - # nullable=True, - # comment="Deprecated", - # ), sa.Column( "run_hash", sa.String, @@ -153,12 +147,6 @@ nullable=True, comment="Parent's (group-nodes) node ID", ), - # sa.Column( - # "position", - # JSONB, - # nullable=True, - # doc="Deprecated", - # ), sa.Column( "boot_options", JSONB, From 893e9edd19a8f91b75787c0a48fa24cd6cdd6297 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 21 Jan 2025 08:30:10 +0100 Subject: [PATCH 50/55] remove unused model --- .../projects/models.py | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/models.py b/services/web/server/src/simcore_service_webserver/projects/models.py index 72e755ae31ad..eab20e2f848f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/models.py +++ b/services/web/server/src/simcore_service_webserver/projects/models.py @@ -7,7 +7,6 @@ from models_library.api_schemas_webserver.projects import ProjectPatch from models_library.folders import FolderID from models_library.projects import ClassifierID, ProjectID -from models_library.projects_nodes_io import NodeID from models_library.projects_ui import StudyUI from models_library.users import UserID from models_library.utils.common_validators import ( @@ -107,33 +106,3 @@ def to_domain_model(self) -> dict[str, Any]: self.model_dump(exclude_unset=True, by_alias=False), rename={"trashed_at": "trashed"}, ) - - -class NodeDB(BaseModel): - node_id: NodeID - required_resources: dict[str, Any] - created: datetime - modified: datetime - project_uuid: ProjectID - project_node_id: int - key: str - version: str - label: str - progress: float | None - thumbnail: HttpUrl | None - input_access: dict[str, Any] - input_nodes: list[NodeID] - inputs: dict[str, Any] - inputs_units: dict[str, Any] - output_nodes: list[NodeID] - outputs: dict[str, Any] - run_hash: str | None - state: dict[str, Any] | None - parent: str | None - boot_options: dict[str, Any] | None - - -__all__: tuple[str, ...] = ( - "ProjectDict", - "ProjectProxy", -) From febe93544904223d11aed65dc48947e5fd81c4f1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 21 Jan 2025 11:46:29 +0100 Subject: [PATCH 51/55] update comment --- .../src/simcore_postgres_database/models/projects_nodes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py index e6228c076311..a5991e7f9db8 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects_nodes.py @@ -79,7 +79,7 @@ "progress", sa.Numeric, nullable=True, - comment="Progress value", + comment="Progress value (0-100)", ), sa.Column( "thumbnail", From 3a168e9829adf34417e721ee7dd45ef2cb9bf9be Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 21 Jan 2025 14:15:01 +0100 Subject: [PATCH 52/55] update script --- .../37858a94d3de_move_workbench_column.py | 108 --------- .../ecd4eadaa781_extract_workbench_column.py | 227 ++++++++++++++++++ 2 files changed, 227 insertions(+), 108 deletions(-) delete mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/37858a94d3de_move_workbench_column.py create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd4eadaa781_extract_workbench_column.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/37858a94d3de_move_workbench_column.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/37858a94d3de_move_workbench_column.py deleted file mode 100644 index 6a6848e12758..000000000000 --- a/packages/postgres-database/src/simcore_postgres_database/migration/versions/37858a94d3de_move_workbench_column.py +++ /dev/null @@ -1,108 +0,0 @@ -"""move workbench column - -Revision ID: 37858a94d3de -Revises: f19905923355 -Create Date: 2025-01-20 13:23:14.397622+00:00 - -""" -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -# revision identifiers, used by Alembic. -revision = '37858a94d3de' -down_revision = 'f19905923355' -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.add_column('projects_nodes', sa.Column('key', sa.String(), nullable=False, comment='Distinctive name (based on the Docker registry path)')) - op.add_column('projects_nodes', sa.Column('version', sa.String(), nullable=False, comment='Semantic version number')) - op.add_column('projects_nodes', sa.Column('label', sa.String(), nullable=False, comment='Short name ')) - op.add_column('projects_nodes', sa.Column('progress', sa.Numeric(), nullable=True, comment='Progress value')) - op.add_column('projects_nodes', sa.Column('thumbnail', sa.String(), nullable=True, comment='Url of the latest screenshot')) - op.add_column('projects_nodes', sa.Column('input_access', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Map with key - access level pairs')) - op.add_column('projects_nodes', sa.Column('input_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='IDs of the nodes where is connected to')) - op.add_column('projects_nodes', sa.Column('inputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Input properties values')) - op.add_column('projects_nodes', sa.Column('inputs_required', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Required input IDs')) - op.add_column('projects_nodes', sa.Column('inputs_units', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Input units')) - op.add_column('projects_nodes', sa.Column('output_nodes', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Node IDs of those connected to the output')) - op.add_column('projects_nodes', sa.Column('outputs', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='Output properties values')) - op.add_column('projects_nodes', sa.Column('run_hash', sa.String(), nullable=True, comment='HEX digest of the resolved inputs + outputs hash at the time when the last outputs were generated')) - op.add_column('projects_nodes', sa.Column('state', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment='State')) - op.add_column('projects_nodes', sa.Column('parent', sa.String(), nullable=True, comment="Parent's (group-nodes) node ID")) - op.add_column('projects_nodes', sa.Column('boot_options', postgresql.JSONB(astext_type=sa.Text()), nullable=True, comment="Some services provide alternative parameters to be injected at boot time.The user selection should be stored here, and it will overwrite the services's defaults")) - # ### end Alembic commands ### - - op.execute( -""" -UPDATE projects_nodes -SET key = subquery.key, - version = subquery.version, - label = subquery.label, - progress = subquery.progress::numeric, - thumbnail = subquery.thumbnail, - input_access = subquery.input_access::jsonb, - input_nodes = subquery.input_nodes::jsonb, - inputs = subquery.inputs::jsonb, - inputs_required = subquery.inputs_required::jsonb, - inputs_units = subquery.inputs_units::jsonb, - output_nodes = subquery.output_nodes::jsonb, - outputs = subquery.outputs::jsonb, - run_hash = subquery.run_hash, - state = subquery.state::jsonb, - parent = subquery.parent, - boot_options = subquery.boot_options::jsonb -FROM ( - SELECT - projects.uuid AS project_id, - js.key AS node_id, - js.value::jsonb ->> 'key' AS key, - js.value::jsonb ->> 'label' AS label, - js.value::jsonb ->> 'version' AS version, - (js.value::jsonb ->> 'progress')::numeric AS progress, - js.value::jsonb ->> 'thumbnail' AS thumbnail, - js.value::jsonb ->> 'inputAccess' AS input_access, - js.value::jsonb ->> 'inputNodes' AS input_nodes, - js.value::jsonb ->> 'inputs' AS inputs, - js.value::jsonb ->> 'inputsRequired' AS inputs_required, - js.value::jsonb ->> 'inputsUnits' AS inputs_units, - js.value::jsonb ->> 'outputNodes' AS output_nodes, - js.value::jsonb ->> 'outputs' AS outputs, - js.value::jsonb ->> 'runHash' AS run_hash, - js.value::jsonb ->> 'state' AS state, - js.value::jsonb ->> 'parent' AS parent, - js.value::jsonb ->> 'bootOptions' AS boot_options - FROM projects, - json_each(projects.workbench) AS js -) AS subquery -WHERE projects_nodes.project_uuid = subquery.project_id -AND projects_nodes.node_id = subquery.node_id; -""" -) - op.alter_column("projects_nodes", "key", nullable=False) - op.alter_column("projects_nodes", "version", nullable=False) - op.alter_column("projects_nodes", "label", nullable=False) - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('projects_nodes', 'boot_options') - op.drop_column('projects_nodes', 'parent') - op.drop_column('projects_nodes', 'state') - op.drop_column('projects_nodes', 'run_hash') - op.drop_column('projects_nodes', 'outputs') - op.drop_column('projects_nodes', 'output_nodes') - op.drop_column('projects_nodes', 'inputs_units') - op.drop_column('projects_nodes', 'inputs_required') - op.drop_column('projects_nodes', 'inputs') - op.drop_column('projects_nodes', 'input_nodes') - op.drop_column('projects_nodes', 'input_access') - op.drop_column('projects_nodes', 'thumbnail') - op.drop_column('projects_nodes', 'progress') - op.drop_column('projects_nodes', 'label') - op.drop_column('projects_nodes', 'version') - op.drop_column('projects_nodes', 'key') - # ### end Alembic commands ### diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd4eadaa781_extract_workbench_column.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd4eadaa781_extract_workbench_column.py new file mode 100644 index 000000000000..264419b14d63 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/ecd4eadaa781_extract_workbench_column.py @@ -0,0 +1,227 @@ +"""extract workbench column + +Revision ID: ecd4eadaa781 +Revises: a3a58471b0f1 +Create Date: 2025-01-21 13:13:18.256109+00:00 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "ecd4eadaa781" +down_revision = "a3a58471b0f1" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "projects_nodes", + sa.Column( + "key", + sa.String(), + nullable=False, + comment="Distinctive name (based on the Docker registry path)", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "version", sa.String(), nullable=False, comment="Semantic version number" + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "label", sa.String(), nullable=False, comment="Short name used for display" + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "progress", sa.Numeric(), nullable=True, comment="Progress value (0-100)" + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "thumbnail", + sa.String(), + nullable=True, + comment="Url of the latest screenshot", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "input_access", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Map with key - access level pairs", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "input_nodes", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="IDs of the nodes where is connected to", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "inputs", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Input properties values", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "inputs_required", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Required input IDs", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "inputs_units", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Input units", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "output_nodes", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Node IDs of those connected to the output", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "outputs", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Output properties values", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "run_hash", + sa.String(), + nullable=True, + comment="HEX digest of the resolved inputs + outputs hash at the time when the last outputs were generated", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "state", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="State", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "parent", + sa.String(), + nullable=True, + comment="Parent's (group-nodes) node ID", + ), + ) + op.add_column( + "projects_nodes", + sa.Column( + "boot_options", + postgresql.JSONB(astext_type=sa.Text()), + nullable=True, + comment="Some services provide alternative parameters to be injected at boot time.The user selection should be stored here, and it will overwrite the services's defaults", + ), + ) + # ### end Alembic commands ### + + op.execute( + """ +UPDATE projects_nodes +SET key = subquery.key, + version = subquery.version, + label = subquery.label, + progress = subquery.progress::numeric, + thumbnail = subquery.thumbnail, + input_access = subquery.input_access::jsonb, + input_nodes = subquery.input_nodes::jsonb, + inputs = subquery.inputs::jsonb, + inputs_required = subquery.inputs_required::jsonb, + inputs_units = subquery.inputs_units::jsonb, + output_nodes = subquery.output_nodes::jsonb, + outputs = subquery.outputs::jsonb, + run_hash = subquery.run_hash, + state = subquery.state::jsonb, + parent = subquery.parent, + boot_options = subquery.boot_options::jsonb +FROM ( + SELECT + projects.uuid AS project_id, + js.key AS node_id, + js.value::jsonb ->> 'key' AS key, + js.value::jsonb ->> 'label' AS label, + js.value::jsonb ->> 'version' AS version, + (js.value::jsonb ->> 'progress')::numeric AS progress, + js.value::jsonb ->> 'thumbnail' AS thumbnail, + js.value::jsonb ->> 'inputAccess' AS input_access, + js.value::jsonb ->> 'inputNodes' AS input_nodes, + js.value::jsonb ->> 'inputs' AS inputs, + js.value::jsonb ->> 'inputsRequired' AS inputs_required, + js.value::jsonb ->> 'inputsUnits' AS inputs_units, + js.value::jsonb ->> 'outputNodes' AS output_nodes, + js.value::jsonb ->> 'outputs' AS outputs, + js.value::jsonb ->> 'runHash' AS run_hash, + js.value::jsonb ->> 'state' AS state, + js.value::jsonb ->> 'parent' AS parent, + js.value::jsonb ->> 'bootOptions' AS boot_options + FROM projects, + json_each(projects.workbench) AS js +) AS subquery +WHERE projects_nodes.project_uuid = subquery.project_id +AND projects_nodes.node_id = subquery.node_id; +""" + ) + op.alter_column("projects_nodes", "key", nullable=False) + op.alter_column("projects_nodes", "version", nullable=False) + op.alter_column("projects_nodes", "label", nullable=False) + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("projects_nodes", "boot_options") + op.drop_column("projects_nodes", "parent") + op.drop_column("projects_nodes", "state") + op.drop_column("projects_nodes", "run_hash") + op.drop_column("projects_nodes", "outputs") + op.drop_column("projects_nodes", "output_nodes") + op.drop_column("projects_nodes", "inputs_units") + op.drop_column("projects_nodes", "inputs_required") + op.drop_column("projects_nodes", "inputs") + op.drop_column("projects_nodes", "input_nodes") + op.drop_column("projects_nodes", "input_access") + op.drop_column("projects_nodes", "thumbnail") + op.drop_column("projects_nodes", "progress") + op.drop_column("projects_nodes", "label") + op.drop_column("projects_nodes", "version") + op.drop_column("projects_nodes", "key") + # ### end Alembic commands ### From 58c1aedfba7d75f796ec4fb1e09965f85198dd18 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 21 Jan 2025 14:22:23 +0100 Subject: [PATCH 53/55] fix sonar --- .../simcore_service_webserver/projects/projects_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_service.py index 615d76e97bf3..e34c784da73a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_service.py @@ -975,8 +975,8 @@ async def update_project_node_state( permission="write", # NOTE: MD: before only read was sufficient, double check this ) - # TODO: https://github.com/ITISFoundation/osparc-simcore/issues/7046 - # delete this once workbench is removed from the projects table + # Delete this once workbench is removed from the projects table + # See: https://github.com/ITISFoundation/osparc-simcore/issues/7046 updated_project, _ = await db.update_project_node_data( user_id=user_id, project_uuid=project_id, From 65b43a1d49a4b1095c28df067705a1a7ab817583 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 21 Jan 2025 15:54:49 +0100 Subject: [PATCH 54/55] fix names --- .../src/simcore_service_webserver/projects/_crud_api_create.py | 2 +- .../src/simcore_service_webserver/projects/_wallets_handlers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 12182a1f3cc3..e083e16b28ae 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -191,7 +191,7 @@ async def _copy() -> None: owner=Owner( user_id=user_id, **await get_user_fullname(app, user_id=user_id) ), - notification_cb=projects_api.create_user_notification_cb( + notification_cb=projects_service.create_user_notification_cb( user_id, ProjectID(f"{source_project['uuid']}"), app ), )(_copy)() diff --git a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py index c74a1a982bd7..346958c34879 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py @@ -141,7 +141,7 @@ async def pay_project_debt(request: web.Request): body_params = await parse_request_body_as(_PayProjectDebtBody, request) # Ensure the project exists - await projects_api.get_project_for_user( + await projects_service.get_project_for_user( request.app, project_uuid=f"{path_params.project_id}", user_id=req_ctx.user_id, From fc408e1118131e8beee6551d6ca2861ef80be0bd Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 21 Jan 2025 15:58:07 +0100 Subject: [PATCH 55/55] fix import --- .../src/simcore_service_webserver/projects/_states_handlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py index 881ec6635fea..956226d7f32d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py @@ -34,6 +34,7 @@ from ..users.exceptions import UserDefaultWalletNotFoundError from ..utils_aiohttp import envelope_json_response from ..wallets.errors import WalletNotEnoughCreditsError +from . import api as projects_api from . import projects_service from ._common.models import ProjectPathParams, RequestContext from .exceptions import ( @@ -131,7 +132,7 @@ async def open_project(request: web.Request) -> web.Response: ), ) - await projects_service.check_project_financial_status( + await projects_api.check_project_financial_status( request.app, project_id=path_params.project_id, product_name=req_ctx.product_name,