diff --git a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py index 014aff565298..a7f9ec22fd04 100644 --- a/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py +++ b/packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port.py @@ -114,10 +114,9 @@ def check_value(cls, v: DataItemValue, info: ValidationInfo) -> DataItemValue: and not isinstance(v, PortLink) ): if port_utils.is_file_type(property_type): - if not isinstance(v, (FileLink, DownloadLink)): - raise ValueError( - f"{property_type!r} value does not validate against any of FileLink, DownloadLink or PortLink schemas" - ) + if not isinstance(v, FileLink | DownloadLink): + msg = f"{property_type!r} value does not validate against any of FileLink, DownloadLink or PortLink schemas" + raise ValueError(msg) elif property_type == "ref_contentSchema": v, _ = validate_port_content( port_key=info.data.get("key"), @@ -125,10 +124,11 @@ def check_value(cls, v: DataItemValue, info: ValidationInfo) -> DataItemValue: unit=None, content_schema=info.data.get("content_schema", {}), ) - elif isinstance(v, (list, dict)): - raise TypeError( + elif isinstance(v, list | dict): + msg = ( f"Containers as {v} currently only supported within content_schema." ) + raise TypeError(msg) return v @field_validator("value_item", "value_concrete", mode="before") @@ -194,28 +194,29 @@ async def get_value( ) async def _evaluate() -> ItemValue | None: + # NOTE: review types returned by this function !!! if isinstance(self.value, PortLink): # this is a link to another node's port - other_port_itemvalue: None | ( - ItemValue - ) = await port_utils.get_value_link_from_port_link( - self.value, - # pylint: disable=protected-access - self._node_ports._node_ports_creator_cb, - file_link_type=file_link_type, + other_port_itemvalue: ItemValue | None = ( + await port_utils.get_value_link_from_port_link( + self.value, + # pylint: disable=protected-access + self._node_ports._node_ports_creator_cb, + file_link_type=file_link_type, + ) ) return other_port_itemvalue if isinstance(self.value, FileLink): # let's get the download/upload link from storage - url_itemvalue: None | ( - AnyUrl - ) = await port_utils.get_download_link_from_storage( - # pylint: disable=protected-access - user_id=self._node_ports.user_id, - value=self.value, - link_type=file_link_type, + url_itemvalue: AnyUrl | None = ( + await port_utils.get_download_link_from_storage( + # pylint: disable=protected-access + user_id=self._node_ports.user_id, + value=self.value, + link_type=file_link_type, + ) ) return url_itemvalue @@ -256,15 +257,15 @@ async def _evaluate() -> ItemConcreteValue | None: if isinstance(self.value, PortLink): # this is a link to another node - other_port_concretevalue: None | ( - ItemConcreteValue - ) = await port_utils.get_value_from_link( - # pylint: disable=protected-access - key=self.key, - value=self.value, - file_to_key_map=self.file_to_key_map, - node_port_creator=self._node_ports._node_ports_creator_cb, # noqa: SLF001 - progress_bar=progress_bar, + other_port_concretevalue: None | ItemConcreteValue = ( + await port_utils.get_value_from_link( + # pylint: disable=protected-access + key=self.key, + value=self.value, + file_to_key_map=self.file_to_key_map, + node_port_creator=self._node_ports._node_ports_creator_cb, # noqa: SLF001 + progress_bar=progress_bar, + ) ) value = other_port_concretevalue diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py index eba9954771cd..10103909a631 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/repositories/comp_tasks/_utils.py @@ -400,6 +400,9 @@ async def generate_tasks_list_from_project( raise WalletNotEnoughCreditsError( wallet_name=wallet_info.wallet_name, wallet_credit_amount=wallet_info.wallet_credit_amount, + user_id=user_id, + product_name=product_name, + project_id=project.uuid, ) assert rabbitmq_rpc_client # nosec diff --git a/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py b/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py index f47250b651e9..6e11cf8b40c2 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/db/tables.py @@ -10,17 +10,21 @@ ) from simcore_postgres_database.models.projects import ProjectType, projects from simcore_postgres_database.models.projects_networks import projects_networks +from simcore_postgres_database.models.projects_nodes import projects_nodes -__all__ = [ +__all__: tuple[str, ...] = ( + "NodeClass", + "ProjectType", + "StateType", "comp_pipeline", + "comp_run_snapshot_tasks", "comp_runs", "comp_tasks", "groups_extra_properties", - "NodeClass", - "projects_networks", "projects", - "ProjectType", - "StateType", + "projects_networks", + "projects_nodes", "user_to_groups", - "comp_run_snapshot_tasks", -] +) + +# nopycln: file diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dags.py b/services/director-v2/src/simcore_service_director_v2/utils/dags.py index 13a2dfe8f393..f0a55669c836 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dags.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dags.py @@ -29,6 +29,7 @@ def create_complete_dag(workbench: NodesDict) -> nx.DiGraph: dag_graph: nx.DiGraph = nx.DiGraph() for node_id, node in workbench.items(): assert node.state # nosec + dag_graph.add_node( node_id, name=node.label, @@ -43,6 +44,9 @@ def create_complete_dag(workbench: NodesDict) -> nx.DiGraph: if node.input_nodes: for input_node_id in node.input_nodes: predecessor_node = workbench.get(f"{input_node_id}") + assert ( # nosec + predecessor_node + ), f"Node {input_node_id} not found in workbench" if predecessor_node: dag_graph.add_edge(str(input_node_id), node_id) diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index d704cd3043a6..07364b9219ad 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -14109,12 +14109,6 @@ components: - $ref: '#/components/schemas/NodeState-Input' - type: 'null' description: The node's state object - required_resources: - anyOf: - - additionalProperties: true - type: object - - type: 'null' - title: Required Resources bootOptions: anyOf: - type: object @@ -14252,12 +14246,6 @@ components: - $ref: '#/components/schemas/NodeState-Output' - type: 'null' description: The node's state object - required_resources: - anyOf: - - additionalProperties: true - type: object - - type: 'null' - title: Required Resources bootOptions: anyOf: - type: object diff --git a/services/web/server/tests/data/fake-project.json b/services/web/server/tests/data/fake-project.json index 24f73b87d1f8..62171dadd6ba 100644 --- a/services/web/server/tests/data/fake-project.json +++ b/services/web/server/tests/data/fake-project.json @@ -77,7 +77,7 @@ "x": 1073, "y": 307 }, - "progress": 100, + "progress": 100.0, "state": { "currentStatus": "NOT_STARTED", "lock_state": { diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index 2b457d3579a4..3fa5482fddfc 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -15,6 +15,7 @@ import sqlalchemy as sa from aiohttp.test_utils import TestClient from aioresponses import aioresponses +from deepdiff import DeepDiff from faker import Faker from models_library.api_schemas_directorv2.dynamic_services import ( GetProjectInactivityResponse, @@ -23,7 +24,10 @@ from models_library.products import ProductName from pydantic import TypeAdapter from pytest_mock import MockerFixture -from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.assert_checks import ( + assert_equal_ignoring_none, + assert_status, +) from pytest_simcore.helpers.webserver_parametrizations import ( ExpectedResponse, MockedStorageSubsystem, @@ -174,7 +178,9 @@ async def _assert_get_same_project( project_permalink = data.pop("permalink", None) folder_id = data.pop("folderId", None) - assert data == {k: project[k] for k in data} + assert not DeepDiff( + data, {k: project[k] for k in data}, exclude_paths="root['lastChangeDate']" + ) if project_state: assert ProjectStateOutputSchema.model_validate(project_state) @@ -215,7 +221,11 @@ async def test_list_projects( project_permalink = got.pop("permalink") folder_id = got.pop("folderId") - assert got == {k: template_project[k] for k in got} + assert not DeepDiff( + got, + {k: template_project[k] for k in got}, + exclude_paths="root['lastChangeDate']", + ) assert not ProjectStateOutputSchema( **project_state @@ -228,7 +238,11 @@ async def test_list_projects( project_permalink = got.pop("permalink", None) folder_id = got.pop("folderId") - assert got == {k: user_project[k] for k in got} + assert not DeepDiff( + got, + {k: user_project[k] for k in got}, + exclude_paths="root['lastChangeDate']", + ) assert ProjectStateOutputSchema(**project_state) assert project_permalink is None @@ -245,7 +259,12 @@ async def test_list_projects( project_permalink = got.pop("permalink", None) folder_id = got.pop("folderId") - assert got == {k: user_project[k] for k in got} + assert not DeepDiff( + got, + {k: user_project[k] for k in got}, + exclude_paths="root['lastChangeDate']", + ) + assert not ProjectStateOutputSchema( **project_state ).share_state.locked, "Single user does not lock" @@ -263,7 +282,11 @@ async def test_list_projects( project_permalink = got.pop("permalink") folder_id = got.pop("folderId") - assert got == {k: template_project[k] for k in got} + assert not DeepDiff( + got, + {k: template_project[k] for k in got}, + exclude_paths="root['lastChangeDate']", + ) assert not ProjectStateOutputSchema( **project_state ).share_state.locked, "Templates are not locked" @@ -632,7 +655,7 @@ async def test_new_template_from_project( ) assert len(templates) == 1 - assert templates[0] == template_project + assert_equal_ignoring_none(template_project, templates[0]) assert template_project["name"] == user_project["name"] assert template_project["description"] == user_project["description"] diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py index e1cfe13f8b2f..59958de8acf2 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handlers__patch.py @@ -12,6 +12,7 @@ import pytest from aiohttp.test_utils import TestClient +from deepdiff import DeepDiff from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_users import UserInfoDict @@ -168,7 +169,9 @@ async def test_patch_project_node( "output_1": { "store": 0, "path": "9934cba6-4b51-11ef-968a-02420a00f1c1/571ffc8d-fa6e-411f-afc8-9c62d08dd2fa/matus.txt", + "label": "matus.txt", "eTag": "d41d8cd98f00b204e9800998ecf8427e", + "dataset": None, } } } @@ -185,7 +188,6 @@ async def test_patch_project_node( _tested_node = data["workbench"][node_id] assert _tested_node["label"] == "testing-string" - assert _tested_node["progress"] is None assert _tested_node["key"] == _patch_key["key"] assert _tested_node["version"] == _patch_version["version"] assert _tested_node["inputs"] == _patch_inputs["inputs"] @@ -262,10 +264,14 @@ async def test_patch_project_node_inputs_notifies( await assert_status(resp, expected) assert mocked_notify_project_node_update.call_count > 1 # 1 message per node updated - assert [ - call_args[0][2] - for call_args in mocked_notify_project_node_update.await_args_list - ] == list(user_project["workbench"].keys()) + assert not DeepDiff( + [ + call_args[0][2] + for call_args in mocked_notify_project_node_update.await_args_list + ], + list(user_project["workbench"].keys()), + ignore_order=True, + ) @pytest.mark.parametrize( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py index e8a1536c5e48..bd2b14a4ad00 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py @@ -12,6 +12,7 @@ import pytest from aiohttp.test_utils import TestClient from aioresponses import aioresponses as AioResponsesMock # noqa: N812 +from deepdiff import DeepDiff from models_library.api_schemas_directorv2.computations import TasksOutputs from models_library.api_schemas_webserver.projects import ProjectGet from models_library.utils.fastapi_encoders import jsonable_encoder @@ -74,7 +75,7 @@ def mock_directorv2_service_api_responses( return aioresponses_mocker -@pytest.mark.acceptance_test() +@pytest.mark.acceptance_test @pytest.mark.parametrize( "user_role,expected", [ @@ -109,55 +110,62 @@ async def test_io_workflow( ports_meta, error = await assert_status(resp, expected_status_code=expected) if not error: - assert ports_meta == [ - { - "key": "38a0d401-af4b-4ea7-ab4c-5005c712a546", - "kind": "input", - "content_schema": { - "description": "Input integer value", - "title": "X", - "type": "integer", + diff = DeepDiff( + ports_meta, + [ + { + "key": "38a0d401-af4b-4ea7-ab4c-5005c712a546", + "kind": "input", + "content_schema": { + "description": "Input integer value", + "title": "X", + "type": "integer", + }, }, - }, - { - "key": "fc48252a-9dbb-4e07-bf9a-7af65a18f612", - "kind": "input", - "content_schema": { - "description": "Input integer value", - "title": "Z", - "type": "integer", + { + "key": "fc48252a-9dbb-4e07-bf9a-7af65a18f612", + "kind": "input", + "content_schema": { + "description": "Input integer value", + "title": "Z", + "type": "integer", + }, }, - }, - { - "key": "7bf0741f-bae4-410b-b662-fc34b47c27c9", - "kind": "input", - "content_schema": { - "description": "Input boolean value", - "title": "on", - "type": "boolean", + { + "key": "7bf0741f-bae4-410b-b662-fc34b47c27c9", + "kind": "input", + "content_schema": { + "description": "Input boolean value", + "title": "on", + "type": "boolean", + }, }, - }, - { - "key": "09fd512e-0768-44ca-81fa-0cecab74ec1a", - "kind": "output", - "content_schema": { - "description": "Output integer value", - "title": "Random sleep interval_2", - "type": "integer", + { + "key": "09fd512e-0768-44ca-81fa-0cecab74ec1a", + "kind": "output", + "content_schema": { + "description": "Output integer value", + "title": "Random sleep interval_2", + "type": "integer", + }, }, - }, - { - "key": "76f607b4-8761-4f96-824d-cab670bc45f5", - "kind": "output", - "content_schema": { - "description": "Output integer value", - "title": "Random sleep interval", - "type": "integer", + { + "key": "76f607b4-8761-4f96-824d-cab670bc45f5", + "kind": "output", + "content_schema": { + "description": "Output integer value", + "title": "Random sleep interval", + "type": "integer", + }, }, - }, - ] + ], + ignore_order=True, + ) - assert ports_meta == PROJECTS_METADATA_PORTS_RESPONSE_BODY_DATA + assert not diff + assert not DeepDiff( + ports_meta, PROJECTS_METADATA_PORTS_RESPONSE_BODY_DATA, ignore_order=True + ) # get_project_inputs expected_url = client.app.router["get_project_inputs"].url_for( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index 466cae4766b5..cfa61a8e5df4 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -22,6 +22,7 @@ import sqlalchemy as sa from aiohttp import ClientResponse from aiohttp.test_utils import TestClient, TestServer +from deepdiff import DeepDiff # type: ignore[attr-defined] from faker import Faker from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet from models_library.api_schemas_dynamic_scheduler.dynamic_services import ( @@ -95,10 +96,15 @@ def assert_replaced(current_project, update_data): def _extract(dikt, keys): return {k: dikt[k] for k in keys} - modified = [ + skip = [ "lastChangeDate", + "templateType", + "trashedAt", + "trashedBy", + "workspaceId", + "folderId", ] - keep = [k for k in update_data if k not in modified] + keep = [k for k in update_data if k not in skip] assert _extract(current_project, keep) == _extract(update_data, keep) @@ -1200,7 +1206,7 @@ async def test_get_active_project( ) assert not error assert ProjectStateOutputSchema(**data.pop("state")).share_state.locked - data.pop("folderId") + data.pop("folderId", None) user_project_last_change_date = user_project.pop("lastChangeDate") data_last_change_date = data.pop("lastChangeDate") @@ -2114,7 +2120,11 @@ async def test_open_shared_project_at_same_time( elif data: project_status = ProjectStateOutputSchema(**data.pop("state")) data.pop("folderId") - assert data == {k: shared_project[k] for k in data} + assert not DeepDiff( + data, + {k: shared_project[k] for k in data}, + exclude_paths=["root['lastChangeDate']"], + ) assert project_status.share_state.locked assert project_status.share_state.current_user_groupids assert len(project_status.share_state.current_user_groupids) == 1 diff --git a/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py b/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py index cb63e2625186..9c360f74ae72 100644 --- a/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py +++ b/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py @@ -232,8 +232,11 @@ async def mock_dynamic_service_rpc( """ Mocks the dynamic service RPC calls to avoid actual service calls during tests. """ - return mocker.patch( - "servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.services.retrieve_inputs", + import servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.services + + return mocker.patch.object( + servicelib.rabbitmq.rpc_interfaces.dynamic_scheduler.services, + "retrieve_inputs", autospec=True, ) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py index 9108c5b22be1..73f89dde70c8 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py @@ -78,7 +78,7 @@ async def test_workspaces_full_workflow_with_folders_and_projects( # noqa: PLR0 data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["uuid"] == project["uuid"] assert data["workspaceId"] == added_workspace.workspace_id - assert data["folderId"] is None + assert data.get("folderId") is None # Create folder in workspace url = client.app.router["create_folder"].url_for() diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py index 43d1484ad18b..23a1fe9fc801 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py @@ -93,7 +93,7 @@ async def test_moving_between_private_and_shared_workspaces( base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) - assert data["workspaceId"] is None # <-- Workspace ID is None + assert data.get("workspaceId") is None # <-- Workspace ID is None # Move project from your private workspace to shared workspace base_url = client.app.router["move_project_to_workspace"].url_for( @@ -252,7 +252,7 @@ async def test_moving_between_workspaces_check_removed_from_folder( base_url = client.app.router["get_project"].url_for(project_id=project["uuid"]) resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, status.HTTP_200_OK) - assert data["workspaceId"] is None # <-- Workspace ID is None + assert data.get("workspaceId") is None # <-- Workspace ID is None # Check project_to_folders DB is empty with postgres_db.connect() as con: