Merge branch 'master' into 1922-ensure-no-hang-in-api-server

Author: bisgaard-itis

services/api-server/src/simcore_service_api_server/api/routes/studies.py

@@ -111,8 +111,9 @@ async def clone_study(
             description=description,
         )
         await webserver_api.patch_project(
-            project_id=study_id, patch_params=patch_params
+            project_id=project.uuid, patch_params=patch_params
         )
+        project = await webserver_api.get_project(project_id=project.uuid)
     return _create_study_from_project(project)
 
 

services/api-server/tests/unit/api_studies/test_api_routes_studies.py

@@ -1,6 +1,8 @@
 # pylint: disable=redefined-outer-name
 # pylint: disable=unused-argument
 # pylint: disable=unused-variable
+# pylint: disable=too-many-arguments
+# pylint: disable=too-many-statements
 
 
 import json
@@ -13,6 +15,8 @@
 import pytest
 from faker import Faker
 from fastapi import status
+from models_library.api_schemas_webserver.projects import ProjectGet
+from models_library.generics import Envelope
 from pydantic import TypeAdapter
 from pytest_mock import MockType
 from pytest_simcore.helpers.httpx_calls_capture_models import HttpApiCallCaptureModel
@@ -224,12 +228,8 @@ async def test_clone_study_with_title(
     study_id: StudyID,
     mocked_webserver_rest_api_base: MockRouter,
     patch_webserver_long_running_project_tasks: Callable[[MockRouter], MockRouter],
-    mock_webserver_patch_project: Callable[
-        [
-            MockRouter,
-        ],
-        MockRouter,
-    ],
+    mock_webserver_patch_project: Callable[[MockRouter], MockRouter],
+    mock_webserver_get_project: Callable[[MockRouter], MockRouter],
     hidden: bool | None,
     title: str | None,
     description: str | None,
@@ -238,11 +238,14 @@ async def test_clone_study_with_title(
     # Mocks /projects
     patch_webserver_long_running_project_tasks(mocked_webserver_rest_api_base)
     mock_webserver_patch_project(mocked_webserver_rest_api_base)
+    mock_webserver_get_project(mocked_webserver_rest_api_base)
 
     create_callback = mocked_webserver_rest_api_base["create_projects"].side_effect
     assert create_callback is not None
     patch_callback = mocked_webserver_rest_api_base["project_patch"].side_effect
     assert patch_callback is not None
+    get_callback = mocked_webserver_rest_api_base["project_get"].side_effect
+    assert get_callback is not None
 
     def clone_project_side_effect(request: httpx.Request):
         if hidden is not None:
@@ -260,12 +263,28 @@ def patch_project_side_effect(request: httpx.Request, *args, **kwargs):
             assert _description is not None and _description in description
         return patch_callback(request, *args, **kwargs)
 
+    def get_project_side_effect(request: httpx.Request, *args, **kwargs):
+        # this is needed to return the patched project
+        _project_id = kwargs.get("project_id")
+        assert _project_id is not None
+        result = Envelope[ProjectGet].model_validate(
+            {"data": ProjectGet.model_json_schema()["examples"][0]}
+        )
+        assert result.data is not None
+        if title is not None:
+            result.data.name = title
+        if description is not None:
+            result.data.description = description
+        result.data.uuid = UUID(_project_id)
+        return httpx.Response(status.HTTP_200_OK, content=result.model_dump_json())
+
     mocked_webserver_rest_api_base["create_projects"].side_effect = (
         clone_project_side_effect
     )
     mocked_webserver_rest_api_base["project_patch"].side_effect = (
         patch_project_side_effect
     )
+    mocked_webserver_rest_api_base["project_get"].side_effect = get_project_side_effect
 
     query = dict()
     if hidden is not None:
@@ -286,8 +305,14 @@ def patch_project_side_effect(request: httpx.Request, *args, **kwargs):
     assert mocked_webserver_rest_api_base["create_projects"].called
     if title or description:
         assert mocked_webserver_rest_api_base["project_patch"].called
+        assert mocked_webserver_rest_api_base["project_get"].called
 
     assert resp.status_code == expected_status_code
+    study = Study.model_validate(resp.json())
+    if title is not None:
+        assert study.title == title
+    if description is not None:
+        assert study.description == description
 
 
 async def test_clone_study_not_found(

services/api-server/tests/unit/conftest.py

@@ -796,7 +796,7 @@ def _mock(webserver_mock_router: MockRouter) -> MockRouter:
 
 @pytest.fixture
 def mock_webserver_patch_project(
-    app: FastAPI, faker: Faker, services_mocks_enabled: bool
+    app: FastAPI, services_mocks_enabled: bool
 ) -> Callable[[MockRouter], MockRouter]:
     settings: ApplicationSettings = app.state.settings
     assert settings.API_SERVER_WEBSERVER is not None
@@ -815,6 +815,30 @@ def _patch_project(request: httpx.Request, *args, **kwargs):
     return _mock
 
 
+@pytest.fixture
+def mock_webserver_get_project(
+    app: FastAPI, services_mocks_enabled: bool
+) -> Callable[[MockRouter], MockRouter]:
+    settings: ApplicationSettings = app.state.settings
+    assert settings.API_SERVER_WEBSERVER is not None
+
+    def _mock(webserver_mock_router: MockRouter) -> MockRouter:
+        def _get_project(request: httpx.Request, *args, **kwargs):
+            result = Envelope[ProjectGet].model_validate(
+                {"data": ProjectGet.model_json_schema()["examples"][0]}
+            )
+            return httpx.Response(status.HTTP_200_OK, json=result.model_dump())
+
+        if services_mocks_enabled:
+            webserver_mock_router.get(
+                path__regex=r"/projects/(?P<project_id>[\w-]+)$",
+                name="project_get",
+            ).mock(side_effect=_get_project)
+        return webserver_mock_router
+
+    return _mock
+
+
 @pytest.fixture
 def openapi_dev_specs(project_slug_dir: Path) -> dict[str, Any]:
     openapi_file = (project_slug_dir / "openapi-dev.json").resolve()

services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py

@@ -590,9 +590,7 @@ async def _process_executing_tasks(
         """process executing tasks from the 3rd party backend"""
 
     @abstractmethod
-    async def _release_resources(
-        self, user_id: UserID, project_id: ProjectID, comp_run: CompRunsAtDB
-    ) -> None:
+    async def _release_resources(self, comp_run: CompRunsAtDB) -> None:
         """release resources used by the scheduler for a given user and project"""
 
     async def apply(
@@ -660,7 +658,7 @@ async def apply(
 
                 # 7. Are we done scheduling that pipeline?
                 if not dag.nodes() or pipeline_result in COMPLETED_STATES:
-                    await self._release_resources(user_id, project_id, comp_run)
+                    await self._release_resources(comp_run)
                     # there is nothing left, the run is completed, we're done here
                     _logger.info(
                         "pipeline %s scheduling completed with result %s",

services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_dask.py

@@ -56,7 +56,7 @@
 
 _logger = logging.getLogger(__name__)
 
-_DASK_CLIENT_RUN_REF: Final[str] = "{user_id}:{run_id}"
+_DASK_CLIENT_RUN_REF: Final[str] = "{user_id}:{project_id}:{run_id}"
 
 
 @asynccontextmanager
@@ -65,6 +65,7 @@ async def _cluster_dask_client(
     scheduler: "DaskScheduler",
     *,
     use_on_demand_clusters: bool,
+    project_id: ProjectID,
     run_id: PositiveInt,
     run_metadata: RunMetadataDict,
 ) -> AsyncIterator[DaskClient]:
@@ -76,7 +77,10 @@ async def _cluster_dask_client(
             wallet_id=run_metadata.get("wallet_id"),
         )
     async with scheduler.dask_clients_pool.acquire(
-        cluster, ref=_DASK_CLIENT_RUN_REF.format(user_id=user_id, run_id=run_id)
+        cluster,
+        ref=_DASK_CLIENT_RUN_REF.format(
+            user_id=user_id, project_id=project_id, run_id=run_id
+        ),
     ) as client:
         yield client
 
@@ -106,6 +110,7 @@ async def _start_tasks(
             user_id,
             self,
             use_on_demand_clusters=comp_run.use_on_demand_clusters,
+            project_id=comp_run.project_uuid,
             run_id=comp_run.run_id,
             run_metadata=comp_run.metadata,
         ) as client:
@@ -157,6 +162,7 @@ async def _get_tasks_status(
                 user_id,
                 self,
                 use_on_demand_clusters=comp_run.use_on_demand_clusters,
+                project_id=comp_run.project_uuid,
                 run_id=comp_run.run_id,
                 run_metadata=comp_run.metadata,
             ) as client:
@@ -178,6 +184,7 @@ async def _process_executing_tasks(
                 user_id,
                 self,
                 use_on_demand_clusters=comp_run.use_on_demand_clusters,
+                project_id=comp_run.project_uuid,
                 run_id=comp_run.run_id,
                 run_metadata=comp_run.metadata,
             ) as client:
@@ -225,20 +232,22 @@ async def _process_executing_tasks(
             )
         )
 
-    async def _release_resources(
-        self, user_id: UserID, project_id: ProjectID, comp_run: CompRunsAtDB
-    ) -> None:
+    async def _release_resources(self, comp_run: CompRunsAtDB) -> None:
         """release resources used by the scheduler for a given user and project"""
         with (
             log_catch(_logger, reraise=False),
             log_context(
                 _logger,
                 logging.INFO,
-                msg=f"releasing resources for {user_id=}, {project_id=}, {comp_run.run_id=}",
+                msg=f"releasing resources for {comp_run.user_id=}, {comp_run.project_uuid=}, {comp_run.run_id=}",
             ),
         ):
             await self.dask_clients_pool.release_client_ref(
-                ref=_DASK_CLIENT_RUN_REF.format(user_id=user_id, run_id=comp_run.run_id)
+                ref=_DASK_CLIENT_RUN_REF.format(
+                    user_id=comp_run.user_id,
+                    project_id=comp_run.project_uuid,
+                    run_id=comp_run.run_id,
+                )
             )
 
     async def _stop_tasks(
@@ -250,6 +259,7 @@ async def _stop_tasks(
                 user_id,
                 self,
                 use_on_demand_clusters=comp_run.use_on_demand_clusters,
+                project_id=comp_run.project_uuid,
                 run_id=comp_run.run_id,
                 run_metadata=comp_run.metadata,
             ) as client:
@@ -284,6 +294,7 @@ async def _process_completed_tasks(
                 user_id,
                 self,
                 use_on_demand_clusters=comp_run.use_on_demand_clusters,
+                project_id=comp_run.project_uuid,
                 run_id=comp_run.run_id,
                 run_metadata=comp_run.metadata,
             ) as client:
@@ -304,6 +315,7 @@ async def _process_completed_tasks(
                 user_id,
                 self,
                 use_on_demand_clusters=comp_run.use_on_demand_clusters,
+                project_id=comp_run.project_uuid,
                 run_id=comp_run.run_id,
                 run_metadata=comp_run.metadata,
             ) as client: