Merge branch 'master' into retire-clusters

Author: odeimaiz

.env-devel

@@ -90,8 +90,8 @@ EFS_GROUP_NAME=efs-group
 EFS_DNS_NAME=fs-xxx.efs.us-east-1.amazonaws.com
 EFS_MOUNTED_PATH=/tmp/efs
 EFS_PROJECT_SPECIFIC_DATA_DIRECTORY=project-specific-data
-EFS_ONLY_ENABLED_FOR_USERIDS=[]
 EFS_GUARDIAN_TRACING={}
+EFS_DEFAULT_USER_SERVICE_SIZE_BYTES=10000
 
 # DATCORE_ADAPTER
 DATCORE_ADAPTER_TRACING={}

.github/workflows/ci-testing-deploy.yml

@@ -2460,7 +2460,7 @@ jobs:
 
   integration-tests:
     # NOTE: this is a github required status check!
-    if: ${{ !cancelled() }}
+    if: ${{ always() }}
     needs:
       [
         integration-test-director-v2-01,
@@ -2792,7 +2792,7 @@ jobs:
 
   system-tests:
     # NOTE: this is a github required status check!
-    if: ${{ !cancelled() }}
+    if: ${{ always() }}
     needs:
       [
         system-test-e2e,

api/specs/web-server/_folders.py

@@ -25,6 +25,9 @@
     FoldersListQueryParams,
     FoldersPathParams,
 )
+from simcore_service_webserver.folders._workspaces_handlers import (
+    _FolderWorkspacesPathParams,
+)
 
 router = APIRouter(
     prefix=f"/{API_VTAG}",
@@ -97,3 +100,15 @@ async def delete_folder(
     _path: Annotated[FoldersPathParams, Depends()],
 ):
     ...
+
+
+@router.post(
+    "/folders/{folder_id}/workspaces/{workspace_id}:move",
+    status_code=status.HTTP_204_NO_CONTENT,
+    summary="Move folder to the workspace",
+    tags=["workspaces"],
+)
+async def move_folder_to_workspace(
+    _path: Annotated[_FolderWorkspacesPathParams, Depends()],
+):
+    ...

api/specs/web-server/_projects_workspaces.py

@@ -23,12 +23,12 @@
 )
 
 
-@router.put(
-    "/projects/{project_id}/workspaces/{workspace_id}",
+@router.post(
+    "/projects/{project_id}/workspaces/{workspace_id}:move",
     status_code=status.HTTP_204_NO_CONTENT,
     summary="Move project to the workspace",
 )
-async def replace_project_workspace(
+async def move_project_to_workspace(
     _path: Annotated[_ProjectWorkspacesPathParams, Depends()],
 ):
     ...

packages/models-library/src/models_library/api_schemas_webserver/projects.py

@@ -9,8 +9,16 @@
 from typing import Annotated, Any, Literal, TypeAlias
 
 from models_library.folders import FolderID
+from models_library.utils._original_fastapi_encoders import jsonable_encoder
 from models_library.workspaces import WorkspaceID
-from pydantic import BeforeValidator, ConfigDict, Field, HttpUrl, field_validator
+from pydantic import (
+    BeforeValidator,
+    ConfigDict,
+    Field,
+    HttpUrl,
+    PlainSerializer,
+    field_validator,
+)
 
 from ..api_schemas_long_running_tasks.tasks import TaskGet
 from ..basic_types import LongTruncatedStr, ShortTruncatedStr
@@ -130,12 +138,22 @@ class ProjectPatch(InputSchema):
     name: ShortTruncatedStr | None = Field(default=None)
     description: LongTruncatedStr | None = Field(default=None)
     thumbnail: Annotated[
-        HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator)
+        HttpUrl | None,
+        BeforeValidator(empty_str_to_none_pre_validator),
+        PlainSerializer(lambda x: str(x) if x is not None else None),
     ] = Field(default=None)
     access_rights: dict[GroupIDStr, AccessRights] | None = Field(default=None)
     classifiers: list[ClassifierID] | None = Field(default=None)
     dev: dict | None = Field(default=None)
-    ui: StudyUI | None = Field(default=None)
+    ui: Annotated[
+        StudyUI | None,
+        BeforeValidator(empty_str_to_none_pre_validator),
+        PlainSerializer(
+            lambda obj: jsonable_encoder(
+                obj, exclude_unset=True, by_alias=False
+            )  # For the sake of backward compatibility
+        ),
+    ] = Field(default=None)
     quality: dict[str, Any] | None = Field(default=None)
 
 

packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py

@@ -95,7 +95,7 @@ async def create_project(
         for group_id, permissions in _access_rights.items():
             await update_or_insert_project_group(
                 app,
-                new_project["uuid"],
+                project_id=new_project["uuid"],
                 group_id=int(group_id),
                 read=permissions["read"],
                 write=permissions["write"],

packages/settings-library/src/settings_library/efs.py

@@ -14,10 +14,6 @@ class AwsEfsSettings(BaseCustomSettings):
     EFS_MOUNTED_PATH: Path = Field(
         description="This is the path where EFS is mounted to the EC2 machine",
     )
-    EFS_ONLY_ENABLED_FOR_USERIDS: list[int] = Field(
-        description="This is temporary solution so we can enable it for specific users for testing purpose",
-        examples=[[1]],
-    )
 
 
 NFS_PROTOCOL = "4.1"

services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py

@@ -381,15 +381,6 @@ def LOG_LEVEL(self) -> LogLevel:  # noqa: N802
     def _valid_log_level(cls, value: str) -> str:
         return cls.validate_log_level(value)
 
-    @field_validator("SERVICE_TRACKING_HEARTBEAT", mode="before")
-    @classmethod
-    def _validate_interval(
-        cls, value: str | datetime.timedelta
-    ) -> int | datetime.timedelta:
-        if isinstance(value, str):
-            return int(value)
-        return value
-
 
 def get_application_settings(app: FastAPI) -> ApplicationSettings:
     return cast(ApplicationSettings, app.state.settings)

services/clusters-keeper/tests/unit/test_modules_clusters_management_core.py

@@ -4,6 +4,7 @@
 
 import asyncio
 import dataclasses
+import datetime
 from collections.abc import Awaitable, Callable
 from typing import Final
 from unittest.mock import MagicMock
@@ -36,7 +37,9 @@ def wallet_id(faker: Faker, request: pytest.FixtureRequest) -> WalletID | None:
     return faker.pyint(min_value=1) if request.param == "with_wallet" else None
 
 
-_FAST_TIME_BEFORE_TERMINATION_SECONDS: Final[int] = 10
+_FAST_TIME_BEFORE_TERMINATION_SECONDS: Final[datetime.timedelta] = datetime.timedelta(
+    seconds=10
+)
 
 
 @pytest.fixture
@@ -149,7 +152,7 @@ async def test_cluster_management_core_properly_removes_unused_instances(
     mocked_dask_ping_scheduler.is_scheduler_busy.reset_mock()
 
     # running the cluster management task after the heartbeat came in shall not remove anything
-    await asyncio.sleep(_FAST_TIME_BEFORE_TERMINATION_SECONDS + 1)
+    await asyncio.sleep(_FAST_TIME_BEFORE_TERMINATION_SECONDS.total_seconds() + 1)
     await cluster_heartbeat(initialized_app, user_id=user_id, wallet_id=wallet_id)
     await check_clusters(initialized_app)
     await _assert_cluster_exist_and_state(
@@ -161,7 +164,7 @@ async def test_cluster_management_core_properly_removes_unused_instances(
     mocked_dask_ping_scheduler.is_scheduler_busy.reset_mock()
 
     # after waiting the termination time, running the task shall remove the cluster
-    await asyncio.sleep(_FAST_TIME_BEFORE_TERMINATION_SECONDS + 1)
+    await asyncio.sleep(_FAST_TIME_BEFORE_TERMINATION_SECONDS.total_seconds() + 1)
     await check_clusters(initialized_app)
     await _assert_cluster_exist_and_state(
         ec2_client, instances=created_clusters, state="terminated"
@@ -201,7 +204,7 @@ async def test_cluster_management_core_properly_removes_workers_on_shutdown(
         ec2_client, instance_ids=worker_instance_ids, state="running"
     )
     # after waiting the termination time, running the task shall remove the cluster
-    await asyncio.sleep(_FAST_TIME_BEFORE_TERMINATION_SECONDS + 1)
+    await asyncio.sleep(_FAST_TIME_BEFORE_TERMINATION_SECONDS.total_seconds() + 1)
     await check_clusters(initialized_app)
     await _assert_cluster_exist_and_state(
         ec2_client, instances=created_clusters, state="terminated"
@@ -314,7 +317,7 @@ async def test_cluster_management_core_removes_broken_clusters_after_some_delay(
     mocked_dask_ping_scheduler.is_scheduler_busy.reset_mock()
 
     # waiting for the termination time will now terminate the cluster
-    await asyncio.sleep(_FAST_TIME_BEFORE_TERMINATION_SECONDS + 1)
+    await asyncio.sleep(_FAST_TIME_BEFORE_TERMINATION_SECONDS.total_seconds() + 1)
     await check_clusters(initialized_app)
     await _assert_cluster_exist_and_state(
         ec2_client, instances=created_clusters, state="terminated"

services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/_scheduler_base.py

@@ -540,11 +540,7 @@ async def apply(
         project_id: ProjectID,
         iteration: Iteration,
     ) -> None:
-        """schedules a pipeline for a given user, project and iteration.
-
-        Arguments:
-            wake_up_callback -- a callback function that is called in a separate thread everytime a pipeline node is completed
-        """
+        """apply the scheduling of a pipeline for a given user, project and iteration."""
         with log_context(
             _logger,
             level=logging.INFO,