✨ Is1647/collaboration feature - 1. iteration (OPS ⚠️)

packages/models-library/src/models_library/api_schemas_webserver/projects.py

@@ -267,6 +267,25 @@ def to_domain_model(self) -> dict[str, Any]:
         return self.model_dump(exclude_unset=True, by_alias=False)
 
 
+class ProjectDocument(OutputSchema):
+    uuid: ProjectID
+    workspace_id: WorkspaceID | None
+    name: str
+    description: str
+    thumbnail: HttpUrl | None
+    last_change_date: datetime
+    classifiers: list[ClassifierID]
+    dev: dict | None
+    quality: dict[str, Any]
+    workbench: NodesDict
+    ui: StudyUI | None
+    type: ProjectType
+    template_type: ProjectTemplateType | None
+
+    # config
+    model_config = ConfigDict(from_attributes=True, arbitrary_types_allowed=True)
+
+
 __all__: tuple[str, ...] = (
     "EmptyModel",
     "ProjectCopyOverride",

packages/models-library/src/models_library/api_schemas_webserver/projects_ui.py

@@ -91,13 +91,14 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
 
 class StudyUI(OutputSchema):
     # Model fully controlled by the UI and stored under `projects.ui`
-    icon: HttpUrl | None = None
+    icon: HttpUrl | None = None  # <-- Deprecated
 
     workbench: dict[NodeIDStr, WorkbenchUI] | None = None
     slideshow: dict[NodeIDStr, SlideshowUI] | None = None
     current_node_id: NodeID | None = None
     annotations: dict[NodeIDStr, AnnotationUI] | None = None
-    template_type: Literal["hypertool"] | None = None
+    template_type: Literal["hypertool"] | None = None  # <-- Deprecated
+    mode: Literal["workbench", "app", "guided", "standalone", "pipeline"] | None = None
 
     _empty_is_none = field_validator("*", mode="before")(
         empty_str_to_none_pre_validator

packages/models-library/src/models_library/api_schemas_webserver/socketio.py

@@ -1,5 +1,6 @@
 from ..basic_types import IDStr
 from ..groups import GroupID
+from ..projects import ProjectID
 from ..users import UserID
 
 
@@ -15,3 +16,7 @@ def from_group_id(cls, group_id: GroupID) -> "SocketIORoomStr":
     @classmethod
     def from_user_id(cls, user_id: UserID) -> "SocketIORoomStr":
         return cls(f"user:{user_id}")
+
+    @classmethod
+    def from_project_id(cls, project_id: ProjectID) -> "SocketIORoomStr":
+        return cls(f"project:{project_id}")

packages/models-library/src/models_library/projects_nodes.py

@@ -164,7 +164,7 @@ class Node(BaseModel):
             ge=0,
             le=100,
             description="the node progress value (deprecated in DB, still used for API only)",
-            deprecated=True,
+            deprecated=True,  # <-- I think this is not true, it is still used by the File Picker (frontend node)
         ),
     ] = None
 

packages/service-library/src/servicelib/redis/__init__.py

@@ -8,6 +8,11 @@
     ProjectLockError,
 )
 from ._models import RedisManagerDBConfig
+from ._project_document_version import (
+    PROJECT_DB_UPDATE_REDIS_LOCK_KEY,
+    PROJECT_DOCUMENT_VERSION_KEY,
+    increment_and_return_project_document_version,
+)
 from ._project_lock import (
     get_project_locked_state,
     is_project_locked,
@@ -19,10 +24,13 @@
     "CouldNotAcquireLockError",
     "CouldNotConnectToRedisError",
     "exclusive",
+    "increment_and_return_project_document_version",
     "get_project_locked_state",
     "handle_redis_returns_union_types",
     "is_project_locked",
     "LockLostError",
+    "PROJECT_DB_UPDATE_REDIS_LOCK_KEY",
+    "PROJECT_DOCUMENT_VERSION_KEY",
     "ProjectLockError",
     "RedisClientSDK",
     "RedisClientsManager",

packages/service-library/src/servicelib/redis/_project_document_version.py

@@ -0,0 +1,39 @@
+"""Project document versioning utilities.
+
+This module provides utilities for managing project document versions using Redis.
+The versioning system ensures that all users working on a project are synchronized
+with the latest changes through atomic version incrementing.
+"""
+
+from typing import Final
+
+from models_library.projects import ProjectID
+
+from ._client import RedisClientSDK
+
+# Redis key patterns
+PROJECT_DOCUMENT_VERSION_KEY: Final[str] = "projects:{}:version"
+PROJECT_DB_UPDATE_REDIS_LOCK_KEY: Final[str] = "project_db_update:{}"
+
+
+async def increment_and_return_project_document_version(
+    redis_client: RedisClientSDK, project_uuid: ProjectID
+) -> int:
+    """
+    Atomically increments and returns the project document version using Redis.
+    Returns the incremented version number.
+
+    This function ensures thread-safe version incrementing by using Redis INCR command
+    which is atomic. The version starts at 1 for the first call.
+
+    Args:
+        redis_client: The Redis client SDK instance
+        project_uuid: The project UUID to get/increment version for
+
+    Returns:
+        The new (incremented) version number
+    """
+    version_key = PROJECT_DOCUMENT_VERSION_KEY.format(project_uuid)
+    # If key doesn't exist, it's created with value 0 and then incremented to 1
+    output = await redis_client.redis.incr(version_key)
+    return int(output)

packages/service-library/tests/redis/test_project_document_version.py

@@ -0,0 +1,50 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+
+from typing import cast
+from uuid import UUID
+
+import pytest
+from faker import Faker
+from models_library.projects import ProjectID
+from servicelib.redis import RedisClientSDK
+from servicelib.redis._project_document_version import (
+    increment_and_return_project_document_version,
+)
+
+pytest_simcore_core_services_selection = [
+    "redis",
+]
+pytest_simcore_ops_services_selection = [
+    "redis-commander",
+]
+
+
+@pytest.fixture()
+def project_uuid(faker: Faker) -> ProjectID:
+    return cast(UUID, faker.uuid4(cast_to=None))
+
+
+async def test_project_document_version_workflow(
+    redis_client_sdk: RedisClientSDK, project_uuid: ProjectID
+):
+    """Test the complete workflow of getting and incrementing project document versions."""
+
+    # First increment should return 1
+    new_version = await increment_and_return_project_document_version(
+        redis_client_sdk, project_uuid
+    )
+    assert new_version == 1
+
+    # Second increment should return 2
+    new_version = await increment_and_return_project_document_version(
+        redis_client_sdk, project_uuid
+    )
+    assert new_version == 2
+
+    # Multiple increments should work correctly
+    for expected_version in range(3, 6):
+        new_version = await increment_and_return_project_document_version(
+            redis_client_sdk, project_uuid
+        )
+        assert new_version == expected_version

packages/settings-library/src/settings_library/redis.py

@@ -19,6 +19,7 @@ class RedisDatabase(IntEnum):
     DEFERRED_TASKS = 7
     DYNAMIC_SERVICES = 8
     CELERY_TASKS = 9
+    DOCUMENTS = 10
 
 
 class RedisSettings(BaseCustomSettings):

services/docker-compose-ops.yml

@@ -94,8 +94,9 @@ services:
         announcements:${REDIS_HOST}:${REDIS_PORT}:5:${REDIS_PASSWORD},
         distributed_identifiers:${REDIS_HOST}:${REDIS_PORT}:6:${REDIS_PASSWORD},
         deferred_tasks:${REDIS_HOST}:${REDIS_PORT}:7:${REDIS_PASSWORD},
-        dynamic_services:${REDIS_HOST}:${REDIS_PORT}:8:${REDIS_PASSWORD}
-        celery_tasks:${REDIS_HOST}:${REDIS_PORT}:9:${REDIS_PASSWORD}
+        dynamic_services:${REDIS_HOST}:${REDIS_PORT}:8:${REDIS_PASSWORD},
+        celery_tasks:${REDIS_HOST}:${REDIS_PORT}:9:${REDIS_PASSWORD},
+        documents:${REDIS_HOST}:${REDIS_PORT}:10:${REDIS_PASSWORD}
       # If you add/remove a db, do not forget to update the --databases entry in the docker-compose.yml
     ports:
       - "18081:8081"

services/docker-compose.yml

@@ -1442,7 +1442,7 @@ services:
       # also aof (append only) is also enabled such that we get full durability at the expense
       # of backup size. The backup is written into /data.
       # https://redis.io/topics/persistence
-      [ "redis-server", "--save", "60 1", "--loglevel", "verbose", "--databases", "10", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD}" ]
+      [ "redis-server", "--save", "60 1", "--loglevel", "verbose", "--databases", "11", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD}" ]
     networks:
       - default
       - autoscaling_subnet

services/web/server/src/simcore_service_webserver/folders/_workspaces_repository.py

@@ -10,8 +10,8 @@
 from ..db.plugin import get_asyncpg_engine
 from ..projects import _folders_repository as projects_folders_repository
 from ..projects import _groups_repository as projects_groups_repository
-from ..projects import _projects_repository as _projects_repository
 from ..projects._access_rights_service import check_user_project_permission
+from ..projects.api import patch_project_and_notify_users
 from ..users import users_service
 from ..workspaces.api import check_user_workspace_access
 from . import _folders_repository
@@ -75,14 +75,15 @@ async def move_folder_into_workspace(
 
     # ⬆️ Here we have already guaranties that user has all the right permissions to do this operation ⬆️
 
+    user: dict = await users_service.get_user(app, user_id)
     async with transaction_context(get_asyncpg_engine(app)) as conn:
         # 4. Update workspace ID on the project resource
         for project_id in project_ids:
-            await _projects_repository.patch_project(
+            await patch_project_and_notify_users(
                 app=app,
-                connection=conn,
                 project_uuid=project_id,
-                new_partial_project_data={"workspace_id": workspace_id},
+                patch_project_data={"workspace_id": workspace_id},
+                user_primary_gid=user["primary_gid"],
             )
 
         # 5. BATCH update of folders with workspace_id
@@ -122,7 +123,6 @@ async def move_folder_into_workspace(
         )
 
         # 9. Remove all project permissions, leave only the user who moved the project
-        user = await users_service.get_user(app, user_id=user_id)
         for project_id in project_ids:
             await projects_groups_repository.delete_all_project_groups(
                 app, connection=conn, project_id=project_id

services/web/server/src/simcore_service_webserver/projects/_controller/projects_rest.py

@@ -314,7 +314,7 @@ async def patch_project(request: web.Request):
     path_params = parse_request_path_parameters_as(ProjectPathParams, request)
     project_patch = await parse_request_body_as(ProjectPatch, request)
 
-    await _projects_service.patch_project(
+    await _projects_service.patch_project_for_user(
         request.app,
         user_id=req_ctx.user_id,
         project_uuid=path_params.project_id,

services/web/server/src/simcore_service_webserver/projects/_controller/projects_states_rest.py

@@ -4,6 +4,7 @@
 
 from aiohttp import web
 from models_library.api_schemas_webserver.projects import ProjectGet
+from models_library.api_schemas_webserver.socketio import SocketIORoomStr
 from models_library.projects_state import ProjectState
 from pydantic import BaseModel
 from servicelib.aiohttp import status
@@ -18,6 +19,8 @@
 )
 from simcore_postgres_database.models.users import UserRole
 from simcore_postgres_database.webserver_models import ProjectType
+from simcore_service_webserver.resource_manager.user_sessions import managed_resource
+from simcore_service_webserver.socketio.server import get_socket_server
 
 from ..._meta import API_VTAG as VTAG
 from ...director_v2.exceptions import DirectorV2ServiceError
@@ -103,6 +106,20 @@ async def open_project(request: web.Request) -> web.Response:
         ):
             raise HTTPLockedError(text="Project is locked, try later")
 
+        # Connect the socket_id to a project room
+        with managed_resource(
+            req_ctx.user_id, client_session_id, request.app
+        ) as resource_registry:
+            _socket_id = await resource_registry.get_socket_id()
+        if _socket_id is None:
+            raise web.HTTPBadRequest(
+                text="Cannot open project without a socket_id, please refresh the page"
+            )
+        sio = get_socket_server(request.app)
+        sio.enter_room(
+            _socket_id, SocketIORoomStr.from_project_id(path_params.project_id)
+        )
+
         # the project can be opened, let's update its product links
         await _projects_service.update_project_linked_product(
             request.app, path_params.project_id, req_ctx.product_name

services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py

@@ -41,7 +41,7 @@
 from ..users import users_service
 from ..workspaces.api import check_user_workspace_access, get_user_workspace
 from ..workspaces.errors import WorkspaceAccessForbiddenError
-from . import _folders_repository, _projects_service
+from . import _folders_repository, _projects_repository, _projects_service
 from ._metadata_service import set_project_ancestors
 from ._permalink_service import update_or_pop_permalink_in_project
 from ._projects_repository_legacy import ProjectDBAPI
@@ -162,10 +162,10 @@ async def _copy_files_from_source_project(
     product_name: str,
     task_progress: TaskProgress,
 ):
-    _projects_repository = ProjectDBAPI.get_from_app_context(app)
+    _projects_repository_legacy = ProjectDBAPI.get_from_app_context(app)
 
     needs_lock_source_project: bool = (
-        await _projects_repository.get_project_type(
+        await _projects_repository_legacy.get_project_type(
             TypeAdapter(ProjectID).validate_python(source_project["uuid"])
         )
         != ProjectTypeDB.TEMPLATE
@@ -294,7 +294,7 @@ async def create_project(  # pylint: disable=too-many-arguments,too-many-branche
         f"{from_study=}",
     )
 
-    _projects_repository = ProjectDBAPI.get_from_app_context(request.app)
+    _projects_repository_legacy = ProjectDBAPI.get_from_app_context(request.app)
 
     new_project: ProjectDict = {}
     copy_file_coro = None
@@ -372,7 +372,7 @@ async def create_project(  # pylint: disable=too-many-arguments,too-many-branche
             )
 
         # 3.1 save new project in DB
-        new_project = await _projects_repository.insert_project(
+        new_project = await _projects_repository_legacy.insert_project(
             project=jsonable_encoder(new_project),
             user_id=user_id,
             product_name=product_name,
@@ -408,8 +408,10 @@ async def create_project(  # pylint: disable=too-many-arguments,too-many-branche
 
         # 5. unhide the project if needed since it is now complete
         if not new_project_was_hidden_before_data_was_copied:
-            await _projects_repository.set_hidden_flag(
-                new_project["uuid"], hidden=False
+            await _projects_repository.patch_project(
+                request.app,
+                project_uuid=new_project["uuid"],
+                new_partial_project_data={"hidden": False},
             )
 
         # update the network information in director-v2
@@ -427,7 +429,7 @@ async def create_project(  # pylint: disable=too-many-arguments,too-many-branche
             product_api_base_url,
         )
         # get the latest state of the project (lastChangeDate for instance)
-        new_project, _ = await _projects_repository.get_project_dict_and_type(
+        new_project, _ = await _projects_repository_legacy.get_project_dict_and_type(
             project_uuid=new_project["uuid"]
         )
         # Appends state
@@ -444,7 +446,7 @@ async def create_project(  # pylint: disable=too-many-arguments,too-many-branche
 
         # Adds folderId
         user_specific_project_data_db = (
-            await _projects_repository.get_user_specific_project_data_db(
+            await _projects_repository_legacy.get_user_specific_project_data_db(
                 project_uuid=new_project["uuid"],
                 private_workspace_user_id_or_none=(
                     user_id if workspace_id is None else None
@@ -467,7 +469,7 @@ async def create_project(  # pylint: disable=too-many-arguments,too-many-branche
                 for gid, access in workspace.access_rights.items()
             }
 
-        _project_product_name = await _projects_repository.get_project_product(
+        _project_product_name = await _projects_repository_legacy.get_project_product(
             project_uuid=new_project["uuid"]
         )
         assert (

services/web/server/src/simcore_service_webserver/projects/_crud_api_delete.py

@@ -17,6 +17,7 @@
 from ..storage.api import delete_data_folders_of_project
 from ..users.exceptions import UserNotFoundError
 from ..users.users_service import FullNameDict
+from . import _projects_repository
 from ._access_rights_service import check_user_project_permission
 from ._projects_repository_legacy import ProjectDBAPI
 from .exceptions import (
@@ -70,7 +71,11 @@ async def mark_project_as_deleted(
     # NOTE: if any of the steps below fail, it might results in a
     # services/projects/data that might be incosistent. The GC should
     # be able to detect that and resolve it.
-    await db.set_hidden_flag(project_uuid, hidden=True)
+    await _projects_repository.patch_project(
+        app,
+        project_uuid=project_uuid,
+        new_partial_project_data={"hidden": True},
+    )
 
 
 async def delete_project(