Is7420/permanently delete trashed workspaces (#7482)

Co-authored-by: Giancarlo Romeo <[email protected]>
Co-authored-by: Pedro Crespo-Valero <[email protected]>

Author: 3 people

services/web/server/src/simcore_service_webserver/director_v2/_service.py

@@ -26,7 +26,7 @@
 from ..db.plugin import get_database_engine
 from ..products import products_service
 from ..products.models import Product
-from ..projects import api as projects_service
+from ..projects import projects_wallets_service
 from ..users import preferences_api as user_preferences_service
 from ..users.exceptions import UserDefaultWalletNotFoundError
 from ..wallets import api as wallets_service
@@ -204,7 +204,7 @@ async def get_wallet_info(
         product.is_payment_enabled and app_settings.WEBSERVER_CREDIT_COMPUTATION_ENABLED
     ):
         return None
-    project_wallet = await projects_service.get_project_wallet(
+    project_wallet = await projects_wallets_service.get_project_wallet(
         app, project_id=project_id
     )
     if project_wallet is None:
@@ -219,7 +219,7 @@ async def get_wallet_info(
         project_wallet_id = TypeAdapter(WalletID).validate_python(
             user_default_wallet_preference.value
         )
-        await projects_service.connect_wallet_to_project(
+        await projects_wallets_service.connect_wallet_to_project(
             app,
             product_name=product_name,
             project_id=project_id,

services/web/server/src/simcore_service_webserver/folders/_folders_repository.py

@@ -245,7 +245,9 @@ async def list_(  # pylint: disable=too-many-arguments,too-many-branches
 
     # Ordering and pagination
     list_query = (
-        combined_query.order_by(_to_expression(order_by)).offset(offset).limit(limit)
+        combined_query.order_by(_to_expression(order_by), folders_v2.c.folder_id)
+        .offset(offset)
+        .limit(limit)
     )
 
     async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn:
@@ -258,13 +260,14 @@ async def list_(  # pylint: disable=too-many-arguments,too-many-branches
         return cast(int, total_count), folders
 
 
-async def list_trashed_folders(
+async def list_folders_db_as_admin(
     app: web.Application,
     connection: AsyncConnection | None = None,
     *,
     # filter
     trashed_explicitly: bool | UnSet = UnSet.VALUE,
     trashed_before: datetime | UnSet = UnSet.VALUE,
+    shared_workspace_id: WorkspaceID | UnSet = UnSet.VALUE,  # <-- Workspace filter
     # pagination
     offset: NonNegativeInt,
     limit: int,
@@ -273,28 +276,35 @@ async def list_trashed_folders(
 ) -> tuple[int, list[FolderDB]]:
     """
     NOTE: this is app-wide i.e. no product, user or workspace filtered
-    TODO: check with MD about workspaces
     """
-    base_query = sql.select(*_FOLDER_DB_MODEL_COLS).where(
-        folders_v2.c.trashed.is_not(None)
-    )
+    base_query = sql.select(*_FOLDER_DB_MODEL_COLS)
 
     if is_set(trashed_explicitly):
         assert isinstance(trashed_explicitly, bool)  # nosec
         base_query = base_query.where(
-            folders_v2.c.trashed_explicitly.is_(trashed_explicitly)
+            (folders_v2.c.trashed_explicitly.is_(trashed_explicitly))
+            & (folders_v2.c.trashed.is_not(None))
         )
 
     if is_set(trashed_before):
         assert isinstance(trashed_before, datetime)  # nosec
-        base_query = base_query.where(folders_v2.c.trashed < trashed_before)
+        base_query = base_query.where(
+            (folders_v2.c.trashed < trashed_before)
+            & (folders_v2.c.trashed.is_not(None))
+        )
+
+    if is_set(shared_workspace_id):
+        assert isinstance(shared_workspace_id, int)  # nosec
+        base_query = base_query.where(folders_v2.c.workspace_id == shared_workspace_id)
 
     # Select total count from base_query
     count_query = sql.select(sql.func.count()).select_from(base_query.subquery())
 
     # Ordering and pagination
     list_query = (
-        base_query.order_by(_to_expression(order_by)).offset(offset).limit(limit)
+        base_query.order_by(_to_expression(order_by), folders_v2.c.folder_id)
+        .offset(offset)
+        .limit(limit)
     )
 
     async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn:
@@ -376,7 +386,7 @@ async def update(
     parent_folder_id: FolderID | None | UnSet = UnSet.VALUE,
     trashed: datetime | None | UnSet = UnSet.VALUE,
     trashed_explicitly: bool | UnSet = UnSet.VALUE,
-    trashed_by: UserID | UnSet = UnSet.VALUE,  # who trashed
+    trashed_by: UserID | None | UnSet = UnSet.VALUE,  # who trashed
     workspace_id: WorkspaceID | None | UnSet = UnSet.VALUE,
     user_id: UserID | None | UnSet = UnSet.VALUE,  # ownership
 ) -> FolderDB:
@@ -534,7 +544,7 @@ async def get_all_folders_and_projects_ids_recursively(
     product_name: ProductName,
 ) -> tuple[list[FolderID], list[ProjectID]]:
     """
-    The purpose of this function is to retrieve all projects within the provided folder ID.
+    The purpose of this function is to retrieve all subfolders and projects within the provided folder ID.
     """
 
     async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn:

services/web/server/src/simcore_service_webserver/folders/_folders_rest.py

@@ -205,11 +205,11 @@ async def replace_folder(request: web.Request):
 @login_required
 @permission_required("folder.delete")
 @handle_plugin_requests_exceptions
-async def delete_folder_group(request: web.Request):
+async def delete_folder(request: web.Request):
     req_ctx = FoldersRequestContext.model_validate(request)
     path_params = parse_request_path_parameters_as(FoldersPathParams, request)
 
-    await _folders_service.delete_folder(
+    await _folders_service.delete_folder_with_all_content(
         app=request.app,
         user_id=req_ctx.user_id,
         folder_id=path_params.folder_id,

services/web/server/src/simcore_service_webserver/folders/_folders_service.py

@@ -11,11 +11,8 @@
 from models_library.users import UserID
 from models_library.workspaces import WorkspaceID, WorkspaceQuery, WorkspaceScope
 from pydantic import NonNegativeInt
-from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY
-from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE
-from servicelib.utils import fire_and_forget_task
 
-from ..projects._projects_service import submit_delete_project_task
+from ..projects._projects_service import delete_project_by_user
 from ..users.api import get_user
 from ..workspaces.api import check_user_workspace_access
 from ..workspaces.errors import (
@@ -320,7 +317,7 @@ async def update_folder(
     )
 
 
-async def delete_folder(
+async def delete_folder_with_all_content(
     app: web.Application,
     user_id: UserID,
     folder_id: FolderID,
@@ -352,6 +349,9 @@ async def delete_folder(
 
     # 1. Delete folder content
     # 1.1 Delete all child projects that I am an owner
+    # NOTE: The reason for this is to be cautious and not delete projects by accident that
+    # are not owned by the user (even if the user was granted delete permissions). As a consequence, after deleting the folder,
+    # projects that the user does not own will appear in the root. (Maybe this can be changed as we now have a trash system).
     project_id_list: list[ProjectID] = (
         await _folders_repository.get_projects_recursively_only_if_user_is_owner(
             app,
@@ -362,17 +362,11 @@ async def delete_folder(
         )
     )
 
-    # fire and forget task for project deletion
     for project_id in project_id_list:
-        fire_and_forget_task(
-            submit_delete_project_task(
-                app,
-                project_uuid=project_id,
-                user_id=user_id,
-                simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE,
-            ),
-            task_suffix_name=f"delete_project_task_{project_id}",
-            fire_and_forget_tasks_collection=app[APP_FIRE_AND_FORGET_TASKS_KEY],
+        await delete_project_by_user(
+            app,
+            project_uuid=project_id,
+            user_id=user_id,
         )
 
     # 1.2 Delete all child folders

services/web/server/src/simcore_service_webserver/folders/_trash_rest.py

@@ -39,6 +39,7 @@ async def trash_folder(request: web.Request):
         user_id=user_id,
         folder_id=path_params.folder_id,
         force_stop_first=query_params.force,
+        explicit=True,
     )
 
     return web.json_response(status=status.HTTP_204_NO_CONTENT)

services/web/server/src/simcore_service_webserver/folders/_trash_service.py

@@ -12,6 +12,7 @@
 from models_library.rest_ordering import OrderBy, OrderDirection
 from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE
 from models_library.users import UserID
+from models_library.workspaces import WorkspaceID
 from simcore_postgres_database.utils_repos import transaction_context
 from sqlalchemy.ext.asyncio import AsyncConnection
 
@@ -62,23 +63,24 @@ async def _check_exists_and_access(
     return workspace_is_private
 
 
-async def _folders_db_update(
+async def _folders_db_trashed_state_update(
     app: web.Application,
     connection: AsyncConnection | None = None,
     *,
     product_name: ProductName,
     folder_id: FolderID,
     trashed_at: datetime | None,
-    trashed_by: UserID,
+    trashed_explicitly: bool,
+    trashed_by: UserID | None,
 ):
-    # EXPLICIT un/trash
+    # EXPLICIT or IMPLICIT un/trash
     await _folders_repository.update(
         app,
         connection,
         folders_id_or_ids=folder_id,
         product_name=product_name,
         trashed=trashed_at,
-        trashed_explicitly=trashed_at is not None,
+        trashed_explicitly=trashed_explicitly,
         trashed_by=trashed_by,
     )
 
@@ -110,6 +112,7 @@ async def trash_folder(
     user_id: UserID,
     folder_id: FolderID,
     force_stop_first: bool,
+    explicit: bool,
 ):
 
     workspace_is_private = await _check_exists_and_access(
@@ -122,25 +125,28 @@ async def trash_folder(
     async with transaction_context(get_asyncpg_engine(app)) as connection:
 
         # 1. Trash folder and children
-        await _folders_db_update(
+        await _folders_db_trashed_state_update(
             app,
             connection,
             folder_id=folder_id,
             product_name=product_name,
             trashed_at=trashed_at,
+            trashed_explicitly=explicit,
             trashed_by=user_id,
         )
 
         # 2. Trash all child projects that I am an owner
-        child_projects: list[
-            ProjectID
-        ] = await _folders_repository.get_projects_recursively_only_if_user_is_owner(
-            app,
-            connection,
-            folder_id=folder_id,
-            private_workspace_user_id_or_none=user_id if workspace_is_private else None,
-            user_id=user_id,
-            product_name=product_name,
+        child_projects: list[ProjectID] = (
+            await _folders_repository.get_projects_recursively_only_if_user_is_owner(
+                app,
+                connection,
+                folder_id=folder_id,
+                private_workspace_user_id_or_none=(
+                    user_id if workspace_is_private else None
+                ),
+                user_id=user_id,
+                product_name=product_name,
+            )
         )
 
         for project_id in child_projects:
@@ -169,23 +175,24 @@ async def untrash_folder(
     # 3. UNtrash
 
     # 3.1 UNtrash folder and children
-    await _folders_db_update(
+    await _folders_db_trashed_state_update(
         app,
         folder_id=folder_id,
         product_name=product_name,
         trashed_at=None,
-        trashed_by=user_id,
+        trashed_by=None,
+        trashed_explicitly=False,
     )
 
     # 3.2 UNtrash all child projects that I am an owner
-    child_projects: list[
-        ProjectID
-    ] = await _folders_repository.get_projects_recursively_only_if_user_is_owner(
-        app,
-        folder_id=folder_id,
-        private_workspace_user_id_or_none=user_id if workspace_is_private else None,
-        user_id=user_id,
-        product_name=product_name,
+    child_projects: list[ProjectID] = (
+        await _folders_repository.get_projects_recursively_only_if_user_is_owner(
+            app,
+            folder_id=folder_id,
+            private_workspace_user_id_or_none=user_id if workspace_is_private else None,
+            user_id=user_id,
+            product_name=product_name,
+        )
     )
 
     for project_id in child_projects:
@@ -227,7 +234,7 @@ async def list_explicitly_trashed_folders(
             user_id=user_id,
             product_name=product_name,
             text=None,
-            trashed=True,  # NOTE: lists only expliclty trashed!
+            trashed=True,  # NOTE: lists only explicitly trashed!
             offset=page_params.offset,
             limit=page_params.limit,
             order_by=OrderBy(field=IDStr("trashed"), direction=OrderDirection.ASC),
@@ -275,7 +282,7 @@ async def delete_trashed_folder(
         )
 
     # NOTE: this function deletes folder AND its content recursively!
-    await _folders_service.delete_folder(
+    await _folders_service.delete_folder_with_all_content(
         app, user_id=user_id, folder_id=folder_id, product_name=product_name
     )
 
@@ -298,7 +305,7 @@ async def batch_delete_trashed_folders_as_admin(
         (
             page_params.total_number_of_items,
             expired_trashed_folders,
-        ) = await _folders_repository.list_trashed_folders(
+        ) = await _folders_repository.list_folders_db_as_admin(
             app,
             trashed_explicitly=True,
             trashed_before=trashed_before,
@@ -324,3 +331,48 @@ async def batch_delete_trashed_folders_as_admin(
         raise FolderBatchDeleteError(
             errors=errors, trashed_before=trashed_before, product_name=product_name
         )
+
+
+async def batch_delete_folders_with_content_in_root_workspace_as_admin(
+    app: web.Application,
+    *,
+    workspace_id: WorkspaceID,
+    product_name: ProductName,
+    fail_fast: bool,
+) -> None:
+    """
+    Deletes all folders recursively in the workspace root.
+
+    Raises:
+        FolderBatchDeleteError: If there are errors during the deletion process.
+    """
+    deleted_folder_ids: list[FolderID] = []
+    errors: list[tuple[FolderID, Exception]] = []
+
+    for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE):
+        (
+            page_params.total_number_of_items,
+            folders_for_deletion,
+        ) = await _folders_repository.list_folders_db_as_admin(
+            app,
+            shared_workspace_id=workspace_id,  # <-- Workspace filter
+            offset=page_params.offset,
+            limit=page_params.limit,
+            order_by=OrderBy(field=IDStr("folder_id")),
+        )
+        # BATCH delete
+        for folder in folders_for_deletion:
+            try:
+                await _folders_repository.delete_recursively(
+                    app, folder_id=folder.folder_id, product_name=product_name
+                )
+                deleted_folder_ids.append(folder.folder_id)
+            except Exception as err:  # pylint: disable=broad-exception-caught
+                if fail_fast:
+                    raise
+                errors.append((folder.folder_id, err))
+
+    if errors:
+        raise FolderBatchDeleteError(
+            errors=errors,
+        )

services/web/server/src/simcore_service_webserver/folders/service.py

@@ -0,0 +1,8 @@
+from ._folders_service import delete_folder_with_all_content, list_folders
+
+__all__: tuple[str, ...] = (
+    "list_folders",
+    "delete_folder_with_all_content",
+)
+
+# nopycln: file