improve delete workspace

Author: matusdrobuliak66

services/web/server/src/simcore_service_webserver/folders/_folders_repository.py

@@ -260,13 +260,14 @@ async def list_(  # pylint: disable=too-many-arguments,too-many-branches
         return cast(int, total_count), folders
 
 
-async def list_trashed_folders(
+async def list_folders_db_as_admin(
     app: web.Application,
     connection: AsyncConnection | None = None,
     *,
     # filter
     trashed_explicitly: bool | UnSet = UnSet.VALUE,
     trashed_before: datetime | UnSet = UnSet.VALUE,
+    shared_workspace_id: WorkspaceID | UnSet = UnSet.VALUE,  # <-- Workspace filter
     # pagination
     offset: NonNegativeInt,
     limit: int,
@@ -275,7 +276,6 @@ async def list_trashed_folders(
 ) -> tuple[int, list[FolderDB]]:
     """
     NOTE: this is app-wide i.e. no product, user or workspace filtered
-    TODO: check with MD about workspaces
     """
     base_query = sql.select(*_FOLDER_DB_MODEL_COLS).where(
         folders_v2.c.trashed.is_not(None)
@@ -291,6 +291,10 @@ async def list_trashed_folders(
         assert isinstance(trashed_before, datetime)  # nosec
         base_query = base_query.where(folders_v2.c.trashed < trashed_before)
 
+    if is_set(shared_workspace_id):
+        assert isinstance(shared_workspace_id, int)  # nosec
+        base_query = base_query.where(folders_v2.c.workspace_id == shared_workspace_id)
+
     # Select total count from base_query
     count_query = sql.select(sql.func.count()).select_from(base_query.subquery())
 

services/web/server/src/simcore_service_webserver/folders/_trash_service.py

@@ -12,6 +12,7 @@
 from models_library.rest_ordering import OrderBy, OrderDirection
 from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE
 from models_library.users import UserID
+from models_library.workspaces import WorkspaceID
 from simcore_postgres_database.utils_repos import transaction_context
 from sqlalchemy.ext.asyncio import AsyncConnection
 
@@ -229,7 +230,7 @@ async def list_explicitly_trashed_folders(
             user_id=user_id,
             product_name=product_name,
             text=None,
-            trashed=True,  # NOTE: lists only expliclty trashed!
+            trashed=True,  # NOTE: lists only explicitly trashed!
             offset=page_params.offset,
             limit=page_params.limit,
             order_by=OrderBy(field=IDStr("trashed"), direction=OrderDirection.ASC),
@@ -300,7 +301,7 @@ async def batch_delete_trashed_folders_as_admin(
         (
             page_params.total_number_of_items,
             expired_trashed_folders,
-        ) = await _folders_repository.list_trashed_folders(
+        ) = await _folders_repository.list_folders_db_as_admin(
             app,
             trashed_explicitly=True,
             trashed_before=trashed_before,
@@ -326,3 +327,48 @@ async def batch_delete_trashed_folders_as_admin(
         raise FolderBatchDeleteError(
             errors=errors, trashed_before=trashed_before, product_name=product_name
         )
+
+
+async def batch_delete_folders_with_content_in_root_workspace_as_admin(
+    app: web.Application,
+    *,
+    workspace_id: WorkspaceID,
+    product_name: ProductName,
+    fail_fast: bool,
+) -> None:
+    """
+    Deletes all projects in the workspace root.
+
+    Raises:
+        ProjectsBatchDeleteError: If there are errors during the deletion process.
+    """
+    deleted_folder_ids: list[FolderID] = []
+    errors: list[tuple[FolderID, Exception]] = []
+
+    for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE):
+        (
+            page_params.total_number_of_items,
+            folders_for_deletion,
+        ) = await _folders_repository.list_folders_db_as_admin(
+            app,
+            shared_workspace_id=workspace_id,  # <-- Workspace filter
+            offset=page_params.offset,
+            limit=page_params.limit,
+            order_by=OrderBy(field=IDStr("id")),
+        )
+        # BATCH delete
+        for folder in folders_for_deletion:
+            try:
+                await _folders_repository.delete_recursively(
+                    app, folder_id=folder.folder_id, product_name=product_name
+                )
+                deleted_folder_ids.append(folder.folder_id)
+            except Exception as err:  # pylint: disable=broad-exception-caught
+                if fail_fast:
+                    raise
+                errors.append((folder.folder_id, err))
+
+    if errors:
+        raise FolderBatchDeleteError(
+            errors=errors,
+        )

services/web/server/src/simcore_service_webserver/projects/_projects_repository.py

@@ -11,6 +11,7 @@
 from models_library.projects import ProjectID
 from models_library.rest_ordering import OrderBy, OrderDirection
 from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE
+from models_library.workspaces import WorkspaceID
 from pydantic import NonNegativeInt, PositiveInt
 from simcore_postgres_database.models.projects import projects
 from simcore_postgres_database.models.users import users
@@ -48,18 +49,19 @@ def _to_sql_expression(table: sa.Table, order_by: OrderBy):
     return direction_func(table.columns[order_by.field])
 
 
-async def list_trashed_projects(
+async def list_projects_db_get_as_admin(
     app: web.Application,
     connection: AsyncConnection | None = None,
     *,
     # filter
     trashed_explicitly: bool | UnSet = UnSet.VALUE,
     trashed_before: datetime | UnSet = UnSet.VALUE,
+    shared_workspace_id: WorkspaceID | UnSet = UnSet.VALUE,
     # pagination
     offset: NonNegativeInt = 0,
     limit: PositiveInt = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE,
     # order
-    order_by: OrderBy = OLDEST_TRASHED_FIRST,
+    order_by: OrderBy,
 ) -> tuple[int, list[ProjectDBGet]]:
 
     base_query = sql.select(*PROJECT_DB_COLS).where(projects.c.trashed.is_not(None))
@@ -74,12 +76,16 @@ async def list_trashed_projects(
         assert isinstance(trashed_before, datetime)  # nosec
         base_query = base_query.where(projects.c.trashed < trashed_before)
 
+    if is_set(shared_workspace_id):
+        assert isinstance(shared_workspace_id, int)  # nosec
+        base_query = base_query.where(projects.c.workspace_id == shared_workspace_id)
+
     # Select total count from base_query
     count_query = sql.select(sql.func.count()).select_from(base_query.subquery())
 
     # Ordering and pagination
     list_query = (
-        base_query.order_by(_to_sql_expression(projects, order_by))
+        base_query.order_by(_to_sql_expression(projects, order_by), projects.c.id)
         .offset(offset)
         .limit(limit)
     )

services/web/server/src/simcore_service_webserver/projects/_trash_service.py

@@ -10,6 +10,7 @@
 from models_library.rest_ordering import OrderBy, OrderDirection
 from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE
 from models_library.users import UserID
+from models_library.workspaces import WorkspaceID
 from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY
 from servicelib.utils import fire_and_forget_task
 
@@ -241,7 +242,7 @@ async def batch_delete_trashed_projects_as_admin(
         (
             page_params.total_number_of_items,
             expired_trashed_projects,
-        ) = await _projects_repository.list_trashed_projects(
+        ) = await _projects_repository.list_projects_db_get_as_admin(
             app,
             # both implicit and explicitly trashed
             trashed_before=trashed_before,
@@ -273,3 +274,51 @@ async def batch_delete_trashed_projects_as_admin(
         )
 
     return deleted_project_ids
+
+
+async def batch_delete_projects_in_root_workspace_as_admin(
+    app: web.Application,
+    *,
+    workspace_id: WorkspaceID,
+    fail_fast: bool,
+) -> list[ProjectID]:
+    """
+    Deletes all projects in the workspace root.
+
+    Raises:
+        ProjectsBatchDeleteError: If there are errors during the deletion process.
+    """
+    deleted_project_ids: list[ProjectID] = []
+    errors: list[tuple[ProjectID, Exception]] = []
+
+    for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE):
+        (
+            page_params.total_number_of_items,
+            projects_for_deletion,
+        ) = await _projects_repository.list_projects_db_get_as_admin(
+            app,
+            shared_workspace_id=workspace_id,  # <-- Workspace filter
+            offset=page_params.offset,
+            limit=page_params.limit,
+            order_by=OrderBy(field=IDStr("id")),
+        )
+        # BATCH delete
+        for project in projects_for_deletion:
+            try:
+                await _projects_service_delete.delete_project_as_admin(
+                    app,
+                    project_uuid=project.uuid,
+                )
+                deleted_project_ids.append(project.uuid)
+            except Exception as err:  # pylint: disable=broad-exception-caught
+                if fail_fast:
+                    raise
+                errors.append((project.uuid, err))
+
+    if errors:
+        raise ProjectsBatchDeleteError(
+            errors=errors,
+            deleted_project_ids=deleted_project_ids,
+        )
+
+    return deleted_project_ids

services/web/server/src/simcore_service_webserver/trash/_service.py

@@ -218,3 +218,23 @@ async def safe_delete_expired_trash_as_admin(app: web.Application) -> None:
                         error_context=ctx,
                     )
                 )
+
+            try:
+                deleted_workspace_ids = await workspaces_trash_service.batch_delete_trashed_workspaces_as_admin(
+                    app,
+                    trashed_before=delete_until,
+                    fail_fast=False,
+                )
+
+                _logger.info(
+                    "Deleted %d trashed workspaces", len(deleted_workspace_ids)
+                )
+
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                _logger.warning(
+                    **create_troubleshotting_log_kwargs(
+                        "Error batch deleting expired projects as admin.",
+                        error=exc,
+                        error_context=ctx,
+                    )
+                )

services/web/server/src/simcore_service_webserver/workspaces/_trash_service.py

@@ -20,11 +20,19 @@
 
 from ..db.plugin import get_asyncpg_engine
 from ..folders._folders_service import list_folders
-from ..folders._trash_service import trash_folder, untrash_folder
+from ..folders._trash_service import (
+    batch_delete_folders_with_content_in_root_workspace_as_admin,
+    trash_folder,
+    untrash_folder,
+)
 from ..projects._crud_api_read import ProjectTypeAPI, list_projects
-from ..projects._trash_service import trash_project, untrash_project
+from ..projects._trash_service import (
+    batch_delete_projects_in_root_workspace_as_admin,
+    trash_project,
+    untrash_project,
+)
 from . import _workspaces_repository, _workspaces_service, _workspaces_service_crud_read
-from .errors import WorkspaceNotTrashedError
+from .errors import WorkspaceBatchDeleteError, WorkspaceNotTrashedError
 
 _logger = logging.getLogger(__name__)
 
@@ -301,3 +309,63 @@ async def list_trashed_workspaces(
         )
 
     return trashed_workspace_ids
+
+
+async def batch_delete_trashed_workspaces_as_admin(
+    app: web.Application,
+    *,
+    trashed_before: datetime,
+    fail_fast: bool,
+) -> list[WorkspaceID]:
+
+    deleted_workspace_ids: list[WorkspaceID] = []
+    errors: list[tuple[WorkspaceID, Exception]] = []
+
+    for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE):
+        (
+            page_params.total_number_of_items,
+            expired_trashed_workspaces,
+        ) = await _workspaces_repository.list_workspaces_db_get_as_admin(
+            app,
+            trashed_before=trashed_before,
+            offset=page_params.offset,
+            limit=page_params.limit,
+            order_by=OrderBy(field=IDStr("trashed"), direction=OrderDirection.ASC),
+        )
+        # BATCH delete
+        for trashed_workspace in expired_trashed_workspaces:
+            assert trashed_workspace.trashed  # nosec
+            deleted_workspace_ids.append(trashed_workspace.workspace_id)
+
+            try:
+                await batch_delete_projects_in_root_workspace_as_admin(
+                    app, workspace_id=trashed_workspace.workspace_id, fail_fast=False
+                )
+            except Exception as err:  # pylint: disable=broad-exception-caught
+                if fail_fast:
+                    raise
+                errors.append((trashed_workspace.workspace_id, err))
+
+            try:
+                workspace_db_get = await _workspaces_repository.get_workspace_db_get(
+                    app, workspace_id=trashed_workspace.workspace_id
+                )
+
+                await batch_delete_folders_with_content_in_root_workspace_as_admin(
+                    app,
+                    workspace_id=trashed_workspace.workspace_id,
+                    product_name=workspace_db_get.product_name,
+                    fail_fast=False,
+                )
+            except Exception as err:  # pylint: disable=broad-exception-caught
+                if fail_fast:
+                    raise
+                errors.append((trashed_workspace.workspace_id, err))
+
+    if errors:
+        raise WorkspaceBatchDeleteError(
+            errors=errors,
+            deleted_workspace_ids=deleted_workspace_ids,
+        )
+
+    return deleted_workspace_ids

services/web/server/src/simcore_service_webserver/workspaces/_workspaces_models.py

@@ -0,0 +1,35 @@
+from datetime import datetime
+
+from models_library.groups import GroupID
+from models_library.products import ProductName
+from models_library.users import UserID
+from models_library.workspaces import WorkspaceID
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+)
+
+
+class WorkspaceDBGet(BaseModel):
+    workspace_id: WorkspaceID
+    name: str
+    description: str | None
+    owner_primary_gid: GroupID = Field(
+        ...,
+        description="GID of the group that owns this wallet",
+    )
+    thumbnail: str | None
+    created: datetime = Field(
+        ...,
+        description="Timestamp on creation",
+    )
+    modified: datetime = Field(
+        ...,
+        description="Timestamp of last modification",
+    )
+    trashed: datetime | None
+    trashed_by: UserID | None
+    product_name: ProductName
+
+    model_config = ConfigDict(from_attributes=True)