✨ web-api: Empty bins for folders and its content (#7237)

Author: pcrespov

services/web/server/src/simcore_service_webserver/folders/_trash_service.py

@@ -3,17 +3,23 @@
 
 import arrow
 from aiohttp import web
-from models_library.folders import FolderID
+from common_library.pagination_tools import iter_pagination_params
+from models_library.access_rights import AccessRights
+from models_library.basic_types import IDStr
+from models_library.folders import FolderDB, FolderID
 from models_library.products import ProductName
 from models_library.projects import ProjectID
+from models_library.rest_ordering import OrderBy, OrderDirection
+from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE
 from models_library.users import UserID
 from simcore_postgres_database.utils_repos import transaction_context
 from sqlalchemy.ext.asyncio import AsyncConnection
 
 from ..db.plugin import get_asyncpg_engine
 from ..projects._trash_service import trash_project, untrash_project
 from ..workspaces.api import check_user_workspace_access
-from . import _folders_repository
+from . import _folders_repository, _folders_service
+from .errors import FolderNotTrashedError
 
 _logger = logging.getLogger(__name__)
 
@@ -186,3 +192,89 @@ async def untrash_folder(
         await untrash_project(
             app, product_name=product_name, user_id=user_id, project_id=project_id
         )
+
+
+def _can_delete(
+    folder_db: FolderDB,
+    my_access_rights: AccessRights,
+    user_id: UserID,
+    until_equal_datetime: datetime | None,
+) -> bool:
+    return bool(
+        folder_db.trashed
+        and (until_equal_datetime is None or folder_db.trashed < until_equal_datetime)
+        and my_access_rights.delete
+        and folder_db.trashed_by == user_id
+        and folder_db.trashed_explicitly
+    )
+
+
+async def list_explicitly_trashed_folders(
+    app: web.Application,
+    *,
+    product_name: ProductName,
+    user_id: UserID,
+    until_equal_datetime: datetime | None = None,
+) -> list[FolderID]:
+    trashed_folder_ids: list[FolderID] = []
+
+    for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE):
+        (
+            folders,
+            page_params.total_number_of_items,
+        ) = await _folders_service.list_folders_full_depth(
+            app,
+            user_id=user_id,
+            product_name=product_name,
+            text=None,
+            trashed=True,
+            offset=page_params.offset,
+            limit=page_params.limit,
+            order_by=OrderBy(field=IDStr("trashed"), direction=OrderDirection.ASC),
+        )
+
+        # NOTE: Applying POST-FILTERING
+        trashed_folder_ids.extend(
+            [
+                f.folder_db.folder_id
+                for f in folders
+                if _can_delete(
+                    f.folder_db,
+                    my_access_rights=f.my_access_rights,
+                    user_id=user_id,
+                    until_equal_datetime=until_equal_datetime,
+                )
+            ]
+        )
+    return trashed_folder_ids
+
+
+async def delete_trashed_folder(
+    app: web.Application,
+    *,
+    product_name: ProductName,
+    user_id: UserID,
+    folder_id: FolderID,
+    until_equal_datetime: datetime | None = None,
+) -> None:
+
+    folder = await _folders_service.get_folder(
+        app, user_id=user_id, folder_id=folder_id, product_name=product_name
+    )
+
+    if not _can_delete(
+        folder.folder_db,
+        folder.my_access_rights,
+        user_id=user_id,
+        until_equal_datetime=until_equal_datetime,
+    ):
+        raise FolderNotTrashedError(
+            folder_id=folder_id,
+            user_id=user_id,
+            reason="Cannot delete trashed folder since it does not fit current criteria",
+        )
+
+    # NOTE: this function deletes folder AND its content recursively!
+    await _folders_service.delete_folder(
+        app, user_id=user_id, folder_id=folder_id, product_name=product_name
+    )

services/web/server/src/simcore_service_webserver/folders/errors.py

@@ -19,3 +19,13 @@ class FolderAccessForbiddenError(FoldersValueError):
 
 class FolderGroupNotFoundError(FoldersValueError):
     msg_template = "Folder group not found. {reason}"
+
+
+class FoldersRuntimeError(WebServerBaseError, RuntimeError):
+    ...
+
+
+class FolderNotTrashedError(FoldersRuntimeError):
+    msg_template = (
+        "Cannot delete folder {folder_id} since it was not trashed first: {reason}"
+    )

services/web/server/src/simcore_service_webserver/folders/folders_trash_service.py

@@ -0,0 +1,8 @@
+from ._trash_service import delete_trashed_folder, list_explicitly_trashed_folders
+
+__all__: tuple[str, ...] = (
+    "delete_trashed_folder",
+    "list_explicitly_trashed_folders",
+)
+
+# nopycln: file

services/web/server/src/simcore_service_webserver/projects/_common/exceptions_handlers.py

@@ -1,4 +1,6 @@
+import itertools
 import logging
+from collections import Counter
 
 from servicelib.aiohttp import status
 from servicelib.rabbitmq.rpc_interfaces.catalog.errors import (
@@ -174,16 +176,37 @@
     ),
 }
 
+
+_ERRORS = [
+    _FOLDER_ERRORS,
+    _NODE_ERRORS,
+    _OTHER_ERRORS,
+    _PRICING_ERRORS,
+    _PROJECT_ERRORS,
+    _WALLET_ERRORS,
+    _WORKSPACE_ERRORS,
+]
+
+
+def _assert_duplicate():
+    duplicates = {
+        exc.__name__: count
+        for exc, count in Counter(itertools.chain(*[d.keys() for d in _ERRORS])).items()
+        if count > 1
+    }
+    if duplicates:
+        msg = f"Found duplicated exceptions: {duplicates}"
+        raise AssertionError(msg)
+    return True
+
+
+assert _assert_duplicate()  # nosec
+
 _TO_HTTP_ERROR_MAP: ExceptionToHttpErrorMap = {
-    **_FOLDER_ERRORS,
-    **_NODE_ERRORS,
-    **_OTHER_ERRORS,
-    **_PRICING_ERRORS,
-    **_PROJECT_ERRORS,
-    **_WALLET_ERRORS,
-    **_WORKSPACE_ERRORS,
+    k: v for dikt in _ERRORS for k, v in dikt.items()
 }
 
+
 handle_plugin_requests_exceptions = exception_handling_decorator(
     to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP)
 )

services/web/server/src/simcore_service_webserver/projects/_trash_service.py

@@ -1,6 +1,6 @@
 import asyncio
-import datetime
 import logging
+from datetime import datetime
 
 import arrow
 from aiohttp import web
@@ -9,6 +9,7 @@
 from models_library.products import ProductName
 from models_library.projects import ProjectID
 from models_library.rest_ordering import OrderBy, OrderDirection
+from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE
 from models_library.users import UserID
 from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY
 from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE
@@ -132,7 +133,7 @@ async def untrash_project(
 def _can_delete(
     project: ProjectDict,
     user_id: UserID,
-    until_equal_datetime: datetime.datetime | None,
+    until_equal_datetime: datetime | None,
 ) -> bool:
     """
     This is the current policy to delete trashed project
@@ -158,19 +159,19 @@ def _can_delete(
     )
 
 
-async def list_trashed_projects(
+async def list_explicitly_trashed_projects(
     app: web.Application,
     *,
     product_name: ProductName,
     user_id: UserID,
-    until_equal_datetime: datetime.datetime | None = None,
+    until_equal_datetime: datetime | None = None,
 ) -> list[ProjectID]:
     """
     Lists all projects that were trashed until a specific datetime (if !=None).
     """
     trashed_projects: list[ProjectID] = []
 
-    for page_params in iter_pagination_params(limit=100):
+    for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE):
         (
             projects,
             page_params.total_number_of_items,
@@ -202,12 +203,12 @@ async def list_trashed_projects(
     return trashed_projects
 
 
-async def delete_trashed_project(
+async def delete_explicitly_trashed_project(
     app: web.Application,
     *,
     user_id: UserID,
     project_id: ProjectID,
-    until_equal_datetime: datetime.datetime | None = None,
+    until_equal_datetime: datetime | None = None,
 ) -> None:
     """
     Deletes a project that was explicitly trashed by the user from a specific datetime (if provided, otherwise all).

services/web/server/src/simcore_service_webserver/projects/projects_trash_service.py

@@ -0,0 +1,11 @@
+from ._trash_service import (
+    delete_explicitly_trashed_project,
+    list_explicitly_trashed_projects,
+)
+
+__all__: tuple[str, ...] = (
+    "delete_explicitly_trashed_project",
+    "list_explicitly_trashed_projects",
+)
+
+# nopycln: file

services/web/server/src/simcore_service_webserver/trash/_service.py

@@ -1,6 +1,7 @@
 import asyncio
 import logging
 from datetime import timedelta
+from typing import Final
 
 import arrow
 from aiohttp import web
@@ -9,31 +10,38 @@
 from servicelib.logging_errors import create_troubleshotting_log_kwargs
 from servicelib.logging_utils import log_context
 
-from ..projects import _trash_service
+from ..folders import folders_trash_service
+from ..projects import projects_trash_service
 from .settings import get_plugin_settings
 
 _logger = logging.getLogger(__name__)
 
+_TIP: Final[str] = (
+    "`empty_trash_safe` is set `fail_fast=False`."
+    "\nErrors while deletion are ignored."
+    "\nNew runs might resolve them"
+)
 
-async def empty_trash_safe(
+
+async def _empty_explicitly_trashed_projects(
     app: web.Application, product_name: ProductName, user_id: UserID
 ):
-    assert app  # nosec
-
-    trashed_projects_ids = await _trash_service.list_trashed_projects(
-        app=app, product_name=product_name, user_id=user_id
+    trashed_projects_ids = (
+        await projects_trash_service.list_explicitly_trashed_projects(
+            app=app, product_name=product_name, user_id=user_id
+        )
     )
 
     with log_context(
         _logger,
         logging.DEBUG,
-        "Deleting %s trashed projects",
+        "Deleting %s explicitly trashed projects",
         len(trashed_projects_ids),
     ):
         for project_id in trashed_projects_ids:
             try:
 
-                await _trash_service.delete_trashed_project(
+                await projects_trash_service.delete_explicitly_trashed_project(
                     app,
                     user_id=user_id,
                     project_id=project_id,
@@ -49,13 +57,56 @@ async def empty_trash_safe(
                             "product_name": product_name,
                             "user_id": user_id,
                         },
-                        tip="`empty_trash_safe` is set `fail_fast=False`."
-                        "\nErrors while deletion are ignored."
-                        "\nNew runs might resolve them",
+                        tip=_TIP,
                     )
                 )
 
 
+async def _empty_trashed_folders(
+    app: web.Application, product_name: ProductName, user_id: UserID
+):
+    trashed_folders_ids = await folders_trash_service.list_explicitly_trashed_folders(
+        app=app, product_name=product_name, user_id=user_id
+    )
+
+    with log_context(
+        _logger,
+        logging.DEBUG,
+        "Deleting %s trashed folders (and all its content)",
+        len(trashed_folders_ids),
+    ):
+        for folder_id in trashed_folders_ids:
+            try:
+                await folders_trash_service.delete_trashed_folder(
+                    app,
+                    product_name=product_name,
+                    user_id=user_id,
+                    folder_id=folder_id,
+                )
+
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                _logger.warning(
+                    **create_troubleshotting_log_kwargs(
+                        "Error deleting a trashed folders (and content) while emptying trash.",
+                        error=exc,
+                        error_context={
+                            "folder_id": folder_id,
+                            "product_name": product_name,
+                            "user_id": user_id,
+                        },
+                        tip=_TIP,
+                    )
+                )
+
+
+async def empty_trash_safe(
+    app: web.Application, *, product_name: ProductName, user_id: UserID
+):
+    await _empty_explicitly_trashed_projects(app, product_name, user_id)
+
+    await _empty_trashed_folders(app, product_name, user_id)
+
+
 async def prune_trash(app: web.Application) -> list[str]:
     """Deletes expired items in the trash"""
     settings = get_plugin_settings(app)