safe empty trash

Author: pcrespov

services/web/server/src/simcore_service_webserver/projects/_trash_service.py

@@ -129,11 +129,33 @@ async def untrash_project(
     )
 
 
-def _get_trashed_fields(project: ProjectDict):
+def _can_delete(
+    project: ProjectDict,
+    user_id: UserID,
+    until_equal_datetime: datetime.datetime | None,
+) -> bool:
+    """
+    This is the current policy to delete trashed project
+
+    """
     trashed_at = project.get("trashed")
     trashed_by = project.get("trashedBy")
     trashed_explicitly = project.get("trashedExplicitly")
-    return trashed_at, trashed_by, trashed_explicitly
+
+    assert trashed_at is not None  # nosec
+    assert trashed_by is not None  # nosec
+
+    is_shared = len(project["accessRights"]) > 1
+
+    return bool(
+        trashed_at
+        and (until_equal_datetime is None or trashed_at < until_equal_datetime)
+        # NOTE: current policy is more restricted until
+        # logic is adapted to deal with the other cases
+        and trashed_by == user_id
+        and not is_shared
+        and trashed_explicitly
+    )
 
 
 async def list_trashed_projects(
@@ -170,18 +192,11 @@ async def list_trashed_projects(
         # This filtering couldn't be handled at the database level when `projects_repo`
         # was refactored, as defining a custom trash_filter was needed to allow more
         # flexibility in filtering options.
-
-        for project in projects:
-            trashed_at, trashed_by, trashed_explicitly = _get_trashed_fields(project)
-
-            if (
-                trashed_at
-                and (until_equal_datetime is None or trashed_at < until_equal_datetime)
-                and trashed_by == user_id
-                and trashed_explicitly
-            ):
-                trashed_projects.append(project["uuid"])
-
+        trashed_projects = [
+            project["uuid"]
+            for project in projects
+            if _can_delete(project, user_id, until_equal_datetime)
+        ]
     return trashed_projects
 
 
@@ -206,22 +221,16 @@ async def delete_trashed_project(
     if not project:
         raise ProjectNotFoundError(project_uuid=project_id, user_id=user_id)
 
-    trashed_at, trashed_by, trashed_explicitly = _get_trashed_fields(project)
-
-    if (
-        not trashed_at
-        or (until_equal_datetime is not None and until_equal_datetime < trashed_at)
-        or trashed_by != user_id
-        or not trashed_explicitly
-    ):
+    if not _can_delete(project, user_id, until_equal_datetime):
+        # safety check
         raise ProjectNotTrashedError(
             project_uuid=project_id,
             user_id=user_id,
-            reason="Project was not trashed explicitly by the user from the specified datetime",
+            reason="Cannot delete trashed project since it does not fit current criteria",
         )
-    # FIXME: locking while deleting?
-    # FIXME: this can be heavy. Rather call delete_trashed_project as a task
+
     await projects_service.delete_project_by_user(
+        # FIXME: locking while deleting?
         app,
         user_id=user_id,
         project_uuid=project_id,

services/web/server/src/simcore_service_webserver/trash/_service.py

@@ -2,44 +2,77 @@
 import logging
 from datetime import timedelta
 
+import arrow
 from aiohttp import web
 from models_library.products import ProductName
 from models_library.users import UserID
+from servicelib.logging_errors import create_troubleshotting_log_kwargs
+from servicelib.logging_utils import log_context
 
 from ..projects import _trash_service
 from .settings import get_plugin_settings
 
 _logger = logging.getLogger(__name__)
 
 
-async def empty_trash(app: web.Application, product_name: ProductName, user_id: UserID):
+async def empty_trash_safe(
+    app: web.Application, product_name: ProductName, user_id: UserID
+):
     assert app  # nosec
-    # filter trashed=True and set them to False
-    _logger.debug(
-        "CODE PLACEHOLDER: all projects marked as trashed of %s in %s are deleted",
-        f"{user_id=}",
-        f"{product_name=}",
-    )
 
     trashed_projects_ids = await _trash_service.list_trashed_projects(
         app=app, product_name=product_name, user_id=user_id
     )
 
-    for project_id in trashed_projects_ids:
-        # TODO: handle error. should not be fail-fast!
-        await _trash_service.delete_trashed_project(
-            app, user_id=user_id, project_id=project_id
-        )
+    with log_context(
+        _logger,
+        logging.DEBUG,
+        "Deleting %s trashed projects",
+        len(trashed_projects_ids),
+    ):
+        for project_id in trashed_projects_ids:
+            try:
+
+                await _trash_service.delete_trashed_project(
+                    app,
+                    user_id=user_id,
+                    project_id=project_id,
+                )
+
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                _logger.warning(
+                    **create_troubleshotting_log_kwargs(
+                        "Error deleting a trashed project while emptying trash.",
+                        error=exc,
+                        error_context={
+                            "project_id": project_id,
+                            "product_name": product_name,
+                            "user_id": user_id,
+                        },
+                        tip="`empty_trash_safe` is set `fail_fast=False`."
+                        "\nErrors while deletion are ignored."
+                        "\nNew runs might resolve them",
+                    )
+                )
 
 
 async def prune_trash(app: web.Application) -> list[str]:
     """Deletes expired items in the trash"""
     settings = get_plugin_settings(app)
+
+    # app-wide
     retention = timedelta(days=settings.TRASH_RETENTION_DAYS)
+    expiration_dt = arrow.now().datetime - retention
 
+    # TODO:
+    #   for each product
+    #      list_trashed_projects
+    #      sort by owner
+    #      as owner start deleting
     _logger.debug(
-        "CODE PLACEHOLDER: **ALL** projects marked as trashed during %s days are deleted",
+        "CODE PLACEHOLDER: **ALL** items marked as trashed during %s days are deleted (those marked before %s)",
         retention,
+        expiration_dt,
     )
     await asyncio.sleep(5)
 

services/web/server/tests/unit/with_dbs/03/test_trash.py

@@ -836,8 +836,6 @@ async def test_trash_project_explitictly_and_empty_trash_bin(
 
     # force EMPTY trash
     resp = await client.delete("/v0/trash")
-    # TODO: POST trash:empty -> logs number of elements and
-    # starts delete in the background ? Assume many many items and can take really long
     await assert_status(resp, status.HTTP_204_NO_CONTENT)
 
     # waits for deletion