diff --git a/services/web/server/src/simcore_service_webserver/director_v2/_service.py b/services/web/server/src/simcore_service_webserver/director_v2/_service.py index ff7402fef7ed..10bad99a0c43 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2/_service.py +++ b/services/web/server/src/simcore_service_webserver/director_v2/_service.py @@ -26,7 +26,7 @@ from ..db.plugin import get_database_engine from ..products import products_service from ..products.models import Product -from ..projects import api as projects_service +from ..projects import projects_wallets_service from ..users import preferences_api as user_preferences_service from ..users.exceptions import UserDefaultWalletNotFoundError from ..wallets import api as wallets_service @@ -204,7 +204,7 @@ async def get_wallet_info( product.is_payment_enabled and app_settings.WEBSERVER_CREDIT_COMPUTATION_ENABLED ): return None - project_wallet = await projects_service.get_project_wallet( + project_wallet = await projects_wallets_service.get_project_wallet( app, project_id=project_id ) if project_wallet is None: @@ -219,7 +219,7 @@ async def get_wallet_info( project_wallet_id = TypeAdapter(WalletID).validate_python( user_default_wallet_preference.value ) - await projects_service.connect_wallet_to_project( + await projects_wallets_service.connect_wallet_to_project( app, product_name=product_name, project_id=project_id, diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py index f57a1c6df848..0076c48ae0e6 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py @@ -245,7 +245,9 @@ async def list_( # pylint: disable=too-many-arguments,too-many-branches # Ordering and pagination list_query = ( - combined_query.order_by(_to_expression(order_by)).offset(offset).limit(limit) + combined_query.order_by(_to_expression(order_by), folders_v2.c.folder_id) + .offset(offset) + .limit(limit) ) async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: @@ -258,13 +260,14 @@ async def list_( # pylint: disable=too-many-arguments,too-many-branches return cast(int, total_count), folders -async def list_trashed_folders( +async def list_folders_db_as_admin( app: web.Application, connection: AsyncConnection | None = None, *, # filter trashed_explicitly: bool | UnSet = UnSet.VALUE, trashed_before: datetime | UnSet = UnSet.VALUE, + shared_workspace_id: WorkspaceID | UnSet = UnSet.VALUE, # <-- Workspace filter # pagination offset: NonNegativeInt, limit: int, @@ -273,28 +276,35 @@ async def list_trashed_folders( ) -> tuple[int, list[FolderDB]]: """ NOTE: this is app-wide i.e. no product, user or workspace filtered - TODO: check with MD about workspaces """ - base_query = sql.select(*_FOLDER_DB_MODEL_COLS).where( - folders_v2.c.trashed.is_not(None) - ) + base_query = sql.select(*_FOLDER_DB_MODEL_COLS) if is_set(trashed_explicitly): assert isinstance(trashed_explicitly, bool) # nosec base_query = base_query.where( - folders_v2.c.trashed_explicitly.is_(trashed_explicitly) + (folders_v2.c.trashed_explicitly.is_(trashed_explicitly)) + & (folders_v2.c.trashed.is_not(None)) ) if is_set(trashed_before): assert isinstance(trashed_before, datetime) # nosec - base_query = base_query.where(folders_v2.c.trashed < trashed_before) + base_query = base_query.where( + (folders_v2.c.trashed < trashed_before) + & (folders_v2.c.trashed.is_not(None)) + ) + + if is_set(shared_workspace_id): + assert isinstance(shared_workspace_id, int) # nosec + base_query = base_query.where(folders_v2.c.workspace_id == shared_workspace_id) # Select total count from base_query count_query = sql.select(sql.func.count()).select_from(base_query.subquery()) # Ordering and pagination list_query = ( - base_query.order_by(_to_expression(order_by)).offset(offset).limit(limit) + base_query.order_by(_to_expression(order_by), folders_v2.c.folder_id) + .offset(offset) + .limit(limit) ) async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: @@ -376,7 +386,7 @@ async def update( parent_folder_id: FolderID | None | UnSet = UnSet.VALUE, trashed: datetime | None | UnSet = UnSet.VALUE, trashed_explicitly: bool | UnSet = UnSet.VALUE, - trashed_by: UserID | UnSet = UnSet.VALUE, # who trashed + trashed_by: UserID | None | UnSet = UnSet.VALUE, # who trashed workspace_id: WorkspaceID | None | UnSet = UnSet.VALUE, user_id: UserID | None | UnSet = UnSet.VALUE, # ownership ) -> FolderDB: @@ -534,7 +544,7 @@ async def get_all_folders_and_projects_ids_recursively( product_name: ProductName, ) -> tuple[list[FolderID], list[ProjectID]]: """ - The purpose of this function is to retrieve all projects within the provided folder ID. + The purpose of this function is to retrieve all subfolders and projects within the provided folder ID. """ async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py b/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py index c0fe144e086d..5219a500f1a0 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py @@ -205,11 +205,11 @@ async def replace_folder(request: web.Request): @login_required @permission_required("folder.delete") @handle_plugin_requests_exceptions -async def delete_folder_group(request: web.Request): +async def delete_folder(request: web.Request): req_ctx = FoldersRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(FoldersPathParams, request) - await _folders_service.delete_folder( + await _folders_service.delete_folder_with_all_content( app=request.app, user_id=req_ctx.user_id, folder_id=path_params.folder_id, diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_service.py b/services/web/server/src/simcore_service_webserver/folders/_folders_service.py index ffe542dccf74..8b0b63465b01 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_service.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_service.py @@ -11,11 +11,8 @@ from models_library.users import UserID from models_library.workspaces import WorkspaceID, WorkspaceQuery, WorkspaceScope from pydantic import NonNegativeInt -from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY -from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE -from servicelib.utils import fire_and_forget_task -from ..projects._projects_service import submit_delete_project_task +from ..projects._projects_service import delete_project_by_user from ..users.api import get_user from ..workspaces.api import check_user_workspace_access from ..workspaces.errors import ( @@ -320,7 +317,7 @@ async def update_folder( ) -async def delete_folder( +async def delete_folder_with_all_content( app: web.Application, user_id: UserID, folder_id: FolderID, @@ -352,6 +349,9 @@ async def delete_folder( # 1. Delete folder content # 1.1 Delete all child projects that I am an owner + # NOTE: The reason for this is to be cautious and not delete projects by accident that + # are not owned by the user (even if the user was granted delete permissions). As a consequence, after deleting the folder, + # projects that the user does not own will appear in the root. (Maybe this can be changed as we now have a trash system). project_id_list: list[ProjectID] = ( await _folders_repository.get_projects_recursively_only_if_user_is_owner( app, @@ -362,17 +362,11 @@ async def delete_folder( ) ) - # fire and forget task for project deletion for project_id in project_id_list: - fire_and_forget_task( - submit_delete_project_task( - app, - project_uuid=project_id, - user_id=user_id, - simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, - ), - task_suffix_name=f"delete_project_task_{project_id}", - fire_and_forget_tasks_collection=app[APP_FIRE_AND_FORGET_TASKS_KEY], + await delete_project_by_user( + app, + project_uuid=project_id, + user_id=user_id, ) # 1.2 Delete all child folders diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py b/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py index 6540a43eef92..dc46de0a1117 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py +++ b/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py @@ -39,6 +39,7 @@ async def trash_folder(request: web.Request): user_id=user_id, folder_id=path_params.folder_id, force_stop_first=query_params.force, + explicit=True, ) return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py index f5003ee3dba1..6f5765bfce95 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py +++ b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py @@ -12,6 +12,7 @@ from models_library.rest_ordering import OrderBy, OrderDirection from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE from models_library.users import UserID +from models_library.workspaces import WorkspaceID from simcore_postgres_database.utils_repos import transaction_context from sqlalchemy.ext.asyncio import AsyncConnection @@ -62,23 +63,24 @@ async def _check_exists_and_access( return workspace_is_private -async def _folders_db_update( +async def _folders_db_trashed_state_update( app: web.Application, connection: AsyncConnection | None = None, *, product_name: ProductName, folder_id: FolderID, trashed_at: datetime | None, - trashed_by: UserID, + trashed_explicitly: bool, + trashed_by: UserID | None, ): - # EXPLICIT un/trash + # EXPLICIT or IMPLICIT un/trash await _folders_repository.update( app, connection, folders_id_or_ids=folder_id, product_name=product_name, trashed=trashed_at, - trashed_explicitly=trashed_at is not None, + trashed_explicitly=trashed_explicitly, trashed_by=trashed_by, ) @@ -110,6 +112,7 @@ async def trash_folder( user_id: UserID, folder_id: FolderID, force_stop_first: bool, + explicit: bool, ): workspace_is_private = await _check_exists_and_access( @@ -122,25 +125,28 @@ async def trash_folder( async with transaction_context(get_asyncpg_engine(app)) as connection: # 1. Trash folder and children - await _folders_db_update( + await _folders_db_trashed_state_update( app, connection, folder_id=folder_id, product_name=product_name, trashed_at=trashed_at, + trashed_explicitly=explicit, trashed_by=user_id, ) # 2. Trash all child projects that I am an owner - child_projects: list[ - ProjectID - ] = await _folders_repository.get_projects_recursively_only_if_user_is_owner( - app, - connection, - folder_id=folder_id, - private_workspace_user_id_or_none=user_id if workspace_is_private else None, - user_id=user_id, - product_name=product_name, + child_projects: list[ProjectID] = ( + await _folders_repository.get_projects_recursively_only_if_user_is_owner( + app, + connection, + folder_id=folder_id, + private_workspace_user_id_or_none=( + user_id if workspace_is_private else None + ), + user_id=user_id, + product_name=product_name, + ) ) for project_id in child_projects: @@ -169,23 +175,24 @@ async def untrash_folder( # 3. UNtrash # 3.1 UNtrash folder and children - await _folders_db_update( + await _folders_db_trashed_state_update( app, folder_id=folder_id, product_name=product_name, trashed_at=None, - trashed_by=user_id, + trashed_by=None, + trashed_explicitly=False, ) # 3.2 UNtrash all child projects that I am an owner - child_projects: list[ - ProjectID - ] = await _folders_repository.get_projects_recursively_only_if_user_is_owner( - app, - folder_id=folder_id, - private_workspace_user_id_or_none=user_id if workspace_is_private else None, - user_id=user_id, - product_name=product_name, + child_projects: list[ProjectID] = ( + await _folders_repository.get_projects_recursively_only_if_user_is_owner( + app, + folder_id=folder_id, + private_workspace_user_id_or_none=user_id if workspace_is_private else None, + user_id=user_id, + product_name=product_name, + ) ) for project_id in child_projects: @@ -227,7 +234,7 @@ async def list_explicitly_trashed_folders( user_id=user_id, product_name=product_name, text=None, - trashed=True, # NOTE: lists only expliclty trashed! + trashed=True, # NOTE: lists only explicitly trashed! offset=page_params.offset, limit=page_params.limit, order_by=OrderBy(field=IDStr("trashed"), direction=OrderDirection.ASC), @@ -275,7 +282,7 @@ async def delete_trashed_folder( ) # NOTE: this function deletes folder AND its content recursively! - await _folders_service.delete_folder( + await _folders_service.delete_folder_with_all_content( app, user_id=user_id, folder_id=folder_id, product_name=product_name ) @@ -298,7 +305,7 @@ async def batch_delete_trashed_folders_as_admin( ( page_params.total_number_of_items, expired_trashed_folders, - ) = await _folders_repository.list_trashed_folders( + ) = await _folders_repository.list_folders_db_as_admin( app, trashed_explicitly=True, trashed_before=trashed_before, @@ -324,3 +331,48 @@ async def batch_delete_trashed_folders_as_admin( raise FolderBatchDeleteError( errors=errors, trashed_before=trashed_before, product_name=product_name ) + + +async def batch_delete_folders_with_content_in_root_workspace_as_admin( + app: web.Application, + *, + workspace_id: WorkspaceID, + product_name: ProductName, + fail_fast: bool, +) -> None: + """ + Deletes all folders recursively in the workspace root. + + Raises: + FolderBatchDeleteError: If there are errors during the deletion process. + """ + deleted_folder_ids: list[FolderID] = [] + errors: list[tuple[FolderID, Exception]] = [] + + for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + ( + page_params.total_number_of_items, + folders_for_deletion, + ) = await _folders_repository.list_folders_db_as_admin( + app, + shared_workspace_id=workspace_id, # <-- Workspace filter + offset=page_params.offset, + limit=page_params.limit, + order_by=OrderBy(field=IDStr("folder_id")), + ) + # BATCH delete + for folder in folders_for_deletion: + try: + await _folders_repository.delete_recursively( + app, folder_id=folder.folder_id, product_name=product_name + ) + deleted_folder_ids.append(folder.folder_id) + except Exception as err: # pylint: disable=broad-exception-caught + if fail_fast: + raise + errors.append((folder.folder_id, err)) + + if errors: + raise FolderBatchDeleteError( + errors=errors, + ) diff --git a/services/web/server/src/simcore_service_webserver/folders/service.py b/services/web/server/src/simcore_service_webserver/folders/service.py new file mode 100644 index 000000000000..823402a98e0e --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/folders/service.py @@ -0,0 +1,8 @@ +from ._folders_service import delete_folder_with_all_content, list_folders + +__all__: tuple[str, ...] = ( + "list_folders", + "delete_folder_with_all_content", +) + +# nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_states_rest.py b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_states_rest.py index 252708921fe0..2587982f90bf 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_controller/projects_states_rest.py +++ b/services/web/server/src/simcore_service_webserver/projects/_controller/projects_states_rest.py @@ -28,8 +28,7 @@ from ...security.decorators import permission_required from ...users import api from ...utils_aiohttp import envelope_json_response -from .. import _projects_service -from .. import api as projects_api +from .. import _projects_service, projects_wallets_service from ..exceptions import ProjectStartsTooManyDynamicNodesError from ._rest_exceptions import handle_plugin_requests_exceptions from ._rest_schemas import ProjectPathParams, RequestContext @@ -87,7 +86,7 @@ async def open_project(request: web.Request) -> web.Response: ), ) - await projects_api.check_project_financial_status( + await projects_wallets_service.check_project_financial_status( request.app, project_id=path_params.project_id, product_name=req_ctx.product_name, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index 571c778057fd..573298fbe714 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -21,7 +21,7 @@ from ..catalog import catalog_service from ..folders import _folders_repository -from ..workspaces._workspaces_service import check_user_workspace_access +from ..workspaces.api import check_user_workspace_access from . import _projects_service from ._projects_repository import batch_get_trashed_by_primary_gid from ._projects_repository_legacy import ProjectDBAPI @@ -91,7 +91,7 @@ async def list_projects( # pylint: disable=too-many-arguments folder_id: FolderID | None, # attrs filter project_type: ProjectTypeAPI, - show_hidden: bool, + show_hidden: bool, # NOTE: Be careful, this filters only hidden projects trashed: bool | None, # search search_by_multi_columns: str | None = None, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py index d3312bdc1ebd..343d184ff31e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py @@ -11,6 +11,7 @@ from models_library.projects import ProjectID from models_library.rest_ordering import OrderBy, OrderDirection from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE +from models_library.workspaces import WorkspaceID from pydantic import NonNegativeInt, PositiveInt from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.users import users @@ -48,18 +49,19 @@ def _to_sql_expression(table: sa.Table, order_by: OrderBy): return direction_func(table.columns[order_by.field]) -async def list_trashed_projects( +async def list_projects_db_get_as_admin( app: web.Application, connection: AsyncConnection | None = None, *, # filter trashed_explicitly: bool | UnSet = UnSet.VALUE, trashed_before: datetime | UnSet = UnSet.VALUE, + shared_workspace_id: WorkspaceID | UnSet = UnSet.VALUE, # pagination offset: NonNegativeInt = 0, limit: PositiveInt = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, # order - order_by: OrderBy = OLDEST_TRASHED_FIRST, + order_by: OrderBy, ) -> tuple[int, list[ProjectDBGet]]: base_query = sql.select(*PROJECT_DB_COLS).where(projects.c.trashed.is_not(None)) @@ -74,12 +76,16 @@ async def list_trashed_projects( assert isinstance(trashed_before, datetime) # nosec base_query = base_query.where(projects.c.trashed < trashed_before) + if is_set(shared_workspace_id): + assert isinstance(shared_workspace_id, int) # nosec + base_query = base_query.where(projects.c.workspace_id == shared_workspace_id) + # Select total count from base_query count_query = sql.select(sql.func.count()).select_from(base_query.subquery()) # Ordering and pagination list_query = ( - base_query.order_by(_to_sql_expression(projects, order_by)) + base_query.order_by(_to_sql_expression(projects, order_by), projects.c.id) .offset(offset) .limit(limit) ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_service.py b/services/web/server/src/simcore_service_webserver/projects/_trash_service.py index ca48c60d878a..dab8f3aafa42 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_service.py @@ -10,6 +10,7 @@ from models_library.rest_ordering import OrderBy, OrderDirection from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE from models_library.users import UserID +from models_library.workspaces import WorkspaceID from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from servicelib.utils import fire_and_forget_task @@ -241,7 +242,7 @@ async def batch_delete_trashed_projects_as_admin( ( page_params.total_number_of_items, expired_trashed_projects, - ) = await _projects_repository.list_trashed_projects( + ) = await _projects_repository.list_projects_db_get_as_admin( app, # both implicit and explicitly trashed trashed_before=trashed_before, @@ -273,3 +274,51 @@ async def batch_delete_trashed_projects_as_admin( ) return deleted_project_ids + + +async def batch_delete_projects_in_root_workspace_as_admin( + app: web.Application, + *, + workspace_id: WorkspaceID, + fail_fast: bool, +) -> list[ProjectID]: + """ + Deletes all projects in the workspace root. + + Raises: + ProjectsBatchDeleteError: If there are errors during the deletion process. + """ + deleted_project_ids: list[ProjectID] = [] + errors: list[tuple[ProjectID, Exception]] = [] + + for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + ( + page_params.total_number_of_items, + projects_for_deletion, + ) = await _projects_repository.list_projects_db_get_as_admin( + app, + shared_workspace_id=workspace_id, # <-- Workspace filter + offset=page_params.offset, + limit=page_params.limit, + order_by=OrderBy(field=IDStr("id")), + ) + # BATCH delete + for project in projects_for_deletion: + try: + await _projects_service_delete.delete_project_as_admin( + app, + project_uuid=project.uuid, + ) + deleted_project_ids.append(project.uuid) + except Exception as err: # pylint: disable=broad-exception-caught + if fail_fast: + raise + errors.append((project.uuid, err)) + + if errors: + raise ProjectsBatchDeleteError( + errors=errors, + deleted_project_ids=deleted_project_ids, + ) + + return deleted_project_ids diff --git a/services/web/server/src/simcore_service_webserver/projects/api.py b/services/web/server/src/simcore_service_webserver/projects/api.py index bde3efa68d9d..3c9c409f2adf 100644 --- a/services/web/server/src/simcore_service_webserver/projects/api.py +++ b/services/web/server/src/simcore_service_webserver/projects/api.py @@ -5,24 +5,20 @@ check_user_project_permission, has_user_project_access_rights, ) +from ._crud_api_read import list_projects from ._groups_service import ( create_project_group_without_checking_permissions, delete_project_group_without_checking_permissions, ) -from ._wallets_service import ( - check_project_financial_status, - connect_wallet_to_project, - get_project_wallet, -) +from ._projects_service import delete_project_by_user __all__: tuple[str, ...] = ( - "check_project_financial_status", "check_user_project_permission", - "connect_wallet_to_project", "create_project_group_without_checking_permissions", "delete_project_group_without_checking_permissions", - "get_project_wallet", "has_user_project_access_rights", + "list_projects", + "delete_project_by_user", ) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_wallets_service.py b/services/web/server/src/simcore_service_webserver/projects/projects_wallets_service.py new file mode 100644 index 000000000000..f96ccecfda2b --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/projects_wallets_service.py @@ -0,0 +1,14 @@ +from ._wallets_service import ( + check_project_financial_status, + connect_wallet_to_project, + get_project_wallet, +) + +__all__: tuple[str, ...] = ( + "check_project_financial_status", + "connect_wallet_to_project", + "get_project_wallet", +) + + +# nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/trash/_service.py b/services/web/server/src/simcore_service_webserver/trash/_service.py index 1f38043faf5e..502d4d2c1373 100644 --- a/services/web/server/src/simcore_service_webserver/trash/_service.py +++ b/services/web/server/src/simcore_service_webserver/trash/_service.py @@ -13,6 +13,7 @@ from ..folders import folders_trash_service from ..products import products_service from ..projects import projects_trash_service +from ..workspaces import workspaces_trash_service from .settings import get_plugin_settings _logger = logging.getLogger(__name__) @@ -100,6 +101,43 @@ async def _empty_explicitly_trashed_folders_and_content( ) +async def _empty_explicitely_trashed_workspaces_and_content( + app: web.Application, product_name: ProductName, user_id: UserID +): + trashed_workspaces_ids = await workspaces_trash_service.list_trashed_workspaces( + app=app, product_name=product_name, user_id=user_id + ) + + with log_context( + _logger, + logging.DEBUG, + "Deleting %s trashed workspaces (and all its content)", + len(trashed_workspaces_ids), + ): + for workspace_id in trashed_workspaces_ids: + try: + await workspaces_trash_service.delete_trashed_workspace( + app, + product_name=product_name, + user_id=user_id, + workspace_id=workspace_id, + ) + + except Exception as exc: # pylint: disable=broad-exception-caught + _logger.warning( + **create_troubleshotting_log_kwargs( + "Error deleting a trashed workspace (and content) while emptying trash.", + error=exc, + error_context={ + "workspace_id": workspace_id, + "product_name": product_name, + "user_id": user_id, + }, + tip=_TIP, + ) + ) + + async def safe_empty_trash( app: web.Application, *, @@ -115,6 +153,9 @@ async def safe_empty_trash( # Delete explicitly trashed folders (and all implicitly trashed sub-folders and projects) await _empty_explicitly_trashed_folders_and_content(app, product_name, user_id) + # Delete explicitly trashed workspaces (and all implicitly trashed sub-folders and projects) + await _empty_explicitely_trashed_workspaces_and_content(app, product_name, user_id) + async def safe_delete_expired_trash_as_admin(app: web.Application) -> None: settings = get_plugin_settings(app) @@ -123,24 +164,40 @@ async def safe_delete_expired_trash_as_admin(app: web.Application) -> None: app_products_names = await products_service.list_products_names(app) - for product_name in app_products_names: + with log_context( + _logger, + logging.DEBUG, + "Deleting items marked as trashed before %s [trashed_at < %s will be deleted]", + retention, + delete_until, + ): ctx = { "delete_until": delete_until, "retention": retention, - "product_name": product_name, } - with log_context( - _logger, - logging.DEBUG, - "Deleting items marked as trashed before %s in %s [trashed_at < %s will be deleted]", - retention, - product_name, - delete_until, - ): - try: + try: + deleted_workspace_ids = ( + await workspaces_trash_service.batch_delete_trashed_workspaces_as_admin( + app, + trashed_before=delete_until, + fail_fast=False, + ) + ) + _logger.info("Deleted %d trashed workspaces", len(deleted_workspace_ids)) + + except Exception as exc: # pylint: disable=broad-exception-caught + _logger.warning( + **create_troubleshotting_log_kwargs( + "Error batch deleting expired workspaces as admin.", + error=exc, + error_context=ctx, + ) + ) + for product_name in app_products_names: + try: await folders_trash_service.batch_delete_trashed_folders_as_admin( app, trashed_before=delete_until, @@ -149,31 +206,31 @@ async def safe_delete_expired_trash_as_admin(app: web.Application) -> None: ) except Exception as exc: # pylint: disable=broad-exception-caught + ctx_with_product = {**ctx, "product_name": product_name} _logger.warning( **create_troubleshotting_log_kwargs( "Error batch deleting expired trashed folders as admin.", error=exc, - error_context=ctx, + error_context=ctx_with_product, ) ) - try: - - deleted_project_ids = ( - await projects_trash_service.batch_delete_trashed_projects_as_admin( - app, - trashed_before=delete_until, - fail_fast=False, - ) + try: + deleted_project_ids = ( + await projects_trash_service.batch_delete_trashed_projects_as_admin( + app, + trashed_before=delete_until, + fail_fast=False, ) + ) - _logger.info("Deleted %d trashed projects", len(deleted_project_ids)) + _logger.info("Deleted %d trashed projects", len(deleted_project_ids)) - except Exception as exc: # pylint: disable=broad-exception-caught - _logger.warning( - **create_troubleshotting_log_kwargs( - "Error batch deleting expired projects as admin.", - error=exc, - error_context=ctx, - ) + except Exception as exc: # pylint: disable=broad-exception-caught + _logger.warning( + **create_troubleshotting_log_kwargs( + "Error batch deleting expired projects as admin.", + error=exc, + error_context=ctx, ) + ) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py index 37c84b3682ac..d2ac47751b70 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py @@ -12,7 +12,7 @@ from . import _groups_repository as workspaces_groups_db from . import _workspaces_repository as workspaces_workspaces_repository from ._groups_repository import WorkspaceGroupGetDB -from ._workspaces_service import check_user_workspace_access +from ._workspaces_service_crud_read import check_user_workspace_access from .errors import WorkspaceAccessForbiddenError log = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py index 41776bc57a66..e8e4bbdc89e7 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py @@ -11,7 +11,7 @@ from ..login.decorators import get_user_id, login_required from ..products import products_web from ..security.decorators import permission_required -from . import _trash_services +from . import _trash_service from ._common.exceptions_handlers import handle_plugin_requests_exceptions from ._common.models import WorkspacesPathParams, WorkspaceTrashQueryParams @@ -33,7 +33,7 @@ async def trash_workspace(request: web.Request): WorkspaceTrashQueryParams, request ) - await _trash_services.trash_workspace( + await _trash_service.trash_workspace( request.app, product_name=product_name, user_id=user_id, @@ -53,7 +53,7 @@ async def untrash_workspace(request: web.Request): product_name = products_web.get_product_name(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) - await _trash_services.untrash_workspace( + await _trash_service.untrash_workspace( request.app, product_name=product_name, user_id=user_id, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_service.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_service.py new file mode 100644 index 000000000000..0628977f4df6 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_service.py @@ -0,0 +1,388 @@ +import logging +from datetime import datetime + +import arrow +from aiohttp import web +from common_library.pagination_tools import iter_pagination_params +from models_library.basic_types import IDStr +from models_library.folders import FolderID +from models_library.products import ProductName +from models_library.projects import Project, ProjectID +from models_library.rest_ordering import OrderBy, OrderDirection +from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE +from models_library.users import UserID +from models_library.workspaces import ( + UserWorkspaceWithAccessRights, + WorkspaceID, + WorkspaceUpdates, +) +from simcore_postgres_database.utils_repos import transaction_context +from simcore_service_webserver.folders.service import list_folders +from simcore_service_webserver.projects.api import list_projects +from simcore_service_webserver.projects.models import ProjectTypeAPI + +from ..db.plugin import get_asyncpg_engine +from ..folders._trash_service import ( + batch_delete_folders_with_content_in_root_workspace_as_admin, + trash_folder, + untrash_folder, +) +from ..projects._trash_service import ( + batch_delete_projects_in_root_workspace_as_admin, + trash_project, + untrash_project, +) +from . import _workspaces_repository, _workspaces_service, _workspaces_service_crud_read +from .errors import WorkspaceBatchDeleteError, WorkspaceNotTrashedError + +_logger = logging.getLogger(__name__) + + +async def _check_exists_and_access( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + workspace_id: WorkspaceID, +): + await _workspaces_service_crud_read.check_user_workspace_access( + app=app, + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + permission="delete", + ) + + +async def _list_root_child_folders( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + workspace_id: WorkspaceID, +) -> list[FolderID]: + + child_folders: list[FolderID] = [] + for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + ( + folders, + page_params.total_number_of_items, + ) = await list_folders( + app, + user_id=user_id, + product_name=product_name, + folder_id=None, + workspace_id=workspace_id, + trashed=None, + offset=page_params.offset, + limit=page_params.limit, + order_by=OrderBy(field=IDStr("trashed"), direction=OrderDirection.ASC), + ) + + child_folders.extend([folder.folder_db.folder_id for folder in folders]) + + return child_folders + + +async def _list_root_child_projects( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + workspace_id: WorkspaceID, +) -> list[ProjectID]: + + child_projects: list[ProjectID] = [] + for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + ( + projects, + page_params.total_number_of_items, + ) = await list_projects( + app, + user_id=user_id, + product_name=product_name, + show_hidden=False, + workspace_id=workspace_id, + project_type=ProjectTypeAPI.all, + folder_id=None, + trashed=None, + offset=page_params.offset, + limit=page_params.limit, + order_by=OrderBy( + field=IDStr("last_change_date"), direction=OrderDirection.DESC + ), + ) + + child_projects.extend([Project(**project).uuid for project in projects]) + + return child_projects + + +async def trash_workspace( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + workspace_id: WorkspaceID, + force_stop_first: bool, +): + await _check_exists_and_access( + app, product_name=product_name, user_id=user_id, workspace_id=workspace_id + ) + + trashed_at = arrow.utcnow().datetime + + async with transaction_context(get_asyncpg_engine(app)) as connection: + # EXPLICIT trash + await _workspaces_repository.update_workspace( + app, + connection, + product_name=product_name, + workspace_id=workspace_id, + updates=WorkspaceUpdates(trashed=trashed_at, trashed_by=user_id), + ) + + # IMPLICIT trash + child_folders: list[FolderID] = await _list_root_child_folders( + app, + product_name=product_name, + user_id=user_id, + workspace_id=workspace_id, + ) + + for folder_id in child_folders: + await trash_folder( + app, + product_name=product_name, + user_id=user_id, + folder_id=folder_id, + force_stop_first=force_stop_first, + explicit=False, + ) + + child_projects: list[ProjectID] = await _list_root_child_projects( + app, + product_name=product_name, + user_id=user_id, + workspace_id=workspace_id, + ) + + for project_id in child_projects: + await trash_project( + app, + product_name=product_name, + user_id=user_id, + project_id=project_id, + force_stop_first=force_stop_first, + explicit=False, + ) + + +async def untrash_workspace( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + workspace_id: WorkspaceID, +): + await _check_exists_and_access( + app, product_name=product_name, user_id=user_id, workspace_id=workspace_id + ) + + async with transaction_context(get_asyncpg_engine(app)) as connection: + # EXPLICIT UNtrash + await _workspaces_repository.update_workspace( + app, + connection, + product_name=product_name, + workspace_id=workspace_id, + updates=WorkspaceUpdates(trashed=None, trashed_by=None), + ) + + child_folders: list[FolderID] = await _list_root_child_folders( + app, + product_name=product_name, + user_id=user_id, + workspace_id=workspace_id, + ) + + for folder_id in child_folders: + await untrash_folder( + app, + product_name=product_name, + user_id=user_id, + folder_id=folder_id, + ) + + child_projects: list[ProjectID] = await _list_root_child_projects( + app, + product_name=product_name, + user_id=user_id, + workspace_id=workspace_id, + ) + + for project_id in child_projects: + await untrash_project( + app, product_name=product_name, user_id=user_id, project_id=project_id + ) + + +def _can_delete( + workspace: UserWorkspaceWithAccessRights, + user_id: UserID, + until_equal_datetime: datetime | None, +) -> bool: + return bool( + workspace.trashed + and (until_equal_datetime is None or workspace.trashed < until_equal_datetime) + and workspace.my_access_rights.delete + and workspace.trashed_by == user_id + ) + + +async def delete_trashed_workspace( + app: web.Application, + *, + product_name: ProductName, + user_id: UserID, + workspace_id: WorkspaceID, + until_equal_datetime: datetime | None = None, +) -> None: + + workspace = await _workspaces_service_crud_read.get_workspace( + app, + user_id=user_id, + product_name=product_name, + workspace_id=workspace_id, + ) + + if not _can_delete( + workspace, + user_id=user_id, + until_equal_datetime=until_equal_datetime, + ): + raise WorkspaceNotTrashedError( + workspace_id=workspace_id, + user_id=user_id, + reason="Cannot delete trashed workspace since it does not fit current criteria", + ) + + # NOTE: this function deletes workspace AND its content recursively! + await _workspaces_service.delete_workspace_with_all_content( + app, + user_id=user_id, + product_name=product_name, + workspace_id=workspace_id, + ) + + +async def list_trashed_workspaces( + app: web.Application, + product_name: ProductName, + user_id: UserID, + until_equal_datetime: datetime | None = None, +) -> list[WorkspaceID]: + trashed_workspace_ids: list[WorkspaceID] = [] + + for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + ( + page_params.total_number_of_items, + workspaces, + ) = await _workspaces_service_crud_read.list_workspaces( + app, + user_id=user_id, + product_name=product_name, + filter_trashed=True, + filter_by_text=None, + offset=page_params.offset, + limit=page_params.limit, + order_by=OrderBy(field=IDStr("trashed"), direction=OrderDirection.ASC), + ) + + # NOTE: Applying POST-FILTERING + trashed_workspace_ids.extend( + [ + ws.workspace_id + for ws in workspaces + if _can_delete( + ws, + user_id=user_id, + until_equal_datetime=until_equal_datetime, + ) + ] + ) + + return trashed_workspace_ids + + +async def batch_delete_trashed_workspaces_as_admin( + app: web.Application, + *, + trashed_before: datetime, + fail_fast: bool, +) -> list[WorkspaceID]: + + deleted_workspace_ids: list[WorkspaceID] = [] + errors: list[tuple[WorkspaceID, Exception]] = [] + + for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + ( + page_params.total_number_of_items, + expired_trashed_workspaces, + ) = await _workspaces_repository.list_workspaces_db_get_as_admin( + app, + trashed_before=trashed_before, + offset=page_params.offset, + limit=page_params.limit, + order_by=OrderBy( + field=IDStr("workspace_id"), direction=OrderDirection.DESC + ), + ) + # BATCH delete + for trashed_workspace in expired_trashed_workspaces: + assert trashed_workspace.trashed # nosec + deleted_workspace_ids.append(trashed_workspace.workspace_id) + + workspace_db_get = await _workspaces_repository.get_workspace_db_get( + app, workspace_id=trashed_workspace.workspace_id + ) + + try: + await batch_delete_projects_in_root_workspace_as_admin( + app, workspace_id=trashed_workspace.workspace_id, fail_fast=False + ) + except Exception as err: # pylint: disable=broad-exception-caught + if fail_fast: + raise + errors.append((trashed_workspace.workspace_id, err)) + + try: + await batch_delete_folders_with_content_in_root_workspace_as_admin( + app, + workspace_id=trashed_workspace.workspace_id, + product_name=workspace_db_get.product_name, + fail_fast=False, + ) + except Exception as err: # pylint: disable=broad-exception-caught + if fail_fast: + raise + errors.append((trashed_workspace.workspace_id, err)) + + try: + await _workspaces_repository.delete_workspace( + app, + workspace_id=trashed_workspace.workspace_id, + product_name=workspace_db_get.product_name, + ) + except Exception as err: # pylint: disable=broad-exception-caught + if fail_fast: + raise + errors.append((trashed_workspace.workspace_id, err)) + + if errors: + raise WorkspaceBatchDeleteError( + errors=errors, + deleted_workspace_ids=deleted_workspace_ids, + ) + + return deleted_workspace_ids diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py deleted file mode 100644 index 611f3fb56e9e..000000000000 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py +++ /dev/null @@ -1,129 +0,0 @@ -import logging - -import arrow -from aiohttp import web -from models_library.folders import FolderID -from models_library.products import ProductName -from models_library.projects import ProjectID -from models_library.users import UserID -from models_library.workspaces import WorkspaceID, WorkspaceUpdates -from simcore_postgres_database.utils_repos import transaction_context - -from ..db.plugin import get_asyncpg_engine -from ..folders._trash_service import trash_folder, untrash_folder -from ..projects._trash_service import trash_project, untrash_project -from . import _workspaces_repository, _workspaces_service - -_logger = logging.getLogger(__name__) - - -async def _check_exists_and_access( - app: web.Application, - *, - product_name: ProductName, - user_id: UserID, - workspace_id: WorkspaceID, -): - await _workspaces_service.check_user_workspace_access( - app=app, - user_id=user_id, - workspace_id=workspace_id, - product_name=product_name, - permission="delete", - ) - - -async def trash_workspace( - app: web.Application, - *, - product_name: ProductName, - user_id: UserID, - workspace_id: WorkspaceID, - force_stop_first: bool, -): - await _check_exists_and_access( - app, product_name=product_name, user_id=user_id, workspace_id=workspace_id - ) - - trashed_at = arrow.utcnow().datetime - - async with transaction_context(get_asyncpg_engine(app)) as connection: - # EXPLICIT trash - await _workspaces_repository.update_workspace( - app, - connection, - product_name=product_name, - workspace_id=workspace_id, - updates=WorkspaceUpdates(trashed=trashed_at, trashed_by=user_id), - ) - - # IMPLICIT trash - child_folders: list[FolderID] = [ - # NOTE: follows up with https://github.com/ITISFoundation/osparc-simcore/issues/7034 - ] - - for folder_id in child_folders: - await trash_folder( - app, - product_name=product_name, - user_id=user_id, - folder_id=folder_id, - force_stop_first=force_stop_first, - ) - - child_projects: list[ProjectID] = [ - # NOTE: follows up with https://github.com/ITISFoundation/osparc-simcore/issues/7034 - ] - - for project_id in child_projects: - await trash_project( - app, - product_name=product_name, - user_id=user_id, - project_id=project_id, - force_stop_first=force_stop_first, - explicit=False, - ) - - -async def untrash_workspace( - app: web.Application, - *, - product_name: ProductName, - user_id: UserID, - workspace_id: WorkspaceID, -): - await _check_exists_and_access( - app, product_name=product_name, user_id=user_id, workspace_id=workspace_id - ) - - async with transaction_context(get_asyncpg_engine(app)) as connection: - # EXPLICIT UNtrash - await _workspaces_repository.update_workspace( - app, - connection, - product_name=product_name, - workspace_id=workspace_id, - updates=WorkspaceUpdates(trashed=None, trashed_by=None), - ) - - child_folders: list[FolderID] = [ - # NOTE: follows up with https://github.com/ITISFoundation/osparc-simcore/issues/7034 - ] - - for folder_id in child_folders: - await untrash_folder( - app, - product_name=product_name, - user_id=user_id, - folder_id=folder_id, - ) - - child_projects: list[ProjectID] = [ - # NOTE: follows up with https://github.com/ITISFoundation/osparc-simcore/issues/7034 - ] - - for project_id in child_projects: - await untrash_project( - app, product_name=product_name, user_id=user_id, project_id=project_id - ) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_models.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_models.py new file mode 100644 index 000000000000..f4b488e9b9bf --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_models.py @@ -0,0 +1,35 @@ +from datetime import datetime + +from models_library.groups import GroupID +from models_library.products import ProductName +from models_library.users import UserID +from models_library.workspaces import WorkspaceID +from pydantic import ( + BaseModel, + ConfigDict, + Field, +) + + +class WorkspaceDBGet(BaseModel): + workspace_id: WorkspaceID + name: str + description: str | None + owner_primary_gid: GroupID = Field( + ..., + description="GID of the group that owns this wallet", + ) + thumbnail: str | None + created: datetime = Field( + ..., + description="Timestamp on creation", + ) + modified: datetime = Field( + ..., + description="Timestamp of last modification", + ) + trashed: datetime | None + trashed_by: UserID | None + product_name: ProductName + + model_config = ConfigDict(from_attributes=True) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py index 63f014bb85d2..4e3744331503 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py @@ -1,7 +1,9 @@ import logging +from datetime import datetime from typing import cast from aiohttp import web +from common_library.exclude import UnSet, is_set from models_library.groups import GroupID from models_library.products import ProductName from models_library.rest_ordering import OrderBy, OrderDirection @@ -30,6 +32,7 @@ from sqlalchemy.sql import Select, select from ..db.plugin import get_asyncpg_engine +from ._workspaces_models import WorkspaceDBGet from .errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError _logger = logging.getLogger(__name__) @@ -45,6 +48,7 @@ workspaces.c.modified, workspaces.c.trashed, workspaces.c.trashed_by, + workspaces.c.product_name, ) @@ -197,6 +201,26 @@ async def get_workspace_for_user( return UserWorkspaceWithAccessRights.model_validate(row) +async def get_workspace_db_get( + app: web.Application, + connection: AsyncConnection | None = None, + *, + workspace_id: WorkspaceID, +) -> WorkspaceDBGet: + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + result = await conn.execute( + select( + *_WORKSPACE_SELECTION_COLS, + ) + .select_from(workspaces) + .where(workspaces.c.workspace_id == workspace_id) + ) + row = result.one_or_none() + if row is None: + raise WorkspaceNotFoundError(reason=f"Workspace {workspace_id} not found.") + return WorkspaceDBGet.model_validate(row) + + async def update_workspace( app: web.Application, connection: AsyncConnection | None = None, @@ -241,3 +265,56 @@ async def delete_workspace( & (workspaces.c.product_name == product_name) ) ) + + +assert set(WorkspaceDBGet.model_fields.keys()) == { + col.name for col in _WORKSPACE_SELECTION_COLS +} + + +async def list_workspaces_db_get_as_admin( + app: web.Application, + connection: AsyncConnection | None = None, + *, + # filter + trashed_before: datetime | UnSet = UnSet.VALUE, + # pagination + offset: NonNegativeInt, + limit: int, + # ordering + order_by: OrderBy, +) -> tuple[int, list[WorkspaceDBGet]]: + """ + NOTE: This is an internal function used for administrative purposes. + Ex. It lists trashed workspaces across the application for cleanup tasks. + """ + base_query = select(*_WORKSPACE_SELECTION_COLS).where( + workspaces.c.trashed.is_not(None) + ) + + if is_set(trashed_before): + assert isinstance(trashed_before, datetime) # nosec + base_query = base_query.where(workspaces.c.trashed < trashed_before) + + # Select total count from base_query + count_query = select(func.count()).select_from(base_query.subquery()) + + # Ordering and pagination + if order_by.direction == OrderDirection.ASC: + list_query = base_query.order_by( + asc(getattr(workspaces.c, order_by.field)), workspaces.c.workspace_id + ) + else: + list_query = base_query.order_by( + desc(getattr(workspaces.c, order_by.field)), workspaces.c.workspace_id + ) + list_query = list_query.offset(offset).limit(limit) + + async with pass_or_acquire_connection(get_asyncpg_engine(app), connection) as conn: + total_count = await conn.scalar(count_query) + + result = await conn.stream(list_query) + workspaces_list: list[WorkspaceDBGet] = [ + WorkspaceDBGet.model_validate(row) async for row in result + ] + return cast(int, total_count), workspaces_list diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py index eb40b338ef3c..79a6d74b0ede 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py @@ -23,7 +23,7 @@ from ..login.decorators import login_required from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _workspaces_service +from . import _workspaces_service, _workspaces_service_crud_read from ._common.exceptions_handlers import handle_plugin_requests_exceptions from ._common.models import ( WorkspacesFilters, @@ -76,7 +76,7 @@ async def list_workspaces(request: web.Request): query_params.filters = WorkspacesFilters() assert query_params.filters - total_count, workspaces = await _workspaces_service.list_workspaces( + total_count, workspaces = await _workspaces_service_crud_read.list_workspaces( app=request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, @@ -110,7 +110,7 @@ async def get_workspace(request: web.Request): req_ctx = WorkspacesRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) - workspace = await _workspaces_service.get_workspace( + workspace = await _workspaces_service_crud_read.get_workspace( app=request.app, workspace_id=path_params.workspace_id, user_id=req_ctx.user_id, @@ -153,7 +153,7 @@ async def delete_workspace(request: web.Request): req_ctx = WorkspacesRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) - await _workspaces_service.delete_workspace( + await _workspaces_service.delete_workspace_with_all_content( app=request.app, user_id=req_ctx.user_id, workspace_id=path_params.workspace_id, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py index e96e7937eb4d..30d02bfdb4fb 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py @@ -3,20 +3,26 @@ import logging from aiohttp import web +from common_library.pagination_tools import iter_pagination_params +from models_library.basic_types import IDStr +from models_library.folders import FolderID from models_library.products import ProductName -from models_library.rest_ordering import OrderBy +from models_library.projects import Project, ProjectID +from models_library.rest_ordering import OrderBy, OrderDirection +from models_library.rest_pagination import MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE from models_library.users import UserID from models_library.workspaces import ( UserWorkspaceWithAccessRights, WorkspaceID, WorkspaceUpdates, ) -from pydantic import NonNegativeInt -from ..projects._projects_repository_legacy_utils import PermissionStr +from ..folders.service import delete_folder_with_all_content, list_folders +from ..projects.api import delete_project_by_user, list_projects +from ..projects.models import ProjectTypeAPI from ..users.api import get_user from . import _workspaces_repository as db -from .errors import WorkspaceAccessForbiddenError +from ._workspaces_service_crud_read import check_user_workspace_access _logger = logging.getLogger(__name__) @@ -47,47 +53,6 @@ async def create_workspace( ) -async def get_workspace( - app: web.Application, - *, - user_id: UserID, - workspace_id: WorkspaceID, - product_name: ProductName, -) -> UserWorkspaceWithAccessRights: - return await get_user_workspace( - app=app, - user_id=user_id, - workspace_id=workspace_id, - product_name=product_name, - permission="read", - ) - - -async def list_workspaces( - app: web.Application, - *, - user_id: UserID, - product_name: ProductName, - filter_trashed: bool | None, - filter_by_text: str | None, - offset: NonNegativeInt, - limit: int, - order_by: OrderBy, -) -> tuple[int, list[UserWorkspaceWithAccessRights]]: - total_count, workspaces = await db.list_workspaces_for_user( - app, - user_id=user_id, - product_name=product_name, - filter_trashed=filter_trashed, - filter_by_text=filter_by_text, - offset=offset, - limit=limit, - order_by=order_by, - ) - - return total_count, workspaces - - async def update_workspace( app: web.Application, *, @@ -118,12 +83,12 @@ async def update_workspace( ) -async def delete_workspace( +async def delete_workspace_with_all_content( app: web.Application, *, + product_name: ProductName, user_id: UserID, workspace_id: WorkspaceID, - product_name: ProductName, ) -> None: await check_user_workspace_access( app=app, @@ -133,67 +98,69 @@ async def delete_workspace( permission="delete", ) - await db.delete_workspace( - app, - workspace_id=workspace_id, - product_name=product_name, - ) + # Get all root projects + for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + ( + projects, + page_params.total_number_of_items, + ) = await list_projects( + app, + user_id=user_id, + product_name=product_name, + show_hidden=False, + workspace_id=workspace_id, + project_type=ProjectTypeAPI.all, + folder_id=None, + trashed=None, + offset=page_params.offset, + limit=page_params.limit, + order_by=OrderBy( + field=IDStr("last_change_date"), direction=OrderDirection.DESC + ), + ) + workspace_root_projects: list[ProjectID] = [ + Project(**project).uuid for project in projects + ] -async def get_user_workspace( - app: web.Application, - *, - user_id: UserID, - workspace_id: WorkspaceID, - product_name: ProductName, - permission: PermissionStr | None, -) -> UserWorkspaceWithAccessRights: - """ + # Delete projects properly + for project_uuid in workspace_root_projects: + await delete_project_by_user( + app, project_uuid=project_uuid, user_id=user_id + ) - Here checking access is optional. A use case is when the caller has guarantees that - `user_id` has granted access and we do not want to re-check + # Get all root folders + for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + ( + folders, + page_params.total_number_of_items, + ) = await list_folders( + app, + user_id=user_id, + product_name=product_name, + workspace_id=workspace_id, + folder_id=None, + trashed=None, + offset=page_params.offset, + limit=page_params.limit, + order_by=OrderBy(field=IDStr("folder_id"), direction=OrderDirection.ASC), + ) - Raises: - WorkspaceAccessForbiddenError: if permission not None and user_id does not have access - """ - workspace: UserWorkspaceWithAccessRights = await db.get_workspace_for_user( - app=app, user_id=user_id, workspace_id=workspace_id, product_name=product_name - ) + workspace_root_folders: list[FolderID] = [ + folder.folder_db.folder_id for folder in folders + ] - # NOTE: check here is optional - if permission is not None: - has_user_granted_permission = getattr( - workspace.my_access_rights, permission, False - ) - if not has_user_granted_permission: - raise WorkspaceAccessForbiddenError( + # Delete folders properly + for folder_id in workspace_root_folders: + await delete_folder_with_all_content( + app, user_id=user_id, - workspace_id=workspace_id, product_name=product_name, - permission_checked=permission, + folder_id=folder_id, ) - return workspace - -async def check_user_workspace_access( - app: web.Application, - *, - user_id: UserID, - workspace_id: WorkspaceID, - product_name: ProductName, - permission: PermissionStr, -) -> UserWorkspaceWithAccessRights: - """ - As `get_user_workspace` but here access check is required - - Raises: - WorkspaceAccessForbiddenError - """ - return await get_user_workspace( + await db.delete_workspace( app, - user_id=user_id, workspace_id=workspace_id, product_name=product_name, - # NOTE: check here is required - permission=permission, ) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service_crud_read.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service_crud_read.py new file mode 100644 index 000000000000..f0c570597283 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service_crud_read.py @@ -0,0 +1,119 @@ +# pylint: disable=unused-argument + +import logging + +from aiohttp import web +from models_library.products import ProductName +from models_library.rest_ordering import OrderBy +from models_library.users import UserID +from models_library.workspaces import ( + UserWorkspaceWithAccessRights, + WorkspaceID, +) +from pydantic import NonNegativeInt + +from ..projects._projects_repository_legacy_utils import PermissionStr +from . import _workspaces_repository as db +from .errors import WorkspaceAccessForbiddenError + +_logger = logging.getLogger(__name__) + + +async def get_workspace( + app: web.Application, + *, + user_id: UserID, + workspace_id: WorkspaceID, + product_name: ProductName, +) -> UserWorkspaceWithAccessRights: + return await get_user_workspace( + app=app, + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + permission="read", + ) + + +async def list_workspaces( + app: web.Application, + *, + user_id: UserID, + product_name: ProductName, + filter_trashed: bool | None, + filter_by_text: str | None, + offset: NonNegativeInt, + limit: int, + order_by: OrderBy, +) -> tuple[int, list[UserWorkspaceWithAccessRights]]: + total_count, workspaces = await db.list_workspaces_for_user( + app, + user_id=user_id, + product_name=product_name, + filter_trashed=filter_trashed, + filter_by_text=filter_by_text, + offset=offset, + limit=limit, + order_by=order_by, + ) + + return total_count, workspaces + + +async def get_user_workspace( + app: web.Application, + *, + user_id: UserID, + workspace_id: WorkspaceID, + product_name: ProductName, + permission: PermissionStr | None, +) -> UserWorkspaceWithAccessRights: + """ + + Here checking access is optional. A use case is when the caller has guarantees that + `user_id` has granted access and we do not want to re-check + + Raises: + WorkspaceAccessForbiddenError: if permission not None and user_id does not have access + """ + workspace: UserWorkspaceWithAccessRights = await db.get_workspace_for_user( + app=app, user_id=user_id, workspace_id=workspace_id, product_name=product_name + ) + + # NOTE: check here is optional + if permission is not None: + has_user_granted_permission = getattr( + workspace.my_access_rights, permission, False + ) + if not has_user_granted_permission: + raise WorkspaceAccessForbiddenError( + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + permission_checked=permission, + ) + return workspace + + +async def check_user_workspace_access( + app: web.Application, + *, + user_id: UserID, + workspace_id: WorkspaceID, + product_name: ProductName, + permission: PermissionStr, +) -> UserWorkspaceWithAccessRights: + """ + As `get_user_workspace` but here access check is required + + Raises: + WorkspaceAccessForbiddenError + """ + return await get_user_workspace( + app, + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + # NOTE: check here is required + permission=permission, + ) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/api.py b/services/web/server/src/simcore_service_webserver/workspaces/api.py index 117d8d6d43d4..7667400c1923 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/api.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/api.py @@ -1,6 +1,6 @@ # mypy: disable-error-code=truthy-function -from ._workspaces_service import ( +from ._workspaces_service_crud_read import ( check_user_workspace_access, get_user_workspace, get_workspace, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/errors.py b/services/web/server/src/simcore_service_webserver/workspaces/errors.py index 9dff56fb87c2..66991deaaef9 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/errors.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/errors.py @@ -1,8 +1,10 @@ from ..errors import WebServerBaseError -class WorkspacesValueError(WebServerBaseError, ValueError): - ... +class WorkspacesValueError(WebServerBaseError, ValueError): ... + + +class WorkspacesRuntimeError(WebServerBaseError, RuntimeError): ... class WorkspaceNotFoundError(WorkspacesValueError): @@ -13,6 +15,10 @@ class WorkspaceAccessForbiddenError(WorkspacesValueError): msg_template = "Workspace access forbidden. {reason}" +class WorkspaceBatchDeleteError(WorkspacesValueError): + msg_template = "One or more workspaces could not be deleted: {errors}" + + # Workspace groups @@ -22,3 +28,7 @@ class WorkspaceGroupNotFoundError(WorkspacesValueError): class WorkspaceFolderInconsistencyError(WorkspacesValueError): msg_template = "Folder {folder_id} does not exists in the workspace {workspace_id}" + + +class WorkspaceNotTrashedError(WorkspacesRuntimeError): + msg_template = "Cannot delete workspace {workspace_id} since it was not trashed first: {reason}" diff --git a/services/web/server/src/simcore_service_webserver/workspaces/workspaces_trash_service.py b/services/web/server/src/simcore_service_webserver/workspaces/workspaces_trash_service.py new file mode 100644 index 000000000000..ecf550947f4b --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/workspaces/workspaces_trash_service.py @@ -0,0 +1,13 @@ +from ._trash_service import ( + batch_delete_trashed_workspaces_as_admin, + delete_trashed_workspace, + list_trashed_workspaces, +) + +__all__: tuple[str, ...] = ( + "delete_trashed_workspace", + "list_trashed_workspaces", + "batch_delete_trashed_workspaces_as_admin", +) + +# nopycln: file diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_repository.py b/services/web/server/tests/unit/with_dbs/02/test_projects_repository.py index 6dccd486a0be..378e4c9c0d34 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_repository.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_repository.py @@ -10,6 +10,8 @@ import pytest from aiohttp.test_utils import TestClient from common_library.users_enums import UserRole +from models_library.basic_types import IDStr +from models_library.rest_ordering import OrderBy, OrderDirection from pytest_simcore.helpers.webserver_login import UserInfoDict from simcore_service_webserver.projects import ( _projects_repository as projects_service_repository, @@ -137,10 +139,11 @@ async def test_list_trashed_projects(client: TestClient, trashed_project: Projec ( total_count, trashed_projects, - ) = await projects_service_repository.list_trashed_projects( + ) = await projects_service_repository.list_projects_db_get_as_admin( client.app, trashed_explicitly=True, trashed_before=arrow.utcnow().datetime + timedelta(days=1), + order_by=OrderBy(field=IDStr("trashed"), direction=OrderDirection.ASC), ) assert total_count == 1 diff --git a/services/web/server/tests/unit/with_dbs/03/trash/test_trash.py b/services/web/server/tests/unit/with_dbs/03/trash/test_trash_rest.py similarity index 77% rename from services/web/server/tests/unit/with_dbs/03/trash/test_trash.py rename to services/web/server/tests/unit/with_dbs/03/trash/test_trash_rest.py index 396dfcc7c977..b3f66e0813f9 100644 --- a/services/web/server/tests/unit/with_dbs/03/trash/test_trash.py +++ b/services/web/server/tests/unit/with_dbs/03/trash/test_trash_rest.py @@ -8,11 +8,13 @@ import asyncio from collections.abc import AsyncIterable +from copy import deepcopy from unittest.mock import MagicMock from uuid import UUID import arrow import pytest +import sqlalchemy as sa from aiohttp.test_utils import TestClient from models_library.api_schemas_webserver.folders_v2 import FolderGet from models_library.api_schemas_webserver.projects import ProjectGet, ProjectListItem @@ -21,7 +23,10 @@ from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict +from pytest_simcore.helpers.webserver_projects import create_project from servicelib.aiohttp import status +from simcore_postgres_database.models.folders_v2 import folders_v2 +from simcore_postgres_database.models.projects import projects from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects._groups_service import ProjectGroupGet from simcore_service_webserver.projects.models import ProjectDict @@ -489,7 +494,102 @@ async def workspace( "https://github.com/ITISFoundation/osparc-simcore/pull/6690" ) async def test_trash_empty_workspace( - client: TestClient, logged_user: UserInfoDict, workspace: WorkspaceGet + client: TestClient, + logged_user: UserInfoDict, + workspace: WorkspaceGet, + mocked_catalog: None, + mocked_dynamic_services_interface: dict[str, MagicMock], +): + assert client.app + + assert workspace.trashed_at is None + assert workspace.trashed_by is None + + # LIST NOT trashed (default) + resp = await client.get("/v0/workspaces") + await assert_status(resp, status.HTTP_200_OK) + + page = Page[WorkspaceGet].model_validate(await resp.json()) + assert page.meta.total == 1 + assert page.data[0] == workspace + + # LIST trashed + resp = await client.get("/v0/workspaces", params={"filters": '{"trashed": true}'}) + await assert_status(resp, status.HTTP_200_OK) + + page = Page[WorkspaceGet].model_validate(await resp.json()) + assert page.meta.total == 0 + + # ------------- + + _exclude_attrs = {"trashed_by", "trashed_at", "modified_at"} + + # TRASH + trashing_at = arrow.utcnow().datetime + resp = await client.post( + f"/v0/workspaces/{workspace.workspace_id}:trash" # <-- TESTING TRASHING + ) + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # LIST NOT trashed (default) + resp = await client.get("/v0/workspaces") + await assert_status(resp, status.HTTP_200_OK) + + page = Page[WorkspaceGet].model_validate(await resp.json()) + assert page.meta.total == 0 + + # LIST trashed + resp = await client.get("/v0/workspaces", params={"filters": '{"trashed": true}'}) + await assert_status(resp, status.HTTP_200_OK) + + page = Page[WorkspaceGet].model_validate(await resp.json()) + assert page.meta.total == 1 + assert page.data[0].model_dump(exclude=_exclude_attrs) == workspace.model_dump( + exclude=_exclude_attrs + ) + assert page.data[0].trashed_at is not None + assert trashing_at < page.data[0].trashed_at + assert page.data[0].trashed_by == logged_user["primary_gid"] + + # -------- + + # UN_TRASH + resp = await client.post(f"/v0/workspaces/{workspace.workspace_id}:untrash") + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # LIST NOT trashed (default) + resp = await client.get("/v0/workspaces") + await assert_status(resp, status.HTTP_200_OK) + + page = Page[WorkspaceGet].model_validate(await resp.json()) + assert page.meta.total == 1 + assert page.data[0].model_dump(exclude=_exclude_attrs) == workspace.model_dump( + exclude=_exclude_attrs + ) + + assert page.data[0].trashed_at is None + assert page.data[0].trashed_by is None + + # LIST trashed + resp = await client.get("/v0/workspaces", params={"filters": '{"trashed": true}'}) + await assert_status(resp, status.HTTP_200_OK) + + page = Page[WorkspaceGet].model_validate(await resp.json()) + assert page.meta.total == 0 + + +@pytest.mark.acceptance_test( + "https://github.com/ITISFoundation/osparc-simcore/issues/7034" +) +async def test_trash_workspace( + client: TestClient, + logged_user: UserInfoDict, + workspace: WorkspaceGet, + user_project: ProjectDict, + fake_project: ProjectDict, + mocked_catalog: None, + mocked_dynamic_services_interface: dict[str, MagicMock], + postgres_db: sa.engine.Engine, ): assert client.app @@ -511,13 +611,129 @@ async def test_trash_empty_workspace( page = Page[WorkspaceGet].model_validate(await resp.json()) assert page.meta.total == 0 + # ------------- + # add folders and projects to the workspace + + # CREATE a project **in workspace** + project_data = deepcopy(fake_project) + project_data["workspace_id"] = f"{workspace.workspace_id}" + created_project = await create_project( + client.app, + project_data, + user_id=logged_user["id"], + product_name="osparc", + ) + + # CREATE a folder in workspace + resp = await client.post( + "/v0/folders", + json={ + "name": "Folder", + # "workspaceId": f"{workspace.workspace_id}", + }, + ) + data, _ = await assert_status(resp, status.HTTP_201_CREATED) + folder = FolderGet.model_validate(data) + + # CREATE a SUB-folder + resp = await client.post( + "/v0/folders", + json={ + "name": "SubFolder1", + "parentFolderId": folder.folder_id, + # "workspaceId": f"{workspace.workspace_id}", + }, + ) + data, _ = await assert_status(resp, status.HTTP_201_CREATED) + subfolder = FolderGet.model_validate(data) + + # MOVE project to SUB-folder + project_uuid = UUID(user_project["uuid"]) + resp = await client.put( + f"/v0/projects/{project_uuid}/folders/{subfolder.folder_id}" + ) + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # MOVE root folder with content to workspace + url = client.app.router["move_folder_to_workspace"].url_for( + folder_id=f"{folder.folder_id}", + workspace_id=f"{workspace.workspace_id}", + ) + resp = await client.post(f"{url}") + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # ------------- + # list folders and projects in workspace + + # LIST projects in workspace + url = ( + client.app.router["list_projects"] + .url_for() + .with_query({"workspace_id": f"{workspace.workspace_id}"}) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + + # LIST folders in workspace + url = ( + client.app.router["list_folders"] + .url_for() + .with_query({"workspace_id": f"{workspace.workspace_id}", "folder_id": "null"}) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + + # LIST FOLDERS in workspace + url = ( + client.app.router["list_folders"] + .url_for() + .with_query({"workspace_id": f"{workspace.workspace_id}", "folder_id": "null"}) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + + # LIST FOLDERS in subfolder of workspace + url = ( + client.app.router["list_folders"] + .url_for() + .with_query( + { + "workspace_id": f"{workspace.workspace_id}", + "folder_id": f"{folder.folder_id}", + } + ) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + + # LIST PROJECTS in subfolder of workspace + url = ( + client.app.router["list_projects"] + .url_for() + .with_query( + { + "workspace_id": f"{workspace.workspace_id}", + "folder_id": f"{subfolder.folder_id}", + } + ) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 1 + # ------------- _exclude_attrs = {"trashed_by", "trashed_at", "modified_at"} # TRASH trashing_at = arrow.utcnow().datetime - resp = await client.post(f"/v0/workspaces/{workspace.workspace_id}:trash") + resp = await client.post( + f"/v0/workspaces/{workspace.workspace_id}:trash" # <-- TESTING TRASHING + ) await assert_status(resp, status.HTTP_204_NO_CONTENT) # LIST NOT trashed (default) @@ -540,6 +756,36 @@ async def test_trash_empty_workspace( assert trashing_at < page.data[0].trashed_at assert page.data[0].trashed_by == logged_user["primary_gid"] + # Check additional state in the DB + with postgres_db.connect() as conn: + # 1. Check that both projects were marked as trashed implicitly + trashed_projects = conn.execute( + projects.select().where( + projects.c.workspace_id == workspace.workspace_id, + projects.c.trashed.isnot(None), + ) + ) + trashed_projects = trashed_projects.fetchall() + assert len(trashed_projects) == 2 # Assuming two projects are expected + for project in trashed_projects: + assert project.trashed is not None + assert project.trashed_by == logged_user["id"] + assert project.trashed_explicitly is False + + # 2. Check that both folders were marked as trashed implicitly + trashed_folders = conn.execute( + folders_v2.select().where( + folders_v2.c.workspace_id == workspace.workspace_id, + folders_v2.c.trashed.isnot(None), + ) + ) + trashed_folders = trashed_folders.fetchall() + assert len(trashed_folders) == 2 # Assuming two folders are expected + for folder in trashed_folders: + assert folder.trashed is not None + assert folder.trashed_by == logged_user["id"] + assert folder.trashed_explicitly is False + # -------- # UN_TRASH @@ -566,6 +812,32 @@ async def test_trash_empty_workspace( page = Page[WorkspaceGet].model_validate(await resp.json()) assert page.meta.total == 0 + # Check additional state in the DB + with postgres_db.connect() as conn: + # 1. Check that both projects were marked as trashed implicitly + trashed_projects = conn.execute( + projects.select().where(projects.c.workspace_id == workspace.workspace_id) + ) + trashed_projects = trashed_projects.fetchall() + assert len(trashed_projects) == 2 # Assuming two projects are expected + for project in trashed_projects: + assert project.trashed is None + assert project.trashed_by is None + assert project.trashed_explicitly is False + + # 2. Check that both folders were marked as trashed implicitly + trashed_folders = conn.execute( + folders_v2.select().where( + folders_v2.c.workspace_id == workspace.workspace_id + ) + ) + trashed_folders = trashed_folders.fetchall() + assert len(trashed_folders) == 2 # Assuming two folders are expected + for folder in trashed_folders: + assert folder.trashed is None + assert folder.trashed_by is None + assert folder.trashed_explicitly is False + async def test_trash_subfolder( client: TestClient, diff --git a/services/web/server/tests/unit/with_dbs/03/trash/test_trash_service.py b/services/web/server/tests/unit/with_dbs/03/trash/test_trash_service.py index a58f32f6e3d6..b55f824cfa2c 100644 --- a/services/web/server/tests/unit/with_dbs/03/trash/test_trash_service.py +++ b/services/web/server/tests/unit/with_dbs/03/trash/test_trash_service.py @@ -87,7 +87,6 @@ async def test_trash_service__delete_expired_trash( # UNDER TEST: Run delete_expired_trash await trash_service.safe_delete_expired_trash_as_admin(client.app) - # ASSERT: logged_user tries to get the project and expects 404 resp = await client.get(f"/v0/projects/{user_project_id}") await assert_status(resp, status.HTTP_404_NOT_FOUND) @@ -98,7 +97,7 @@ async def test_trash_service__delete_expired_trash( await assert_status(resp, status.HTTP_404_NOT_FOUND) -async def test_trash_nested_folders_and_projects( +async def test_trash_service__delete_expired_trash_for_nested_folders_and_projects( client: TestClient, logged_user: UserInfoDict, user_project: ProjectDict, @@ -191,3 +190,134 @@ async def test_trash_nested_folders_and_projects( resp = await client.get(f"/v0/projects/{other_user_project['uuid']}") await assert_status(resp, status.HTTP_404_NOT_FOUND) + + +async def test_trash_service__delete_expired_trash_for_workspace( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + other_user: UserInfoDict, + other_user_project: ProjectDict, + mocked_catalog: None, + mocked_director_v2: None, + mocked_dynamic_services_interface: dict[str, MagicMock], +): + assert client.app + assert logged_user["id"] != other_user["id"] + + async with switch_client_session_to(client, logged_user): + # CREATE folders hierarchy for logged_user + resp = await client.post("/v0/folders", json={"name": "Root Folder"}) + data, _ = await assert_status(resp, status.HTTP_201_CREATED) + logged_user_root_folder = data + + resp = await client.post( + "/v0/folders", + json={ + "name": "Sub Folder", + "parentFolderId": logged_user_root_folder["folderId"], + }, + ) + data, _ = await assert_status(resp, status.HTTP_201_CREATED) + logged_user_sub_folder = data + + # MOVE project to subfolder + resp = await client.put( + f"/v0/projects/{user_project['uuid']}/folders/{logged_user_sub_folder['folderId']}" + ) + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # CREATE workspace + resp = await client.post("/v0/workspaces", json={"name": "My Workspace"}) + data, _ = await assert_status(resp, status.HTTP_201_CREATED) + logged_user_workspace = data + + # MOVE root folder with content to workspace + url = client.app.router["move_folder_to_workspace"].url_for( + folder_id=f"{logged_user_root_folder['folderId']}", + workspace_id=f"{logged_user_workspace['workspaceId']}", + ) + resp = await client.post(f"{url}") + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # TRASH workspace + resp = await client.post( + f"/v0/workspaces/{logged_user_workspace['workspaceId']}:trash" + ) + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + async with switch_client_session_to(client, other_user): + # CREATE folders hierarchy for other_user + resp = await client.post("/v0/folders", json={"name": "Root Folder"}) + data, _ = await assert_status(resp, status.HTTP_201_CREATED) + other_user_root_folder = data + + resp = await client.post( + "/v0/folders", + json={ + "name": "Sub Folder (other)", + "parentFolderId": other_user_root_folder["folderId"], + }, + ) + data, _ = await assert_status(resp, status.HTTP_201_CREATED) + other_user_sub_folder = data + + # MOVE project to subfolder + resp = await client.put( + f"/v0/projects/{other_user_project['uuid']}/folders/{other_user_sub_folder['folderId']}" + ) + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # CREATE workspace + resp = await client.post( + "/v0/workspaces", json={"name": "Other User Workspace"} + ) + data, _ = await assert_status(resp, status.HTTP_201_CREATED) + other_user_workspace = data + + # MOVE Folder to workspace + url = client.app.router["move_folder_to_workspace"].url_for( + folder_id=f"{other_user_root_folder['folderId']}", + workspace_id=f"{other_user_workspace['workspaceId']}", + ) + resp = await client.post(f"{url}") + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # TRASH workspace + resp = await client.post( + f"/v0/workspaces/{other_user_workspace['workspaceId']}:trash" + ) + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # UNDER TEST + await trash_service.safe_delete_expired_trash_as_admin(client.app) + + async with switch_client_session_to(client, logged_user): + # Verify logged_user's resources are gone + resp = await client.get( + f"/v0/workspaces/{logged_user_workspace['workspaceId']}" + ) + await assert_status(resp, status.HTTP_403_FORBIDDEN) + + resp = await client.get(f"/v0/folders/{logged_user_root_folder['folderId']}") + await assert_status(resp, status.HTTP_403_FORBIDDEN) + + resp = await client.get(f"/v0/folders/{logged_user_sub_folder['folderId']}") + await assert_status(resp, status.HTTP_403_FORBIDDEN) + + resp = await client.get(f"/v0/projects/{user_project['uuid']}") + await assert_status(resp, status.HTTP_404_NOT_FOUND) + + # Verify other_user's resources are gone + async with switch_client_session_to(client, other_user): + resp = await client.get(f"/v0/workspaces/{other_user_workspace['workspaceId']}") + await assert_status(resp, status.HTTP_403_FORBIDDEN) + + resp = await client.get(f"/v0/folders/{other_user_root_folder['folderId']}") + await assert_status(resp, status.HTTP_403_FORBIDDEN) + + resp = await client.get(f"/v0/folders/{other_user_sub_folder['folderId']}") + await assert_status(resp, status.HTTP_403_FORBIDDEN) + + resp = await client.get(f"/v0/projects/{other_user_project['uuid']}") + await assert_status(resp, status.HTTP_404_NOT_FOUND) diff --git a/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py b/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py index 1ba0859bab31..f7acc51f42d3 100644 --- a/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py +++ b/services/web/server/tests/unit/with_dbs/04/folders/test_folders.py @@ -5,7 +5,6 @@ # pylint: disable=too-many-statements -import asyncio from http import HTTPStatus from unittest import mock @@ -21,7 +20,6 @@ standard_role_response, ) from servicelib.aiohttp import status -from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects._groups_repository import ( GroupID, @@ -495,15 +493,6 @@ async def test_folders_deletion( resp = await client.delete(f"{url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) - fire_and_forget_task: asyncio.Task = next( - iter(client.app[APP_FIRE_AND_FORGET_TASKS_KEY]) - ) - assert fire_and_forget_task.get_name().startswith( - "fire_and_forget_task_delete_project_task_" - ) - await fire_and_forget_task - assert len(client.app[APP_FIRE_AND_FORGET_TASKS_KEY]) == 0 - # list root projects (The project should have been deleted) url = client.app.router["list_projects"].url_for() resp = await client.get(f"{url}") diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py b/services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py index fa008269aaff..2f2e06af8fb1 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/conftest.py @@ -5,6 +5,7 @@ import pytest import sqlalchemy as sa +from pytest_mock import MockerFixture from simcore_postgres_database.models.projects import projects from simcore_postgres_database.models.workspaces import workspaces @@ -15,3 +16,17 @@ def workspaces_clean_db(postgres_db: sa.engine.Engine) -> Iterator[None]: yield con.execute(workspaces.delete()) con.execute(projects.delete()) + + +@pytest.fixture +def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): + mocker.patch( + "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", + spec=True, + return_value=[], + ) + mocker.patch( + "simcore_service_webserver.projects._controller.projects_rest.project_uses_available_services", + spec=True, + return_value=True, + ) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py index aa3d8bb367e4..88b684ddb562 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py @@ -11,6 +11,7 @@ from aiohttp.test_utils import TestClient from models_library.api_schemas_webserver.workspaces import WorkspaceGet from models_library.rest_ordering import OrderDirection +from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_parametrizations import ( @@ -63,6 +64,7 @@ async def test_workspaces_workflow( logged_user: UserInfoDict, user_project: ProjectDict, expected: HTTPStatus, + mock_catalog_api_get_services_for_user_in_product: MockerFixture, workspaces_clean_db: AsyncIterator[None], ): assert client.app diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__delete_workspace_with_content.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__delete_workspace_with_content.py new file mode 100644 index 000000000000..35f98a79f854 --- /dev/null +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__delete_workspace_with_content.py @@ -0,0 +1,171 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments +# pylint: disable=too-many-statements + + +from copy import deepcopy +from http import HTTPStatus +from unittest import mock + +import pytest +from aiohttp.test_utils import TestClient +from models_library.api_schemas_webserver.workspaces import WorkspaceGet +from pytest_mock import MockerFixture +from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.webserver_login import UserInfoDict +from pytest_simcore.helpers.webserver_projects import create_project +from servicelib.aiohttp import status +from simcore_service_webserver.db.models import UserRole +from simcore_service_webserver.projects.models import ProjectDict + + +@pytest.fixture +def mock_storage_delete_data_folders(mocker: MockerFixture) -> mock.Mock: + mocker.patch( + "simcore_service_webserver.dynamic_scheduler.api.list_dynamic_services", + autospec=True, + ) + mocker.patch( + "simcore_service_webserver.projects._projects_service.remove_project_dynamic_services", + autospec=True, + ) + mocker.patch( + "simcore_service_webserver.projects._crud_api_delete.director_v2_service.delete_pipeline", + autospec=True, + ) + return mocker.patch( + "simcore_service_webserver.projects._crud_api_delete.delete_data_folders_of_project", + return_value=None, + ) + + +@pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) +async def test_workspaces_full_workflow_deletion( + client: TestClient, + logged_user: UserInfoDict, + user_project: ProjectDict, + expected: HTTPStatus, + mock_catalog_api_get_services_for_user_in_product: MockerFixture, + fake_project: ProjectDict, + workspaces_clean_db: None, + mock_storage_delete_data_folders: mock.Mock, +): + assert client.app + + # create a new workspace + url = client.app.router["create_workspace"].url_for() + resp = await client.post( + f"{url}", + json={ + "name": "My first workspace", + "description": "Custom description", + "thumbnail": None, + }, + ) + data, _ = await assert_status(resp, status.HTTP_201_CREATED) + added_workspace = WorkspaceGet.model_validate(data) + + # Create project in workspace + project_data = deepcopy(fake_project) + project_data["workspace_id"] = f"{added_workspace.workspace_id}" + first_project = await create_project( + client.app, + project_data, + user_id=logged_user["id"], + product_name="osparc", + ) + second_project = await create_project( + client.app, + project_data, + user_id=logged_user["id"], + product_name="osparc", + ) + third_project = await create_project( + client.app, + project_data, + user_id=logged_user["id"], + product_name="osparc", + ) + + # List project in workspace + url = ( + client.app.router["list_projects"] + .url_for() + .with_query({"workspace_id": f"{added_workspace.workspace_id}"}) + ) + resp = await client.get(f"{url}") + data, _ = await assert_status(resp, status.HTTP_200_OK) + assert len(data) == 3 + + # Create folder in workspace + url = client.app.router["create_folder"].url_for() + resp = await client.post( + f"{url}", + json={ + "name": "Original user folder", + "workspaceId": f"{added_workspace.workspace_id}", + }, + ) + first_folder, _ = await assert_status(resp, status.HTTP_201_CREATED) + + # Create sub folder of previous folder + url = client.app.router["create_folder"].url_for() + resp = await client.post( + f"{url}", + json={ + "name": "Second user folder", + "workspaceId": f"{added_workspace.workspace_id}", + "parentFolderId": f"{first_folder['folderId']}", + }, + ) + second_folder, _ = await assert_status(resp, status.HTTP_201_CREATED) + + # Move first project in specific folder in workspace + url = client.app.router["replace_project_folder"].url_for( + folder_id=f"{first_folder['folderId']}", + project_id=f"{first_project['uuid']}", + ) + resp = await client.put(f"{url}") + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # Move second project in specific folder in workspace + url = client.app.router["replace_project_folder"].url_for( + folder_id=f"{second_folder['folderId']}", + project_id=f"{second_project['uuid']}", + ) + resp = await client.put(f"{url}") + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # --------------------- + # TESTING DELETION + # --------------------- + + # Delete workspace + url = client.app.router["delete_workspace"].url_for( + workspace_id=f"{added_workspace.workspace_id}" + ) + resp = await client.delete(f"{url}") + await assert_status(resp, status.HTTP_204_NO_CONTENT) + + # --------------------- + # Assertions + + resp = await client.get(f"/v0/workspaces/{added_workspace.workspace_id}") + await assert_status(resp, status.HTTP_403_FORBIDDEN) + + resp = await client.get(f"/v0/folders/{first_folder['folderId']}") + await assert_status(resp, status.HTTP_403_FORBIDDEN) + + resp = await client.get(f"/v0/folders/{second_folder['folderId']}") + await assert_status(resp, status.HTTP_403_FORBIDDEN) + + resp = await client.get(f"/v0/projects/{first_project['uuid']}") + await assert_status(resp, status.HTTP_404_NOT_FOUND) + + resp = await client.get(f"/v0/projects/{second_project['uuid']}") + await assert_status(resp, status.HTTP_404_NOT_FOUND) + + resp = await client.get(f"/v0/projects/{third_project['uuid']}") + await assert_status(resp, status.HTTP_404_NOT_FOUND) diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py index ac519fba5071..b536ef5bf01b 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py @@ -5,7 +5,6 @@ # pylint: disable=too-many-statements -import asyncio from copy import deepcopy from http import HTTPStatus from unittest import mock @@ -19,25 +18,10 @@ from pytest_simcore.helpers.webserver_projects import create_project from pytest_simcore.helpers.webserver_workspaces import update_or_insert_workspace_group from servicelib.aiohttp import status -from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects.models import ProjectDict -@pytest.fixture -def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): - mocker.patch( - "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", - spec=True, - return_value=[], - ) - mocker.patch( - "simcore_service_webserver.projects._controller.projects_rest.project_uses_available_services", - spec=True, - return_value=True, - ) - - @pytest.mark.parametrize("user_role,expected", [(UserRole.USER, status.HTTP_200_OK)]) async def test_workspaces_full_workflow_with_folders_and_projects( client: TestClient, @@ -365,16 +349,6 @@ async def test_workspaces_delete_folders( resp = await client.delete(f"{url}") await assert_status(resp, status.HTTP_204_NO_CONTENT) - fire_and_forget_tasks = list(client.app[APP_FIRE_AND_FORGET_TASKS_KEY]) - t1: asyncio.Task = fire_and_forget_tasks[0] - t2: asyncio.Task = fire_and_forget_tasks[1] - assert t1.get_name().startswith("fire_and_forget_task_delete_project_task_") - assert t2.get_name().startswith("fire_and_forget_task_delete_project_task_") - await t1 - await t2 - - assert len(client.app[APP_FIRE_AND_FORGET_TASKS_KEY]) == 0 - # List project in workspace (The projects should have been deleted) url = ( client.app.router["list_projects"] diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py index f3846d1650dd..3ee21f6d55fa 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__list_projects_full_search.py @@ -19,21 +19,6 @@ from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects.models import ProjectDict - -@pytest.fixture -def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): - mocker.patch( - "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", - spec=True, - return_value=[], - ) - mocker.patch( - "simcore_service_webserver.projects._controller.projects_rest.project_uses_available_services", - spec=True, - return_value=True, - ) - - _SEARCH_NAME_1 = "Quantum Solutions" _SEARCH_NAME_2 = "Orion solution" _SEARCH_NAME_3 = "Skyline solutions" diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py index b18252fbdd1c..ad01ef5b947c 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_folders_between_workspaces.py @@ -25,20 +25,6 @@ def user_role() -> UserRole: return UserRole.USER -@pytest.fixture -def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): - mocker.patch( - "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", - spec=True, - return_value=[], - ) - mocker.patch( - "simcore_service_webserver.projects._controller.projects_rest.project_uses_available_services", - spec=True, - return_value=True, - ) - - @pytest.fixture async def moving_folder_id( client: TestClient, diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py index a308040670ba..af37341eda56 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__moving_projects_between_workspaces.py @@ -25,20 +25,6 @@ from simcore_service_webserver.projects.models import ProjectDict -@pytest.fixture -def mock_catalog_api_get_services_for_user_in_product(mocker: MockerFixture): - mocker.patch( - "simcore_service_webserver.projects._crud_api_read.catalog_service.get_services_for_user_in_product", - spec=True, - return_value=[], - ) - mocker.patch( - "simcore_service_webserver.projects._controller.projects_rest.project_uses_available_services", - spec=True, - return_value=True, - ) - - @pytest.mark.parametrize(*standard_role_response(), ids=str) async def test_moving_between_workspaces_user_role_permissions( client: TestClient,