From fe44cfb0d36579cb3547e982765025fdd211b077 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:43:42 +0100 Subject: [PATCH 01/71] trash plugin --- .../garbage_collector/_tasks_trash.py | 4 +- .../projects/_trash_api.py | 26 -------- .../projects/_trash_handlers.py | 15 ----- .../trash/__init__.py | 0 .../simcore_service_webserver/trash/_rest.py | 60 +++++++++++++++++++ .../trash/_service.py | 35 +++++++++++ .../simcore_service_webserver/trash/plugin.py | 33 ++++++++++ 7 files changed, 130 insertions(+), 43 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/trash/__init__.py create mode 100644 services/web/server/src/simcore_service_webserver/trash/_rest.py create mode 100644 services/web/server/src/simcore_service_webserver/trash/_service.py create mode 100644 services/web/server/src/simcore_service_webserver/trash/plugin.py diff --git a/services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_trash.py b/services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_trash.py index a86b71fd3f7f..47d5e7212f23 100644 --- a/services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_trash.py +++ b/services/web/server/src/simcore_service_webserver/garbage_collector/_tasks_trash.py @@ -12,7 +12,7 @@ from tenacity.before_sleep import before_sleep_log from tenacity.wait import wait_exponential -from ..projects._trash_api import prune_all_trashes +from ..trash._service import prune_trash _logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ before_sleep=before_sleep_log(_logger, logging.WARNING), ) async def _run_task(app: web.Application): - if deleted := await prune_all_trashes(app): + if deleted := await prune_trash(app): for name in deleted: _logger.info("Trash item %s expired and was deleted", f"{name}") else: diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_api.py b/services/web/server/src/simcore_service_webserver/projects/_trash_api.py index e15a98423c79..13e07c514756 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_api.py @@ -1,6 +1,5 @@ import asyncio import logging -from datetime import timedelta import arrow from aiohttp import web @@ -17,35 +16,10 @@ from ._access_rights_api import check_user_project_permission from .exceptions import ProjectRunningConflictError from .models import ProjectPatchExtended -from .settings import get_plugin_settings _logger = logging.getLogger(__name__) -async def empty_trash(app: web.Application, product_name: ProductName, user_id: UserID): - assert app # nosec - # filter trashed=True and set them to False - _logger.debug( - "CODE PLACEHOLDER: all projects marked as trashed of %s in %s are deleted", - f"{user_id=}", - f"{product_name=}", - ) - raise NotImplementedError - - -async def prune_all_trashes(app: web.Application) -> list[str]: - settings = get_plugin_settings(app) - retention = timedelta(days=settings.PROJECTS_TRASH_RETENTION_DAYS) - - _logger.debug( - "CODE PLACEHOLDER: **ALL** projects marked as trashed during %s days are deleted", - retention, - ) - await asyncio.sleep(5) - - return [] - - async def _is_project_running( app: web.Application, *, diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py index 963b81c49008..b0bb736c8e1e 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py @@ -53,21 +53,6 @@ routes = web.RouteTableDef() -@routes.delete(f"/{VTAG}/trash", name="empty_trash") -@login_required -@permission_required("project.delete") -@_handle_exceptions -async def empty_trash(request: web.Request): - user_id = get_user_id(request) - product_name = get_product_name(request) - - await _trash_api.empty_trash( - request.app, product_name=product_name, user_id=user_id - ) - - return web.json_response(status=status.HTTP_204_NO_CONTENT) - - @routes.post(f"/{VTAG}/projects/{{project_id}}:trash", name="trash_project") @login_required @permission_required("project.delete") diff --git a/services/web/server/src/simcore_service_webserver/trash/__init__.py b/services/web/server/src/simcore_service_webserver/trash/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/web/server/src/simcore_service_webserver/trash/_rest.py b/services/web/server/src/simcore_service_webserver/trash/_rest.py new file mode 100644 index 000000000000..f5912b042fe8 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/trash/_rest.py @@ -0,0 +1,60 @@ +import logging + +from aiohttp import web +from servicelib.aiohttp import status + +from .._meta import API_VTAG as VTAG +from ..exception_handling import ( + ExceptionToHttpErrorMap, + HttpErrorInfo, + exception_handling_decorator, + to_exceptions_handlers_map, +) +from ..login.decorators import get_user_id, login_required +from ..products.api import get_product_name +from ..projects.exceptions import ProjectRunningConflictError, ProjectStoppingError +from ..security.decorators import permission_required +from . import _service + +_logger = logging.getLogger(__name__) + +# +# EXCEPTIONS HANDLING +# + + +_TO_HTTP_ERROR_MAP: ExceptionToHttpErrorMap = { + ProjectRunningConflictError: HttpErrorInfo( + status.HTTP_409_CONFLICT, + "Current study is in use and cannot be trashed [project_id={project_uuid}]. Please stop all services first and try again", + ), + ProjectStoppingError: HttpErrorInfo( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Something went wrong while stopping services before trashing. Aborting trash.", + ), +} + + +_handle_exceptions = exception_handling_decorator( + to_exceptions_handlers_map(_TO_HTTP_ERROR_MAP) +) + + +# +# ROUTES +# + +routes = web.RouteTableDef() + + +@routes.delete(f"/{VTAG}/trash", name="empty_trash") +@login_required +@permission_required("project.delete") +@_handle_exceptions +async def empty_trash(request: web.Request): + user_id = get_user_id(request) + product_name = get_product_name(request) + + await _service.empty_trash(request.app, product_name=product_name, user_id=user_id) + + return web.json_response(status=status.HTTP_204_NO_CONTENT) diff --git a/services/web/server/src/simcore_service_webserver/trash/_service.py b/services/web/server/src/simcore_service_webserver/trash/_service.py new file mode 100644 index 000000000000..379e94b94749 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/trash/_service.py @@ -0,0 +1,35 @@ +import asyncio +import logging +from datetime import timedelta + +from aiohttp import web +from models_library.products import ProductName +from models_library.users import UserID + +from .settings import get_plugin_settings + +_logger = logging.getLogger(__name__) + + +async def empty_trash(app: web.Application, product_name: ProductName, user_id: UserID): + assert app # nosec + # filter trashed=True and set them to False + _logger.debug( + "CODE PLACEHOLDER: all projects marked as trashed of %s in %s are deleted", + f"{user_id=}", + f"{product_name=}", + ) + raise NotImplementedError + + +async def prune_trash(app: web.Application) -> list[str]: + settings = get_plugin_settings(app) + retention = timedelta(days=settings.PROJECTS_TRASH_RETENTION_DAYS) + + _logger.debug( + "CODE PLACEHOLDER: **ALL** projects marked as trashed during %s days are deleted", + retention, + ) + await asyncio.sleep(5) + + return [] diff --git a/services/web/server/src/simcore_service_webserver/trash/plugin.py b/services/web/server/src/simcore_service_webserver/trash/plugin.py new file mode 100644 index 000000000000..a4cde6415966 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/trash/plugin.py @@ -0,0 +1,33 @@ +""" projects management subsystem + + A project is a document defining a osparc study + It contains metadata about the study (e.g. name, description, owner, etc) and a workbench section that describes the study pipeline +""" +import logging + +from aiohttp import web +from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup + +from .._constants import APP_SETTINGS_KEY +from ..folders.plugin import setup_folders +from ..projects.plugin import setup_projects +from ..workspaces.plugin import setup_workspaces +from . import _rest + +_logger = logging.getLogger(__name__) + + +@app_module_setup( + __name__, + ModuleCategory.ADDON, + settings_name="WEBSERVER_TRASH", + logger=_logger, +) +def setup_trash(app: web.Application): + assert app[APP_SETTINGS_KEY].WEBSERVER_TRASH # nosec + + setup_projects(app) + setup_folders(app) + setup_workspaces(app) + + app.router.add_routes(_rest.routes) From 0ec86ee399f7cd32b4684c6d410f11093902afaf Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:45:53 +0100 Subject: [PATCH 02/71] common models --- api/specs/web-server/_projects_crud.py | 2 +- api/specs/web-server/_projects_groups.py | 2 +- api/specs/web-server/_projects_wallet.py | 2 +- .../projects/_comments_handlers.py | 2 +- .../projects/_common/__init__.py | 0 .../projects/{_common_models.py => _common/models.py} | 2 +- .../projects/_crud_handlers.py | 2 +- .../projects/_folders_handlers.py | 10 +++++----- .../projects/_groups_handlers.py | 2 +- .../projects/_metadata_handlers.py | 2 +- .../projects/_nodes_handlers.py | 2 +- .../projects/_ports_handlers.py | 2 +- .../projects/_projects_nodes_pricing_unit_handlers.py | 2 +- .../projects/_states_handlers.py | 2 +- .../projects/_trash_handlers.py | 3 +-- .../projects/_wallets_handlers.py | 2 +- .../projects/_workspaces_handlers.py | 2 +- 17 files changed, 20 insertions(+), 21 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/projects/_common/__init__.py rename services/web/server/src/simcore_service_webserver/projects/{_common_models.py => _common/models.py} (94%) diff --git a/api/specs/web-server/_projects_crud.py b/api/specs/web-server/_projects_crud.py index 31f26d6425e2..62fe7684c682 100644 --- a/api/specs/web-server/_projects_crud.py +++ b/api/specs/web-server/_projects_crud.py @@ -30,7 +30,7 @@ from models_library.rest_pagination import Page from pydantic import BaseModel from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.projects._common_models import ProjectPathParams +from simcore_service_webserver.projects._common.models import ProjectPathParams from simcore_service_webserver.projects._crud_handlers import ProjectCreateParams from simcore_service_webserver.projects._crud_handlers_models import ( ProjectActiveQueryParams, diff --git a/api/specs/web-server/_projects_groups.py b/api/specs/web-server/_projects_groups.py index 88432bb3cfac..cfc0870d6a81 100644 --- a/api/specs/web-server/_projects_groups.py +++ b/api/specs/web-server/_projects_groups.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, status from models_library.generics import Envelope from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.projects._common_models import ProjectPathParams +from simcore_service_webserver.projects._common.models import ProjectPathParams from simcore_service_webserver.projects._groups_api import ProjectGroupGet from simcore_service_webserver.projects._groups_handlers import ( _ProjectsGroupsBodyParams, diff --git a/api/specs/web-server/_projects_wallet.py b/api/specs/web-server/_projects_wallet.py index e079dd33e171..c9502393b971 100644 --- a/api/specs/web-server/_projects_wallet.py +++ b/api/specs/web-server/_projects_wallet.py @@ -16,7 +16,7 @@ from models_library.projects import ProjectID from models_library.wallets import WalletID from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.projects._common_models import ProjectPathParams +from simcore_service_webserver.projects._common.models import ProjectPathParams router = APIRouter( prefix=f"/{API_VTAG}", diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py index 6ad8b290ba08..c46610057047 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py @@ -31,7 +31,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _comments_api, projects_api -from ._common_models import RequestContext +from ._common.models import RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/projects/_common/__init__.py b/services/web/server/src/simcore_service_webserver/projects/_common/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/web/server/src/simcore_service_webserver/projects/_common_models.py b/services/web/server/src/simcore_service_webserver/projects/_common/models.py similarity index 94% rename from services/web/server/src/simcore_service_webserver/projects/_common_models.py rename to services/web/server/src/simcore_service_webserver/projects/_common/models.py index bb98b168aea3..6f358378f600 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_common_models.py +++ b/services/web/server/src/simcore_service_webserver/projects/_common/models.py @@ -7,7 +7,7 @@ from models_library.projects import ProjectID from pydantic import BaseModel, ConfigDict, Field -from ..models import RequestContext +from ...models import RequestContext assert RequestContext.__name__ # nosec diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index 8173a7a2db6d..09be85531976 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -48,7 +48,7 @@ from ..users.api import get_user_fullname from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError from . import _crud_api_create, _crud_api_read, projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from ._crud_handlers_models import ( ProjectActiveQueryParams, ProjectCreateHeaders, diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py index 2e644a4d598f..c4f1828237b8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py @@ -5,7 +5,7 @@ from models_library.folders import FolderID from models_library.projects import ProjectID from models_library.utils.common_validators import null_or_none_str_to_none_validator -from pydantic import ConfigDict, BaseModel, field_validator +from pydantic import BaseModel, ConfigDict, field_validator from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as from servicelib.aiohttp.typing_extension import Handler @@ -14,7 +14,7 @@ from ..login.decorators import login_required from ..security.decorators import permission_required from . import _folders_api -from ._common_models import RequestContext +from ._common.models import RequestContext from .exceptions import ProjectGroupNotFoundError, ProjectNotFoundError _logger = logging.getLogger(__name__) @@ -44,9 +44,9 @@ class _ProjectsFoldersPathParams(BaseModel): model_config = ConfigDict(extra="forbid") # validators - _null_or_none_str_to_none_validator = field_validator( - "folder_id", mode="before" - )(null_or_none_str_to_none_validator) + _null_or_none_str_to_none_validator = field_validator("folder_id", mode="before")( + null_or_none_str_to_none_validator + ) @routes.put( diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py index bf612944d4b7..d507a2b1eff9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py @@ -21,7 +21,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _groups_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from ._groups_api import ProjectGroupGet from .exceptions import ProjectGroupNotFoundError, ProjectNotFoundError diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index 802c13f79378..df139c6fd30b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -30,7 +30,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _metadata_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .exceptions import ( NodeNotFoundError, ParentNodeNotFoundError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 9ddd88c0df1f..514efafa47d2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -74,7 +74,7 @@ from ..utils_aiohttp import envelope_json_response from ..wallets.errors import WalletAccessForbiddenError, WalletNotEnoughCreditsError from . import nodes_utils, projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from ._nodes_api import NodeScreenshot, get_node_screenshots from .exceptions import ( ClustersKeeperNotAvailableError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py index eaacd9c1aa3f..db8be1b9cfd0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py @@ -33,7 +33,7 @@ from ..projects._access_rights_api import check_user_project_permission from ..security.decorators import permission_required from . import _ports_api, projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .db import ProjectDBAPI from .exceptions import ( NodeNotFoundError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py index 05bb2f8e7676..2748d81061e8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py @@ -21,7 +21,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import projects_api -from ._common_models import RequestContext +from ._common.models import RequestContext from ._nodes_handlers import NodePathParams from .db import ProjectDBAPI from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError diff --git a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py index 8ec0400238cb..67a21b23eceb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py @@ -36,7 +36,7 @@ from ..utils_aiohttp import envelope_json_response from ..wallets.errors import WalletNotEnoughCreditsError from . import projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .exceptions import ( DefaultPricingUnitNotFoundError, ProjectInvalidRightsError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py index b0bb736c8e1e..e555807e2d49 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py @@ -16,10 +16,9 @@ ) from ..login.decorators import get_user_id, login_required from ..products.api import get_product_name -from ..projects._common_models import ProjectPathParams from ..security.decorators import permission_required from . import _trash_api -from ._common_models import RemoveQueryParams +from ._common.models import ProjectPathParams, RemoveQueryParams from .exceptions import ProjectRunningConflictError, ProjectStoppingError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py index 56e7136d299d..dd66c65cb8df 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py @@ -20,7 +20,7 @@ from ..wallets.errors import WalletAccessForbiddenError from . import _wallets_api as wallets_api from . import projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py index ef3d20b3c5ac..b5a6082cb506 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py @@ -17,7 +17,7 @@ from ..security.decorators import permission_required from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError from . import _workspaces_api -from ._common_models import RequestContext +from ._common.models import RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError _logger = logging.getLogger(__name__) From 069b99751204d0e283b056ad09f471c961b80374 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:05:41 +0100 Subject: [PATCH 03/71] create settings --- .../application_settings.py | 41 +- .../trash/settings.py | 18 + .../src/simcore_service_webserver/tree.md | 562 ------------------ 3 files changed, 46 insertions(+), 575 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/trash/settings.py delete mode 100644 services/web/server/src/simcore_service_webserver/tree.md diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index 07941e1e92c6..638e28d615e8 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -53,6 +53,7 @@ from .statics.settings import FrontEndAppSettings, StaticWebserverModuleSettings from .storage.settings import StorageSettings from .studies_dispatcher.settings import StudiesDispatcherSettings +from .trash.settings import TrashSettings from .users.settings import UsersSettings _logger = logging.getLogger(__name__) @@ -175,8 +176,13 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): json_schema_extra={"auto_default_from_env": True}, description="director-v2 service client's plugin", ) + + WEBSERVER_DYNAMIC_SCHEDULER: DynamicSchedulerSettings | None = Field( + json_schema_extra={"auto_default_from_env": True}, + ) + WEBSERVER_EMAIL: SMTPSettings | None = Field( - json_schema_extra={"auto_default_from_env": True}, description="email plugin" + json_schema_extra={"auto_default_from_env": True} ) WEBSERVER_EXPORTER: ExporterSettings | None = Field( json_schema_extra={"auto_default_from_env": True}, description="exporter plugin" @@ -204,9 +210,8 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="payments plugin settings", ) - WEBSERVER_DYNAMIC_SCHEDULER: DynamicSchedulerSettings | None = Field( - description="dynamic-scheduler plugin settings", - json_schema_extra={"auto_default_from_env": True}, + WEBSERVER_PROJECTS: ProjectsSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} ) WEBSERVER_REDIS: RedisSettings | None = Field( @@ -254,15 +259,25 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="tracing plugin", json_schema_extra={"auto_default_from_env": True} ) - WEBSERVER_PROJECTS: ProjectsSettings | None = Field( - description="projects plugin", json_schema_extra={"auto_default_from_env": True} - ) - WEBSERVER_RABBITMQ: RabbitSettings | None = Field( - description="rabbitmq plugin", json_schema_extra={"auto_default_from_env": True} - ) - WEBSERVER_USERS: UsersSettings | None = Field( - description="users plugin", json_schema_extra={"auto_default_from_env": True} - ) + WEBSERVER_TRASH: Annotated[ + TrashSettings, Field(json_schema_extra={"auto_default_from_env": True}) + ] + + WEBSERVER_RABBITMQ: Annotated[ + RabbitSettings | None, + Field( + description="rabbitmq plugin", + json_schema_extra={"auto_default_from_env": True}, + ), + ] + + WEBSERVER_USERS: Annotated[ + UsersSettings | None, + Field( + description="users plugin", + json_schema_extra={"auto_default_from_env": True}, + ), + ] # These plugins only require (for the moment) an entry to toggle between enabled/disabled WEBSERVER_ANNOUNCEMENTS: bool = False diff --git a/services/web/server/src/simcore_service_webserver/trash/settings.py b/services/web/server/src/simcore_service_webserver/trash/settings.py new file mode 100644 index 000000000000..90b1296235dd --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/trash/settings.py @@ -0,0 +1,18 @@ +from aiohttp import web +from pydantic import Field, NonNegativeInt +from settings_library.base import BaseCustomSettings + +from .._constants import APP_SETTINGS_KEY + + +class TrashSettings(BaseCustomSettings): + TRASH_RETENTION_DAYS: NonNegativeInt = Field( + default=7, description="Trashed items will be deleted after this time" + ) + + +def get_plugin_settings(app: web.Application) -> TrashSettings: + settings = app[APP_SETTINGS_KEY].WEBSERVER_TRASH + assert settings, "setup_settings not called?" # nosec + assert isinstance(settings, TrashSettings) # nosec + return settings diff --git a/services/web/server/src/simcore_service_webserver/tree.md b/services/web/server/src/simcore_service_webserver/tree.md deleted file mode 100644 index 0117a6c851ee..000000000000 --- a/services/web/server/src/simcore_service_webserver/tree.md +++ /dev/null @@ -1,562 +0,0 @@ -This is a tree view of my app. It is built in python's aiohttp. - - -├── activity -│   ├── _api.py -│   ├── _handlers.py -│   ├── plugin.py -│   └── settings.py -├── announcements -│   ├── _api.py -│   ├── _handlers.py -│   ├── _models.py -│   ├── plugin.py -│   └── _redis.py -├── api_keys -│   ├── api.py -│   ├── errors.py -│   ├── _exceptions_handlers.py -│   ├── _models.py -│   ├── plugin.py -│   ├── _repository.py -│   ├── _rest.py -│   ├── _rpc.py -│   └── _service.py -├── application.py -├── application_settings.py -├── application_settings_utils.py -├── catalog -│   ├── _api.py -│   ├── _api_units.py -│   ├── client.py -│   ├── _constants.py -│   ├── exceptions.py -│   ├── _handlers_errors.py -│   ├── _handlers.py -│   ├── licenses -│   │   ├── api.py -│   │   ├── errors.py -│   │   ├── _exceptions_handlers.py -│   │   ├── _licensed_items_api.py -│   │   ├── _licensed_items_db.py -│   │   ├── _licensed_items_handlers.py -│   │   ├── _models.py -│   │   └── plugin.py -│   ├── _models.py -│   ├── plugin.py -│   ├── settings.py -│   └── _tags_handlers.py -├── cli.py -├── _constants.py -├── db -│   ├── _aiopg.py -│   ├── _asyncpg.py -│   ├── base_repository.py -│   ├── models.py -│   ├── plugin.py -│   └── settings.py -├── db_listener -│   ├── _db_comp_tasks_listening_task.py -│   ├── plugin.py -│   └── _utils.py -├── diagnostics -│   ├── _handlers.py -│   ├── _healthcheck.py -│   ├── _monitoring.py -│   ├── plugin.py -│   └── settings.py -├── director_v2 -│   ├── _abc.py -│   ├── api.py -│   ├── _api_utils.py -│   ├── _core_base.py -│   ├── _core_computations.py -│   ├── _core_dynamic_services.py -│   ├── _core_utils.py -│   ├── exceptions.py -│   ├── _handlers.py -│   ├── plugin.py -│   └── settings.py -├── dynamic_scheduler -│   ├── api.py -│   ├── plugin.py -│   └── settings.py -├── email -│   ├── _core.py -│   ├── _handlers.py -│   ├── plugin.py -│   ├── settings.py -│   └── utils.py -├── errors.py -├── exception_handling -│   ├── _base.py -│   └── _factory.py -├── exporter -│   ├── exceptions.py -│   ├── _formatter -│   │   ├── archive.py -│   │   ├── _sds.py -│   │   ├── template_json.py -│   │   └── xlsx -│   │   ├── code_description.py -│   │   ├── core -│   │   │   ├── styling_components.py -│   │   │   └── xlsx_base.py -│   │   ├── dataset_description.py -│   │   ├── manifest.py -│   │   ├── utils.py -│   │   └── writer.py -│   ├── _handlers.py -│   ├── plugin.py -│   ├── settings.py -│   └── utils.py -├── folders -│   ├── api.py -│   ├── errors.py -│   ├── _exceptions_handlers.py -│   ├── _folders_api.py -│   ├── _folders_db.py -│   ├── _folders_handlers.py -│   ├── _models.py -│   ├── plugin.py -│   ├── _trash_api.py -│   ├── _trash_handlers.py -│   ├── _workspaces_api.py -│   └── _workspaces_handlers.py -├── garbage_collector -│   ├── _core_disconnected.py -│   ├── _core_guests.py -│   ├── _core_orphans.py -│   ├── _core.py -│   ├── _core_utils.py -│   ├── plugin.py -│   ├── settings.py -│   ├── _tasks_api_keys.py -│   ├── _tasks_core.py -│   ├── _tasks_trash.py -│   └── _tasks_users.py -├── groups -│   ├── api.py -│   ├── _classifiers_api.py -│   ├── _classifiers_handlers.py -│   ├── _common -│   │   ├── exceptions_handlers.py -│   │   └── schemas.py -│   ├── exceptions.py -│   ├── _groups_api.py -│   ├── _groups_db.py -│   ├── _groups_handlers.py -│   └── plugin.py -├── invitations -│   ├── api.py -│   ├── _client.py -│   ├── _core.py -│   ├── errors.py -│   ├── plugin.py -│   └── settings.py -├── login -│   ├── _2fa_api.py -│   ├── _2fa_handlers.py -│   ├── _auth_api.py -│   ├── _auth_handlers.py -│   ├── cli.py -│   ├── _confirmation.py -│   ├── _constants.py -│   ├── decorators.py -│   ├── errors.py -│   ├── handlers_change.py -│   ├── handlers_confirmation.py -│   ├── handlers_registration.py -│   ├── _models.py -│   ├── plugin.py -│   ├── _registration_api.py -│   ├── _registration_handlers.py -│   ├── _registration.py -│   ├── _security.py -│   ├── settings.py -│   ├── _sql.py -│   ├── storage.py -│   ├── utils_email.py -│   └── utils.py -├── log.py -├── long_running_tasks.py -├── __main__.py -├── meta_modeling -│   ├── _function_nodes.py -│   ├── _handlers.py -│   ├── _iterations.py -│   ├── plugin.py -│   ├── _projects.py -│   ├── _results.py -│   └── _version_control.py -├── _meta.py -├── models.py -├── notifications -│   ├── plugin.py -│   ├── project_logs.py -│   ├── _rabbitmq_consumers_common.py -│   ├── _rabbitmq_exclusive_queue_consumers.py -│   ├── _rabbitmq_nonexclusive_queue_consumers.py -│   └── wallet_osparc_credits.py -├── payments -│   ├── api.py -│   ├── _autorecharge_api.py -│   ├── _autorecharge_db.py -│   ├── errors.py -│   ├── _events.py -│   ├── _methods_api.py -│   ├── _methods_db.py -│   ├── _onetime_api.py -│   ├── _onetime_db.py -│   ├── plugin.py -│   ├── _rpc_invoice.py -│   ├── _rpc.py -│   ├── settings.py -│   ├── _socketio.py -│   └── _tasks.py -├── products -│   ├── _api.py -│   ├── api.py -│   ├── _db.py -│   ├── errors.py -│   ├── _events.py -│   ├── _handlers.py -│   ├── _invitations_handlers.py -│   ├── _middlewares.py -│   ├── _model.py -│   ├── plugin.py -│   └── _rpc.py -├── projects -│   ├── _access_rights_api.py -│   ├── _access_rights_db.py -│   ├── api.py -│   ├── _comments_api.py -│   ├── _comments_db.py -│   ├── _comments_handlers.py -│   ├── _common_models.py -│   ├── _crud_api_create.py -│   ├── _crud_api_delete.py -│   ├── _crud_api_read.py -│   ├── _crud_handlers_models.py -│   ├── _crud_handlers.py -│   ├── db.py -│   ├── _db_utils.py -│   ├── exceptions.py -│   ├── _folders_api.py -│   ├── _folders_db.py -│   ├── _folders_handlers.py -│   ├── _groups_api.py -│   ├── _groups_db.py -│   ├── _groups_handlers.py -│   ├── lock.py -│   ├── _metadata_api.py -│   ├── _metadata_db.py -│   ├── _metadata_handlers.py -│   ├── models.py -│   ├── _nodes_api.py -│   ├── _nodes_handlers.py -│   ├── _nodes_utils.py -│   ├── nodes_utils.py -│   ├── _observer.py -│   ├── _permalink_api.py -│   ├── plugin.py -│   ├── _ports_api.py -│   ├── _ports_handlers.py -│   ├── _projects_access.py -│   ├── projects_api.py -│   ├── _projects_db.py -│   ├── _projects_nodes_pricing_unit_handlers.py -│   ├── settings.py -│   ├── _states_handlers.py -│   ├── _tags_api.py -│   ├── _tags_handlers.py -│   ├── _trash_api.py -│   ├── _trash_handlers.py -│   ├── utils.py -│   ├── _wallets_api.py -│   ├── _wallets_handlers.py -│   ├── _workspaces_api.py -│   └── _workspaces_handlers.py -├── publications -│   ├── _handlers.py -│   └── plugin.py -├── rabbitmq.py -├── rabbitmq_settings.py -├── redis.py -├── resource_manager -│   ├── _constants.py -│   ├── plugin.py -│   ├── registry.py -│   ├── settings.py -│   └── user_sessions.py -├── _resources.py -├── resource_usage -│   ├── api.py -│   ├── _client.py -│   ├── _constants.py -│   ├── errors.py -│   ├── _observer.py -│   ├── plugin.pyf -│   ├── _pricing_plans_admin_api.py -│   ├── _pricing_plans_admin_handlers.py -│   ├── _pricing_plans_api.py -│   ├── _pricing_plans_handlers.py -│   ├── _service_runs_api.py -│   ├── _service_runs_handlers.py -│   ├── settings.py -│   └── _utils.py -├── rest -│   ├── _handlers.py -│   ├── healthcheck.py -│   ├── plugin.py -│   ├── settings.py -│   └── _utils.py -├── scicrunch -│   ├── db.py -│   ├── errors.py -│   ├── models.py -│   ├── plugin.py -│   ├── _resolver.py -│   ├── _rest.py -│   ├── service_client.py -│   └── settings.py -├── security -│   ├── api.py -│   ├── _authz_access_model.py -│   ├── _authz_access_roles.py -│   ├── _authz_db.py -│   ├── _authz_policy.py -│   ├── _constants.py -│   ├── decorators.py -│   ├── _identity_api.py -│   ├── _identity_policy.py -│   └── plugin.py -├── session -│   ├── access_policies.py -│   ├── api.py -│   ├── _cookie_storage.py -│   ├── errors.py -│   ├── plugin.py -│   └── settings.py -├── socketio -│   ├── _handlers.py -│   ├── messages.py -│   ├── models.py -│   ├── _observer.py -│   ├── plugin.py -│   ├── server.py -│   └── _utils.py -├── statics -│   ├── _constants.py -│   ├── _events.py -│   ├── _handlers.py -│   ├── plugin.py -│   └── settings.py -├── storage -│   ├── api.py -│   ├── _handlers.py -│   ├── plugin.py -│   ├── schemas.py -│   └── settings.py -├── studies_dispatcher -│   ├── _catalog.py -│   ├── _constants.py -│   ├── _core.py -│   ├── _errors.py -│   ├── _models.py -│   ├── plugin.py -│   ├── _projects_permalinks.py -│   ├── _projects.py -│   ├── _redirects_handlers.py -│   ├── _rest_handlers.py -│   ├── settings.py -│   ├── _studies_access.py -│   └── _users.py -├── tags -│   ├── _api.py -│   ├── _handlers.py -│   ├── plugin.py -│   └── schemas.py -├── tracing.py -├── users -│   ├── _api.py -│   ├── api.py -│   ├── _constants.py -│   ├── _db.py -│   ├── exceptions.py -│   ├── _handlers.py -│   ├── _models.py -│   ├── _notifications_handlers.py -│   ├── _notifications.py -│   ├── plugin.py -│   ├── _preferences_api.py -│   ├── preferences_api.py -│   ├── _preferences_db.py -│   ├── _preferences_handlers.py -│   ├── _preferences_models.py -│   ├── _schemas.py -│   ├── schemas.py -│   ├── settings.py -│   ├── _tokens_handlers.py -│   └── _tokens.py -├── utils_aiohttp.py -├── utils.py -├── utils_rate_limiting.py -├── version_control -│   ├── _core.py -│   ├── db.py -│   ├── errors.py -│   ├── _handlers_base.py -│   ├── _handlers.py -│   ├── models.py -│   ├── plugin.py -│   ├── vc_changes.py -│   └── vc_tags.py -├── wallets -│   ├── _api.py -│   ├── api.py -│   ├── _constants.py -│   ├── _db.py -│   ├── errors.py -│   ├── _events.py -│   ├── _groups_api.py -│   ├── _groups_db.py -│   ├── _groups_handlers.py -│   ├── _handlers.py -│   ├── _payments_handlers.py -│   └── plugin.py -└── workspaces - ├── api.py - ├── errors.py - ├── _exceptions_handlers.py - ├── _groups_api.py - ├── _groups_db.py - ├── _groups_handlers.py - ├── _models.py - ├── plugin.py - ├── _trash_api.py - ├── _trash_handlers.py - ├── _workspaces_api.py - ├── _workspaces_db.py - └── _workspaces_handlers.py - - - - - -The top folders represent plugins that could be interprested as different domains with small compling between each other - -Here are some conventions - -- `plugin` has a setup function to setup the app (e.g. add routes, setup events etc ). Classic `setup_xxx(app)` for aiohttp -- `settings` includes pydantic settings classes specific to the domain -- `exceptions` or `errors` include only exceptions classes - - `_exceptions_handlers` are utils to handle exceptions -- `models` correspond to domain models, i.e. not part of any of the controllers interfaces. Those are denoted `scheme`. - -Then - -- `_handlers` (or _rest) represent the rest handlers (i.e. controller layer) -- `_rpc` contains handlers (server side) to an RPC interface (i.e. controller layer) -- `_api` (or `_service`) represent the business logic of this domain (i.e. service layer) - - the shared service layer for inter-domain logic is called `api` (i.e. without `_`) -- `_db` (or `_repository`) represents the repository layer - - -Could you please apply the rules on this structure and come up with a new tree that follows: -- keeps the domain-drive modular organization -- every domain implements controller-service-repository (CSR) -- highligh the shared service layer for inter-domain logic - - -As an output just recreate the tree adding some comments on it (e.g. with #) but no further explanatio is needed - - - -```plaintext -├── activity -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for activity domain (service layer) -│   ├── repositories -│   │   ├── activity_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for activity domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── announcements -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for announcements domain (service layer) -│   ├── repositories -│   │   ├── announcements_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for announcements domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── api_keys -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for api_keys domain (service layer) -│   ├── repositories -│   │   ├── api_keys_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for api_keys domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── shared_services -│   ├── inter_domain_service.py # Shared service layer for inter-domain logic -├── db -│   ├── connection.py # Database connection logic -│   ├── base_repository.py # Base repository logic -│   ├── plugin.py # Setup function for database -│   ├── settings.py # Database-specific settings -├── catalog -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for catalog domain (service layer) -│   ├── repositories -│   │   ├── catalog_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for catalog domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── users -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for users domain (service layer) -│   ├── repositories -│   │   ├── users_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for users domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── projects -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for projects domain (service layer) -│   ├── repositories -│   │   ├── projects_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for projects domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── shared -│   ├── models -│   │   ├── user.py # Shared user model -│   │   ├── project.py # Shared project model -│   ├── schemas -│   │   ├── user_schema.py # Shared user schemas -│   │   ├── project_schema.py # Shared project schemas -│   ├── utils -│   │   ├── logger.py # Shared logging logic -│   │   ├── validators.py # Shared validation logic -├── application.py # Main application initialization -└── cli.py # Command-line interface logic -``` From df61e6a18621d598264967c2b116d6b09db8bd66 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:10:16 +0100 Subject: [PATCH 04/71] trash settings --- .../client/source/class/osparc/store/StaticInfo.js | 4 ++-- .../src/simcore_service_webserver/application_settings.py | 6 +++--- .../src/simcore_service_webserver/projects/settings.py | 4 ---- .../server/src/simcore_service_webserver/trash/_service.py | 2 +- .../server/src/simcore_service_webserver/trash/settings.py | 4 +++- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/services/static-webserver/client/source/class/osparc/store/StaticInfo.js b/services/static-webserver/client/source/class/osparc/store/StaticInfo.js index 2ac96fd58b06..69a28b57f7b8 100644 --- a/services/static-webserver/client/source/class/osparc/store/StaticInfo.js +++ b/services/static-webserver/client/source/class/osparc/store/StaticInfo.js @@ -70,9 +70,9 @@ qx.Class.define("osparc.store.StaticInfo", { }, getTrashRetentionDays: function() { - const staticKey = "webserverProjects"; + const staticKey = "webserverTrash"; const wsStaticData = this.getValue(staticKey); - const key = "PROJECTS_TRASH_RETENTION_DAYS"; + const key = "TRASH_RETENTION_DAYS"; if (key in wsStaticData) { return wsStaticData[key]; } diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index 638e28d615e8..8570a79ad2ff 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -266,7 +266,6 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): WEBSERVER_RABBITMQ: Annotated[ RabbitSettings | None, Field( - description="rabbitmq plugin", json_schema_extra={"auto_default_from_env": True}, ), ] @@ -274,7 +273,6 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): WEBSERVER_USERS: Annotated[ UsersSettings | None, Field( - description="users plugin", json_schema_extra={"auto_default_from_env": True}, ), ] @@ -455,7 +453,9 @@ def to_client_statics(self) -> dict[str, Any]: "SWARM_STACK_NAME": True, "WEBSERVER_PROJECTS": { "PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES", - "PROJECTS_TRASH_RETENTION_DAYS", + }, + "WEBSERVER_TRASH": { + "TRASH_RETENTION_DAYS", }, "WEBSERVER_LOGIN": { "LOGIN_ACCOUNT_DELETION_RETENTION_DAYS", diff --git a/services/web/server/src/simcore_service_webserver/projects/settings.py b/services/web/server/src/simcore_service_webserver/projects/settings.py index 57c03f915efe..ace293856023 100644 --- a/services/web/server/src/simcore_service_webserver/projects/settings.py +++ b/services/web/server/src/simcore_service_webserver/projects/settings.py @@ -23,10 +23,6 @@ class ProjectsSettings(BaseCustomSettings): description="interval after which services need to be idle in order to be considered inactive", ) - PROJECTS_TRASH_RETENTION_DAYS: NonNegativeInt = Field( - default=7, description="Trashed items will be deleted after this time" - ) - def get_plugin_settings(app: web.Application) -> ProjectsSettings: settings = app[APP_SETTINGS_KEY].WEBSERVER_PROJECTS diff --git a/services/web/server/src/simcore_service_webserver/trash/_service.py b/services/web/server/src/simcore_service_webserver/trash/_service.py index 379e94b94749..874534944962 100644 --- a/services/web/server/src/simcore_service_webserver/trash/_service.py +++ b/services/web/server/src/simcore_service_webserver/trash/_service.py @@ -24,7 +24,7 @@ async def empty_trash(app: web.Application, product_name: ProductName, user_id: async def prune_trash(app: web.Application) -> list[str]: settings = get_plugin_settings(app) - retention = timedelta(days=settings.PROJECTS_TRASH_RETENTION_DAYS) + retention = timedelta(days=settings.TRASH_RETENTION_DAYS) _logger.debug( "CODE PLACEHOLDER: **ALL** projects marked as trashed during %s days are deleted", diff --git a/services/web/server/src/simcore_service_webserver/trash/settings.py b/services/web/server/src/simcore_service_webserver/trash/settings.py index 90b1296235dd..8708e2f54051 100644 --- a/services/web/server/src/simcore_service_webserver/trash/settings.py +++ b/services/web/server/src/simcore_service_webserver/trash/settings.py @@ -7,7 +7,9 @@ class TrashSettings(BaseCustomSettings): TRASH_RETENTION_DAYS: NonNegativeInt = Field( - default=7, description="Trashed items will be deleted after this time" + default=7, + description="Trashed items will be deleted after this time", + alias="PROJECTS_TRASH_RETENTION_DAYS", ) From 261ceee1372755afa456205dc158b0552b98b0d3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:14:24 +0100 Subject: [PATCH 05/71] renames trash settings --- .env-devel | 8 ++++---- services/docker-compose.yml | 6 +++++- .../src/simcore_service_webserver/trash/settings.py | 2 -- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.env-devel b/.env-devel index cc6609460dad..26fb29a093a7 100644 --- a/.env-devel +++ b/.env-devel @@ -334,7 +334,6 @@ LOGIN_ACCOUNT_DELETION_RETENTION_DAYS=31 LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0 LOGIN_REGISTRATION_INVITATION_REQUIRED=0 PROJECTS_INACTIVITY_INTERVAL=00:00:20 -PROJECTS_TRASH_RETENTION_DAYS=7 PROJECTS_MAX_COPY_SIZE_BYTES=30Gib PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES=5 REST_SWAGGER_API_DOC_ENABLED=1 @@ -348,11 +347,12 @@ SIMCORE_VCS_RELEASE_TAG=latest STUDIES_ACCESS_ANONYMOUS_ALLOWED=0 STUDIES_DEFAULT_SERVICE_THUMBNAIL=https://via.placeholder.com/170x120.png TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE=2 -TRACING_OPENTELEMETRY_COLLECTOR_PORT=4318 TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT=http://opentelemetry-collector -TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=100 TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT=http://jaeger:4318 +TRACING_OPENTELEMETRY_COLLECTOR_PORT=4318 +TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=100 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack +TRASH_RETENTION_DAYS=7 TWILIO_ACCOUNT_SID=DUMMY TWILIO_AUTH_TOKEN=DUMMY TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT=["41"] @@ -365,8 +365,8 @@ WEBSERVER_DEV_FEATURES_ENABLED=0 WEBSERVER_DIAGNOSTICS={} WEBSERVER_EMAIL={} WEBSERVER_EXPORTER={} -WEBSERVER_FRONTEND={} WEBSERVER_FOLDERS=1 +WEBSERVER_FRONTEND={} WEBSERVER_GARBAGE_COLLECTOR=null WEBSERVER_GROUPS=1 WEBSERVER_GUNICORN_CMD_ARGS=--timeout=180 diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 766c117244de..0f589739ad38 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -778,7 +778,7 @@ services: PROJECTS_INACTIVITY_INTERVAL: ${PROJECTS_INACTIVITY_INTERVAL} PROJECTS_MAX_COPY_SIZE_BYTES: ${PROJECTS_MAX_COPY_SIZE_BYTES} PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES: ${PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES} - PROJECTS_TRASH_RETENTION_DAYS: ${PROJECTS_TRASH_RETENTION_DAYS} + # WEBSERVER_RABBITMQ RABBIT_HOST: ${RABBIT_HOST} @@ -787,6 +787,10 @@ services: RABBIT_SECURE: ${RABBIT_SECURE} RABBIT_USER: ${RABBIT_USER} + # WEBSERVER_TRASH + TRASH_RETENTION_DAYS: ${TRASH_RETENTION_DAYS} + + # ARBITRARY ENV VARS # see [https://docs.gunicorn.org/en/stable/settings.html#timeout], diff --git a/services/web/server/src/simcore_service_webserver/trash/settings.py b/services/web/server/src/simcore_service_webserver/trash/settings.py index 8708e2f54051..55456f6da2ed 100644 --- a/services/web/server/src/simcore_service_webserver/trash/settings.py +++ b/services/web/server/src/simcore_service_webserver/trash/settings.py @@ -7,9 +7,7 @@ class TrashSettings(BaseCustomSettings): TRASH_RETENTION_DAYS: NonNegativeInt = Field( - default=7, description="Trashed items will be deleted after this time", - alias="PROJECTS_TRASH_RETENTION_DAYS", ) From b4f9d67a5ed5f59cde447a9820c030e1c5b1b2e5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:45:48 +0100 Subject: [PATCH 06/71] fixes test_activity deps --- services/web/server/tests/conftest.py | 2 + .../tests/data/test_activity_config.yml | 53 --------------- .../tests/unit/isolated/test_activity.py | 68 ++++++++++++++----- .../server/tests/unit/with_dbs/conftest.py | 4 +- 4 files changed, 56 insertions(+), 71 deletions(-) delete mode 100644 services/web/server/tests/data/test_activity_config.yml diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index 26bd1d6f5dde..c474c48ab572 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -292,6 +292,8 @@ async def _creator( parent_project_uuid: ProjectID | None = None, parent_node_id: NodeID | None = None, ) -> ProjectDict: + assert client.app + url, project_data, expected_data, headers = await _setup( client, project=project, diff --git a/services/web/server/tests/data/test_activity_config.yml b/services/web/server/tests/data/test_activity_config.yml deleted file mode 100644 index 7fc30d838681..000000000000 --- a/services/web/server/tests/data/test_activity_config.yml +++ /dev/null @@ -1,53 +0,0 @@ -version: "1.0" -main: - host: 127.0.0.1 - log_level: DEBUG - port: 8080 - testing: true - studies_access_enabled: True -director: - host: director - port: 8001 - version: v0 -db: - postgres: - database: simcoredb - endpoint: postgres:5432 - host: postgres - maxsize: 10 - minsize: 10 - password: simcore - port: 5432 - user: simcore -# s3: -# access_key: 'Q3AM3UQ867SPQQA43P2F' -# bucket_name: simcore -# endpoint: play.minio.io:9000 -# secret_key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' -login: - enabled: False - registration_invitation_required: False - registration_confirmation_required: True -smtp: - sender: "OSPARC support " - host: mail.foo.com - port: 25 - tls: False - username: Null - password: Null -storage: - host: storage - port: 11111 - version: v0 -rest: - version: v0 -projects: - enabled: True -session: - secret_key: "REPLACE_ME_with_result__Fernet_generate_key=" -activity: - enabled: True - prometheus_url: http://prometheus:9090 - prometheus_username: fake - prometheus_password: fake - prometheus_api_version: v1 diff --git a/services/web/server/tests/unit/isolated/test_activity.py b/services/web/server/tests/unit/isolated/test_activity.py index b8d97b92c67d..108e99a403f3 100644 --- a/services/web/server/tests/unit/isolated/test_activity.py +++ b/services/web/server/tests/unit/isolated/test_activity.py @@ -3,21 +3,25 @@ # pylint:disable=redefined-outer-name import asyncio +import os from collections.abc import Callable -from pathlib import Path from typing import Any from unittest.mock import MagicMock import pytest -import yaml from aiohttp.client_exceptions import ClientConnectionError from aiohttp.test_utils import TestClient from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status from servicelib.aiohttp.application import create_safe_application from simcore_service_webserver.activity.plugin import setup_activity -from simcore_service_webserver.application_settings import setup_settings +from simcore_service_webserver.application_settings import ( + PrometheusSettings, + setup_settings, +) from simcore_service_webserver.rest.plugin import setup_rest from simcore_service_webserver.security.plugin import setup_security from simcore_service_webserver.session.plugin import setup_session @@ -55,29 +59,61 @@ def mocked_monitoring_down(mocker: MockerFixture) -> None: @pytest.fixture -def app_config(fake_data_dir: Path, osparc_simcore_root_dir: Path) -> dict[str, Any]: - with Path.open(fake_data_dir / "test_activity_config.yml") as fh: - content = fh.read() - config = content.replace( - "${OSPARC_SIMCORE_REPO_ROOTDIR}", str(osparc_simcore_root_dir) - ) +def app_environment( + mock_env_devel_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +) -> EnvVarsDict: + + envs = mock_env_devel_environment | setenvs_from_dict( + monkeypatch, + { + "LOGIN_REGISTRATION_CONFIRMATION_REQUIRED": "True", + "LOGIN_REGISTRATION_INVITATION_REQUIRED": "False", + "POSTGRES_DB": "simcoredb", + "POSTGRES_HOST": "postgres", + "POSTGRES_MAXSIZE": "10", + "POSTGRES_MINSIZE": "10", + "POSTGRES_PASSWORD": "simcore", + "POSTGRES_PORT": "5432", + "POSTGRES_USER": "simcore", + "PROMETHEUS_PASSWORD": "fake", + "PROMETHEUS_URL": "http://prometheus:9090", + "PROMETHEUS_USERNAME": "fake", + "PROMETHEUS_VTAG": "v1", + "SESSION_SECRET_KEY": "REPLACE_ME_with_result__Fernet_generate_key=", + "SMTP_HOST": "mail.foo.com", + "SMTP_PORT": "25", + "STORAGE_HOST": "storage", + "STORAGE_PORT": "11111", + "STORAGE_VTAG": "v0", + "WEBSERVER_LOGIN": "null", + "WEBSERVER_LOGLEVEL": "DEBUG", + "WEBSERVER_PORT": "8080", + "WEBSERVER_STUDIES_ACCESS_ENABLED": "True", + }, + ) + + monkeypatch.delenv("WEBSERVER_ACTIVITY") + envs.pop("WEBSERVER_ACTIVITY") - return yaml.safe_load(config) + return envs @pytest.fixture def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, - app_config: dict[str, Any], mock_orphaned_services: MagicMock, - monkeypatch_setenv_from_app_config: Callable, + app_environment: EnvVarsDict, ): - monkeypatch_setenv_from_app_config(app_config) + # app_environment are in place + assert {key: os.environ[key] for key in app_environment} == app_environment + expected_activity_settings = PrometheusSettings.create_from_envs() + + app = create_safe_application() - app = create_safe_application(app_config) + settings = setup_settings(app) + assert expected_activity_settings == settings.WEBSERVER_ACTIVITY - assert setup_settings(app) setup_session(app) setup_security(app) setup_rest(app) @@ -92,7 +128,7 @@ async def test_has_login_required(client: TestClient): async def test_monitoring_up( - mocked_login_required: None, mocked_monitoring: None, client + mocked_login_required: None, mocked_monitoring: None, client: TestClient ): RUNNING_NODE_ID = "894dd8d5-de3b-4767-950c-7c3ed8f51d8c" diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 6661af40d5e3..31af94ce8e4e 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -136,7 +136,7 @@ def app_cfg(default_app_cfg: AppConfigDict, unused_tcp_port_factory) -> AppConfi def app_environment( monkeypatch: pytest.MonkeyPatch, app_cfg: AppConfigDict, - monkeypatch_setenv_from_app_config: Callable[[AppConfigDict], dict[str, str]], + monkeypatch_setenv_from_app_config: Callable[[AppConfigDict], EnvVarsDict], ) -> EnvVarsDict: # WARNING: this fixture is commonly overriden. Check before renaming. """overridable fixture that defines the ENV for the webserver application @@ -144,7 +144,7 @@ def app_environment( override like so: @pytest.fixture - def app_environment(app_environment: dict[str, str], monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: + def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: monkeypatch.setenv("MODIFIED_ENV", "VALUE") return app_environment | {"MODIFIED_ENV":"VALUE"} """ From c3eac3b806f4a989f636aa7ddbc9d4df11017f41 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:58:33 +0100 Subject: [PATCH 07/71] fixe env devel --- services/web/server/tests/unit/with_dbs/conftest.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 31af94ce8e4e..9e792ac9d20b 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -136,6 +136,7 @@ def app_cfg(default_app_cfg: AppConfigDict, unused_tcp_port_factory) -> AppConfi def app_environment( monkeypatch: pytest.MonkeyPatch, app_cfg: AppConfigDict, + mock_env_devel_environment: EnvVarsDict, monkeypatch_setenv_from_app_config: Callable[[AppConfigDict], EnvVarsDict], ) -> EnvVarsDict: # WARNING: this fixture is commonly overriden. Check before renaming. @@ -148,9 +149,10 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc monkeypatch.setenv("MODIFIED_ENV", "VALUE") return app_environment | {"MODIFIED_ENV":"VALUE"} """ - print("+ web_server:") - cfg = deepcopy(app_cfg) - envs = monkeypatch_setenv_from_app_config(cfg) + + envs = mock_env_devel_environment | monkeypatch_setenv_from_app_config( + deepcopy(app_cfg) + ) # # NOTE: this emulates hostname: "wb-{{.Node.Hostname}}-{{.Task.Slot}}" in docker-compose that From eab8b6fb8abd7e25f09f4687bc1907a255ff86dd Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:36:05 +0100 Subject: [PATCH 08/71] static --- .../server/tests/unit/with_dbs/01/test_statics.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_statics.py b/services/web/server/tests/unit/with_dbs/01/test_statics.py index 1eb8212d986f..33a666b06b54 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_statics.py +++ b/services/web/server/tests/unit/with_dbs/01/test_statics.py @@ -16,7 +16,6 @@ from servicelib.aiohttp import status from servicelib.aiohttp.application import create_safe_application from simcore_postgres_database.models.products import products -from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.application_settings import setup_settings from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.db.plugin import setup_db @@ -55,25 +54,18 @@ def client( aiohttp_client: Callable, app_cfg: AppConfigDict, postgres_db: sa.engine.Engine, - monkeypatch_setenv_from_app_config: Callable, ) -> TestClient: cfg = deepcopy(app_cfg) - port = cfg["main"]["port"] - assert cfg["rest"]["version"] == API_VTAG - monkeypatch_setenv_from_app_config(cfg) - - # fake config - app = create_safe_application(cfg) + app = create_safe_application() settings = setup_settings(app) - print(settings.model_dump_json(indent=1)) - + assert settings.WEBSERVER_STATICWEB setup_rest(app) setup_db(app) setup_products(app) - setup_statics(app) + assert setup_statics(app) return event_loop.run_until_complete( aiohttp_client(app, server_kwargs={"port": port, "host": "localhost"}) From fc434bf0992e32cc82a330e5805aeb3ac04cc254 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 16:42:40 +0100 Subject: [PATCH 09/71] fixes app_environment in tags --- .../tests/unit/with_dbs/03/tags/conftest.py | 50 ++++++++++++------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py index 45682405bfe8..4d77bb5cfecf 100644 --- a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py @@ -2,16 +2,19 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import asyncio from collections.abc import AsyncIterator, Callable from copy import deepcopy from pathlib import Path import pytest from aioresponses import aioresponses +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects from servicelib.aiohttp.application import create_safe_application from simcore_service_webserver.application_settings import setup_settings +from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.db.plugin import setup_db from simcore_service_webserver.director_v2.plugin import setup_director_v2 from simcore_service_webserver.login.plugin import setup_login @@ -35,19 +38,14 @@ @pytest.fixture -def client( - event_loop, - aiohttp_client, - app_cfg, - postgres_db, - mocked_dynamic_services_interface, - mock_orphaned_services, - redis_client, # this ensure redis is properly cleaned - monkeypatch_setenv_from_app_config: Callable, -): - # config app +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_cfg: AppConfigDict, + app_environment: EnvVarsDict, + monkeypatch_setenv_from_app_config: Callable[[AppConfigDict], EnvVarsDict], +) -> EnvVarsDict: cfg = deepcopy(app_cfg) - port = cfg["main"]["port"] + cfg["projects"]["enabled"] = True cfg["resource_manager"][ "garbage_collection_interval_seconds" @@ -56,11 +54,25 @@ def client( "resource_deletion_timeout_seconds" ] = DEFAULT_GARBAGE_COLLECTOR_DELETION_TIMEOUT_SECONDS # reduce deletion delay - monkeypatch_setenv_from_app_config(cfg) + return app_environment | monkeypatch_setenv_from_app_config(cfg) + - app = create_safe_application(cfg) +@pytest.fixture +def client( + event_loop: asyncio.AbstractEventLoop, + aiohttp_client: Callable, + app_cfg: AppConfigDict, + app_environment: EnvVarsDict, + postgres_db, + mocked_dynamic_services_interface, + mock_orphaned_services, + redis_client, # this ensure redis is properly cleaned +): + # config app + app = create_safe_application() - assert setup_settings(app) + settings = setup_settings(app) + assert settings.WEBSERVER_TAGS is not None # setup app setup_db(app) @@ -71,14 +83,16 @@ def client( setup_resource_manager(app) setup_socketio(app) setup_director_v2(app) - setup_tags(app) - assert setup_projects(app) + assert setup_tags(app) + setup_projects(app) setup_products(app) setup_wallets(app) # server and client return event_loop.run_until_complete( - aiohttp_client(app, server_kwargs={"port": port, "host": "localhost"}) + aiohttp_client( + app, server_kwargs={"port": app_cfg["main"]["port"], "host": "localhost"} + ) ) # teardown here ... From 66bc63b82c491b3840b8af2a41f9be399c614900 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 17:03:15 +0100 Subject: [PATCH 10/71] fixes app_environment in gc --- .../test_resource_manager.py | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py index 18cba7971775..ac1866f4ee13 100644 --- a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py @@ -8,7 +8,6 @@ import asyncio from asyncio import Future from collections.abc import AsyncIterator, Awaitable, Callable -from copy import deepcopy from pathlib import Path from typing import Any from unittest import mock @@ -28,6 +27,7 @@ from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_parametrizations import MockedStorageSubsystem from pytest_simcore.helpers.webserver_projects import NewProject @@ -37,8 +37,8 @@ from servicelib.aiohttp.application_setup import is_setup_completed from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_postgres_database.models.users import UserRole -from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.application_settings import setup_settings +from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.db.plugin import setup_db from simcore_service_webserver.director_v2.plugin import setup_director_v2 from simcore_service_webserver.garbage_collector import _core as gc_core @@ -104,45 +104,44 @@ async def _open_project(client, project_uuid: str, client_session_id: str) -> No @pytest.fixture def app_environment( - app_environment: dict[str, str], monkeypatch: pytest.MonkeyPatch -) -> dict[str, str]: - overrides = setenvs_from_dict( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + + # NOTE: undos some app_environment settings + monkeypatch.delenv("WEBSERVER_GARBAGE_COLLECTOR", raising=False) + app_environment.pop("WEBSERVER_GARBAGE_COLLECTOR", None) + + return app_environment | setenvs_from_dict( monkeypatch, { "WEBSERVER_COMPUTATION": "1", "WEBSERVER_NOTIFICATIONS": "1", + # sets TTL of a resource after logout + "RESOURCE_MANAGER_RESOURCE_TTL_S": f"{SERVICE_DELETION_DELAY}", + "GARBAGE_COLLECTOR_INTERVAL_S": "30", }, ) - return app_environment | overrides @pytest.fixture def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, - app_cfg: dict[str, Any], + app_cfg: AppConfigDict, + app_environment: EnvVarsDict, postgres_db: sa.engine.Engine, mock_orphaned_services, redis_client: Redis, - monkeypatch_setenv_from_app_config: Callable, mock_dynamic_scheduler_rabbitmq: None, ) -> TestClient: - cfg = deepcopy(app_cfg) - assert cfg["rest"]["version"] == API_VTAG - assert cfg["rest"]["enabled"] - cfg["projects"]["enabled"] = True - - # sets TTL of a resource after logout - cfg["resource_manager"][ - "resource_deletion_timeout_seconds" - ] = SERVICE_DELETION_DELAY - - monkeypatch_setenv_from_app_config(cfg) - app = create_safe_application(cfg) + app = create_safe_application() - # activates only security+restAPI sub-modules + assert "WEBSERVER_GARBAGE_COLLECTOR" not in app_environment - assert setup_settings(app) + settings = setup_settings(app) + assert settings.WEBSERVER_GARBAGE_COLLECTOR is not None + assert settings.WEBSERVER_PROJECTS is not None setup_db(app) setup_session(app) @@ -151,7 +150,7 @@ def client( setup_login(app) setup_users(app) setup_socketio(app) - setup_projects(app) + assert setup_projects(app) setup_director_v2(app) assert setup_resource_manager(app) setup_rabbitmq(app) @@ -167,7 +166,10 @@ def client( return event_loop.run_until_complete( aiohttp_client( app, - server_kwargs={"port": cfg["main"]["port"], "host": cfg["main"]["host"]}, + server_kwargs={ + "port": app_cfg["main"]["port"], + "host": app_cfg["main"]["host"], + }, ) ) From 5cb9187ce530b0564afe0ce375e0da388c28c187 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:04:05 +0100 Subject: [PATCH 11/71] rm app_cfg --- .../tests/unit/with_dbs/03/tags/conftest.py | 30 +++--- .../with_dbs/03/version_control/conftest.py | 93 ++++++------------- .../test_version_control_handlers.py | 16 ++-- 3 files changed, 54 insertions(+), 85 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py index 4d77bb5cfecf..43c6da4f3447 100644 --- a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py @@ -9,6 +9,7 @@ import pytest from aioresponses import aioresponses +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects @@ -40,21 +41,21 @@ @pytest.fixture def app_environment( monkeypatch: pytest.MonkeyPatch, - app_cfg: AppConfigDict, app_environment: EnvVarsDict, - monkeypatch_setenv_from_app_config: Callable[[AppConfigDict], EnvVarsDict], ) -> EnvVarsDict: - cfg = deepcopy(app_cfg) - - cfg["projects"]["enabled"] = True - cfg["resource_manager"][ - "garbage_collection_interval_seconds" - ] = DEFAULT_GARBAGE_COLLECTOR_INTERVAL_SECONDS # increase speed of garbage collection - cfg["resource_manager"][ - "resource_deletion_timeout_seconds" - ] = DEFAULT_GARBAGE_COLLECTOR_DELETION_TIMEOUT_SECONDS # reduce deletion delay + # NOTE: undos some app_environment settings + monkeypatch.delenv("WEBSERVER_GARBAGE_COLLECTOR", raising=False) + app_environment.pop("WEBSERVER_GARBAGE_COLLECTOR", None) - return app_environment | monkeypatch_setenv_from_app_config(cfg) + return app_environment | setenvs_from_dict( + monkeypatch, + { + # reduce deletion delay + "RESOURCE_MANAGER_RESOURCE_TTL_S": f"{DEFAULT_GARBAGE_COLLECTOR_INTERVAL_SECONDS}", + # increase speed of garbage collection + "GARBAGE_COLLECTOR_INTERVAL_S": f"{DEFAULT_GARBAGE_COLLECTOR_DELETION_TIMEOUT_SECONDS}", + }, + ) @pytest.fixture @@ -68,10 +69,13 @@ def client( mock_orphaned_services, redis_client, # this ensure redis is properly cleaned ): - # config app app = create_safe_application() + assert "WEBSERVER_GARBAGE_COLLECTOR" not in app_environment + settings = setup_settings(app) + assert settings.WEBSERVER_GARBAGE_COLLECTOR is not None + assert settings.WEBSERVER_PROJECTS is not None assert settings.WEBSERVER_TAGS is not None # setup app diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index e9a6244886c7..927d574197e9 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -2,11 +2,8 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -import logging from collections.abc import AsyncIterator, Awaitable, Callable -from copy import deepcopy from pathlib import Path -from typing import Any from unittest import mock from uuid import UUID @@ -21,6 +18,8 @@ from models_library.utils.fastapi_encoders import jsonable_encoder from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_factories import random_project +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_projects import NewProject from servicelib.aiohttp import status @@ -31,7 +30,6 @@ from simcore_service_webserver._meta import API_VTAG as VX from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.db.plugin import APP_AIOPG_ENGINE_KEY -from simcore_service_webserver.log import setup_logging from simcore_service_webserver.projects.db import ProjectDBAPI from simcore_service_webserver.projects.models import ProjectDict from tenacity.asyncio import AsyncRetrying @@ -69,70 +67,35 @@ def catalog_subsystem_mock_override( @pytest.fixture -def app_cfg( - default_app_cfg, - unused_tcp_port_factory, +def app_environment( catalog_subsystem_mock_override: None, - monkeypatch, -) -> dict[str, Any]: - """App's configuration used for every test in this module - - NOTE: Overrides services/web/server/tests/unit/with_dbs/conftest.py::app_cfg to influence app setup - """ - cfg = deepcopy(default_app_cfg) - - monkeypatch.setenv("WEBSERVER_DEV_FEATURES_ENABLED", "1") - - cfg["main"]["port"] = unused_tcp_port_factory() - cfg["main"]["studies_access_enabled"] = True - - exclude = { - "activity", - "clusters", - "computation", - "diagnostics", - "groups", - "publications", - "garbage_collector", - "smtp", - "socketio", - "storage", - "studies_dispatcher", - "tags", - "tracing", - } - include = { - "catalog", - "db", - "login", - "products", - "projects", - "resource_manager", - "rest", - "users", - "version_control", # MODULE UNDER TEST - } - - assert include.intersection(exclude) == set() - - for section in include: - cfg[section]["enabled"] = True - for section in exclude: - cfg[section]["enabled"] = False - - # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG - setup_logging( - level=logging.DEBUG, - log_format_local_dev_enabled=True, - logger_filter_mapping={}, - tracing_settings=None, + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + + return app_environment | setenvs_from_dict( + monkeypatch, + { + # exclude + "WEBSERVER_ACTIVITY": "null", + "WEBSERVER_CLUSTERS": "null", + "WEBSERVER_COMPUTATION": "null", + "WEBSERVER_DIAGNOSTICS": "null", + "WEBSERVER_GROUPS": "0", + "WEBSERVER_PUBLICATIONS": "0", + "WEBSERVER_GARBAGE_COLLECTOR": "null", + "WEBSERVER_SMTP": "null", + "WEBSERVER_SOCKETIO": "0", + "WEBSERVER_STORAGE": "null", + "WEBSERVER_STUDIES_DISPATCHER": "null", + "WEBSERVER_TAGS": "0", + "WEBSERVER_TRACING": "null", + # Module under test + "WEBSERVER_DEV_FEATURES_ENABLED": "1", + "WEBSERVER_VERSION_CONTROL": "1", + }, ) - # Enforces smallest GC in the background task - cfg["resource_manager"]["garbage_collection_interval_seconds"] = 1 - - return cfg - @pytest.fixture async def user_id(logged_user: UserInfoDict) -> UserID: diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py index 325eb63e353e..ac229a3b410d 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py @@ -1,6 +1,8 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable +# pylint: disable=too-many-arguments + from collections.abc import Awaitable, Callable from http import HTTPStatus @@ -23,7 +25,7 @@ ) -async def assert_resp_page( +async def _assert_resp_page( resp: aiohttp.ClientResponse, expected_page_cls: type[Page], expected_total: int, @@ -38,7 +40,7 @@ async def assert_resp_page( return page -async def assert_status_and_body( +async def _assert_status_and_body( resp, expected_cls: HTTPStatus, expected_model: type[BaseModel] ) -> BaseModel: data, _ = await assert_status(resp, expected_cls) @@ -84,7 +86,7 @@ async def test_workflow( # # this project now has a repo resp = await client.get(f"/{VX}/repos/projects") - page = await assert_resp_page( + page = await _assert_resp_page( resp, expected_page_cls=Page[ProjectDict], expected_total=1, expected_count=1 ) @@ -97,8 +99,8 @@ async def test_workflow( assert CheckpointApiModel.model_validate(data) == checkpoint1 # TODO: GET checkpoint with tag + resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints/v1") with pytest.raises(aiohttp.ClientResponseError) as excinfo: - resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints/v1") resp.raise_for_status() assert CheckpointApiModel.model_validate(data) == checkpoint1 @@ -114,7 +116,7 @@ async def test_workflow( # LIST checkpoints resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints") - page = await assert_resp_page( + page = await _assert_resp_page( resp, expected_page_cls=Page[CheckpointApiModel], expected_total=1, @@ -226,7 +228,7 @@ async def test_delete_project_and_repo( # LIST resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints") - await assert_resp_page( + await _assert_resp_page( resp, expected_page_cls=Page[CheckpointApiModel], expected_total=1, @@ -247,7 +249,7 @@ async def test_delete_project_and_repo( # LIST empty resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints") - await assert_resp_page( + await _assert_resp_page( resp, expected_page_cls=Page[CheckpointApiModel], expected_total=0, From 8378ef52490e158bac7465bde07c84d9b991d0d8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:09:54 +0100 Subject: [PATCH 12/71] rm cfg from meta_modeling --- .../with_dbs/03/meta_modeling/conftest.py | 95 ++++++------------- 1 file changed, 29 insertions(+), 66 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py index 3b965a09e46b..ebb469c94bf8 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py @@ -1,80 +1,43 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable - -import logging -from copy import deepcopy -from typing import Any - +# pylint: disable=too-many-arguments import pytest +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_postgres_database.models.users import UserRole -from simcore_service_webserver.log import setup_logging @pytest.fixture def user_role() -> UserRole: - # TODO: user rights still not in place return UserRole.TESTER @pytest.fixture -def app_cfg(default_app_cfg, unused_tcp_port_factory, monkeypatch) -> dict[str, Any]: - """App's configuration used for every test in this module - - NOTE: Overrides services/web/server/tests/unit/with_dbs/conftest.py::app_cfg to influence app setup - """ - cfg = deepcopy(default_app_cfg) - - monkeypatch.setenv("WEBSERVER_DEV_FEATURES_ENABLED", "1") - - cfg["main"]["port"] = unused_tcp_port_factory() - cfg["main"]["studies_access_enabled"] = True - - exclude = { - "activity", - "clusters", - "computation", - "diagnostics", - "garbage_collector", - "groups", - "publications", - "smtp", - "socketio", - "storage", - "studies_dispatcher", - "tags", - "tracing", - } - include = { - "catalog", - "db", - "login", - "meta_modeling", # MODULE UNDER TEST - "products", - "projects", - "redis", - "resource_manager", - "rest", - "users", - "version_control", - } - - assert include.intersection(exclude) == set() - - for section in include: - cfg[section]["enabled"] = True - for section in exclude: - cfg[section]["enabled"] = False - - # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG - setup_logging( - level=logging.DEBUG, - log_format_local_dev_enabled=True, - logger_filter_mapping={}, - tracing_settings=None, +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + return app_environment | setenvs_from_dict( + monkeypatch, + { + # exclude + "WEBSERVER_ACTIVITY": "null", + "WEBSERVER_CLUSTERS": "null", + "WEBSERVER_COMPUTATION": "null", + "WEBSERVER_DIAGNOSTICS": "null", + "WEBSERVER_GROUPS": "0", + "WEBSERVER_PUBLICATIONS": "0", + "WEBSERVER_GARBAGE_COLLECTOR": "null", + "WEBSERVER_SMTP": "null", + "WEBSERVER_SOCKETIO": "0", + "WEBSERVER_STORAGE": "null", + "WEBSERVER_STUDIES_DISPATCHER": "null", + "WEBSERVER_TAGS": "0", + "WEBSERVER_TRACING": "null", + # Module under test + "WEBSERVER_DEV_FEATURES_ENABLED": "1", + "WEBSERVER_VERSION_CONTROL": "1", + "WEBSERVER_META_MODELING": "1", + }, ) - - # Enforces smallest GC in the background task - cfg["resource_manager"]["garbage_collection_interval_seconds"] = 1 - - return cfg From 026ffb5fea5c520cb8db406b0039bf1bb044296f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:16:26 +0100 Subject: [PATCH 13/71] cleanup --- .../01/test_groups_handlers_classifers.py | 87 ++++++++----------- .../with_dbs/03/meta_modeling/conftest.py | 2 +- .../with_dbs/03/version_control/conftest.py | 2 +- 3 files changed, 40 insertions(+), 51 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_groups_handlers_classifers.py b/services/web/server/tests/unit/with_dbs/01/test_groups_handlers_classifers.py index c7367b03b942..02a0ddf581ba 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_groups_handlers_classifers.py +++ b/services/web/server/tests/unit/with_dbs/01/test_groups_handlers_classifers.py @@ -1,61 +1,50 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments import re -from copy import deepcopy import pytest from aiohttp import web_exceptions from aioresponses.core import aioresponses -from simcore_service_webserver.application_settings_utils import AppConfigDict +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict @pytest.fixture -def app_cfg(default_app_cfg: AppConfigDict, unused_tcp_port_factory): - """App's configuration used for every test in this module - - NOTE: Overrides services/web/server/tests/unit/with_dbs/conftest.py::app_cfg to influence app setup - """ - cfg = deepcopy(default_app_cfg) - - cfg["main"]["port"] = unused_tcp_port_factory() - cfg["main"]["studies_access_enabled"] = True - - exclude = { - "tracing", - "smtp", - "storage", - "activity", - "diagnostics", - "tags", - "publications", - "catalog", - "computation", - "products", - "socketio", - "resource_manager", - "projects", - "login", - "users", - } - include = { - "db", - "rest", - "groups", - } - - assert include.intersection(exclude) == set() - - for section in include: - cfg[section]["enabled"] = True - for section in exclude: - cfg[section]["enabled"] = False - - # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG - ## setup_logging(level=logging.DEBUG) - - return cfg +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + + monkeypatch.delenv("WEBSERVER_STUDIES_DISPATCHER", raising=False) + app_environment.pop("WEBSERVER_STUDIES_DISPATCHER", None) + + return app_environment | setenvs_from_dict( + monkeypatch, + { + # exclude + "WEBSERVER_ACTIVITY": "null", + "WEBSERVER_CATALOG": "null", + "WEBSERVER_CLUSTERS": "null", + "WEBSERVER_COMPUTATION": "null", + "WEBSERVER_DIAGNOSTICS": "null", + "WEBSERVER_EMAIL": "null", + "WEBSERVER_GARBAGE_COLLECTOR": "null", + "WEBSERVER_GROUPS": "0", + "WEBSERVER_LOGIN": "null", + "WEBSERVER_PRODUCTS": "0", + "WEBSERVER_PROJECTS": "null", + "WEBSERVER_PUBLICATIONS": "0", + "WEBSERVER_SOCKETIO": "0", + "WEBSERVER_STORAGE": "null", + "WEBSERVER_RESOURCE_MANAGER": "null", + "WEBSERVER_TAGS": "0", + "WEBSERVER_TRACING": "null", + "WEBSERVER_USERS": "null", + }, + ) @pytest.mark.skip(reason="UNDER DEV: test_group_handlers") diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py index ebb469c94bf8..d7e3dc7529e0 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py @@ -29,7 +29,7 @@ def app_environment( "WEBSERVER_GROUPS": "0", "WEBSERVER_PUBLICATIONS": "0", "WEBSERVER_GARBAGE_COLLECTOR": "null", - "WEBSERVER_SMTP": "null", + "WEBSERVER_EMAIL": "null", "WEBSERVER_SOCKETIO": "0", "WEBSERVER_STORAGE": "null", "WEBSERVER_STUDIES_DISPATCHER": "null", diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index 927d574197e9..00f35441f8d4 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -84,7 +84,7 @@ def app_environment( "WEBSERVER_GROUPS": "0", "WEBSERVER_PUBLICATIONS": "0", "WEBSERVER_GARBAGE_COLLECTOR": "null", - "WEBSERVER_SMTP": "null", + "WEBSERVER_EMAIL": "null", "WEBSERVER_SOCKETIO": "0", "WEBSERVER_STORAGE": "null", "WEBSERVER_STUDIES_DISPATCHER": "null", From 04e537f2a26fa0078736368f87d37b26c1f8411d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:20:41 +0100 Subject: [PATCH 14/71] enables trash plugin --- .../web/server/src/simcore_service_webserver/application.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/web/server/src/simcore_service_webserver/application.py b/services/web/server/src/simcore_service_webserver/application.py index a868e3453807..e8825167f66d 100644 --- a/services/web/server/src/simcore_service_webserver/application.py +++ b/services/web/server/src/simcore_service_webserver/application.py @@ -49,6 +49,7 @@ from .studies_dispatcher.plugin import setup_studies_dispatcher from .tags.plugin import setup_tags from .tracing import setup_app_tracing +from .trash.plugin import setup_trash from .users.plugin import setup_users from .version_control.plugin import setup_version_control from .wallets.plugin import setup_wallets @@ -143,6 +144,9 @@ def create_application() -> web.Application: # licenses setup_licenses(app) + # trash add-on + setup_trash(app) + # tagging setup_scicrunch(app) setup_tags(app) From 2c4eb3741768ba648f23cab2ac457433357ef36a Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:31:16 +0100 Subject: [PATCH 15/71] common.models --- api/specs/web-server/_projects_crud.py | 2 +- api/specs/web-server/_projects_groups.py | 2 +- api/specs/web-server/_projects_wallet.py | 2 +- .../application_settings.py | 38 ++++++++++++------- .../projects/_comments_handlers.py | 2 +- .../projects/_common/__init__.py | 0 .../projects/_common/models.py | 26 +++++++++++++ .../projects/_crud_handlers.py | 2 +- .../projects/_folders_handlers.py | 10 ++--- .../projects/_groups_handlers.py | 2 +- .../projects/_metadata_handlers.py | 2 +- .../projects/_nodes_handlers.py | 2 +- .../projects/_ports_handlers.py | 2 +- .../_projects_nodes_pricing_unit_handlers.py | 2 +- .../projects/_states_handlers.py | 2 +- .../projects/_trash_handlers.py | 3 +- .../projects/_wallets_handlers.py | 2 +- .../projects/_workspaces_handlers.py | 2 +- 18 files changed, 69 insertions(+), 34 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/projects/_common/__init__.py create mode 100644 services/web/server/src/simcore_service_webserver/projects/_common/models.py diff --git a/api/specs/web-server/_projects_crud.py b/api/specs/web-server/_projects_crud.py index 31f26d6425e2..62fe7684c682 100644 --- a/api/specs/web-server/_projects_crud.py +++ b/api/specs/web-server/_projects_crud.py @@ -30,7 +30,7 @@ from models_library.rest_pagination import Page from pydantic import BaseModel from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.projects._common_models import ProjectPathParams +from simcore_service_webserver.projects._common.models import ProjectPathParams from simcore_service_webserver.projects._crud_handlers import ProjectCreateParams from simcore_service_webserver.projects._crud_handlers_models import ( ProjectActiveQueryParams, diff --git a/api/specs/web-server/_projects_groups.py b/api/specs/web-server/_projects_groups.py index 88432bb3cfac..cfc0870d6a81 100644 --- a/api/specs/web-server/_projects_groups.py +++ b/api/specs/web-server/_projects_groups.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, status from models_library.generics import Envelope from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.projects._common_models import ProjectPathParams +from simcore_service_webserver.projects._common.models import ProjectPathParams from simcore_service_webserver.projects._groups_api import ProjectGroupGet from simcore_service_webserver.projects._groups_handlers import ( _ProjectsGroupsBodyParams, diff --git a/api/specs/web-server/_projects_wallet.py b/api/specs/web-server/_projects_wallet.py index e079dd33e171..c9502393b971 100644 --- a/api/specs/web-server/_projects_wallet.py +++ b/api/specs/web-server/_projects_wallet.py @@ -16,7 +16,7 @@ from models_library.projects import ProjectID from models_library.wallets import WalletID from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.projects._common_models import ProjectPathParams +from simcore_service_webserver.projects._common.models import ProjectPathParams router = APIRouter( prefix=f"/{API_VTAG}", diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index 07941e1e92c6..f88de60ff097 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -175,8 +175,13 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): json_schema_extra={"auto_default_from_env": True}, description="director-v2 service client's plugin", ) + + WEBSERVER_DYNAMIC_SCHEDULER: DynamicSchedulerSettings | None = Field( + json_schema_extra={"auto_default_from_env": True}, + ) + WEBSERVER_EMAIL: SMTPSettings | None = Field( - json_schema_extra={"auto_default_from_env": True}, description="email plugin" + json_schema_extra={"auto_default_from_env": True} ) WEBSERVER_EXPORTER: ExporterSettings | None = Field( json_schema_extra={"auto_default_from_env": True}, description="exporter plugin" @@ -204,9 +209,8 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="payments plugin settings", ) - WEBSERVER_DYNAMIC_SCHEDULER: DynamicSchedulerSettings | None = Field( - description="dynamic-scheduler plugin settings", - json_schema_extra={"auto_default_from_env": True}, + WEBSERVER_PROJECTS: ProjectsSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} ) WEBSERVER_REDIS: RedisSettings | None = Field( @@ -254,15 +258,19 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="tracing plugin", json_schema_extra={"auto_default_from_env": True} ) - WEBSERVER_PROJECTS: ProjectsSettings | None = Field( - description="projects plugin", json_schema_extra={"auto_default_from_env": True} - ) - WEBSERVER_RABBITMQ: RabbitSettings | None = Field( - description="rabbitmq plugin", json_schema_extra={"auto_default_from_env": True} - ) - WEBSERVER_USERS: UsersSettings | None = Field( - description="users plugin", json_schema_extra={"auto_default_from_env": True} - ) + WEBSERVER_RABBITMQ: Annotated[ + RabbitSettings | None, + Field( + json_schema_extra={"auto_default_from_env": True}, + ), + ] + + WEBSERVER_USERS: Annotated[ + UsersSettings | None, + Field( + json_schema_extra={"auto_default_from_env": True}, + ), + ] # These plugins only require (for the moment) an entry to toggle between enabled/disabled WEBSERVER_ANNOUNCEMENTS: bool = False @@ -440,7 +448,9 @@ def to_client_statics(self) -> dict[str, Any]: "SWARM_STACK_NAME": True, "WEBSERVER_PROJECTS": { "PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES", - "PROJECTS_TRASH_RETENTION_DAYS", + }, + "WEBSERVER_TRASH": { + "TRASH_RETENTION_DAYS", }, "WEBSERVER_LOGIN": { "LOGIN_ACCOUNT_DELETION_RETENTION_DAYS", diff --git a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py index 6ad8b290ba08..c46610057047 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_comments_handlers.py @@ -31,7 +31,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _comments_api, projects_api -from ._common_models import RequestContext +from ._common.models import RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/projects/_common/__init__.py b/services/web/server/src/simcore_service_webserver/projects/_common/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/web/server/src/simcore_service_webserver/projects/_common/models.py b/services/web/server/src/simcore_service_webserver/projects/_common/models.py new file mode 100644 index 000000000000..6f358378f600 --- /dev/null +++ b/services/web/server/src/simcore_service_webserver/projects/_common/models.py @@ -0,0 +1,26 @@ +""" Handlers for STANDARD methods on /projects colletions + +Standard methods or CRUD that states for Create+Read(Get&List)+Update+Delete + +""" + +from models_library.projects import ProjectID +from pydantic import BaseModel, ConfigDict, Field + +from ...models import RequestContext + +assert RequestContext.__name__ # nosec + + +class ProjectPathParams(BaseModel): + project_id: ProjectID + model_config = ConfigDict(populate_by_name=True, extra="forbid") + + +class RemoveQueryParams(BaseModel): + force: bool = Field( + default=False, description="Force removal (even if resource is active)" + ) + + +__all__: tuple[str, ...] = ("RequestContext",) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index 8173a7a2db6d..09be85531976 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -48,7 +48,7 @@ from ..users.api import get_user_fullname from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError from . import _crud_api_create, _crud_api_read, projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from ._crud_handlers_models import ( ProjectActiveQueryParams, ProjectCreateHeaders, diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py index 2e644a4d598f..c4f1828237b8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_handlers.py @@ -5,7 +5,7 @@ from models_library.folders import FolderID from models_library.projects import ProjectID from models_library.utils.common_validators import null_or_none_str_to_none_validator -from pydantic import ConfigDict, BaseModel, field_validator +from pydantic import BaseModel, ConfigDict, field_validator from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as from servicelib.aiohttp.typing_extension import Handler @@ -14,7 +14,7 @@ from ..login.decorators import login_required from ..security.decorators import permission_required from . import _folders_api -from ._common_models import RequestContext +from ._common.models import RequestContext from .exceptions import ProjectGroupNotFoundError, ProjectNotFoundError _logger = logging.getLogger(__name__) @@ -44,9 +44,9 @@ class _ProjectsFoldersPathParams(BaseModel): model_config = ConfigDict(extra="forbid") # validators - _null_or_none_str_to_none_validator = field_validator( - "folder_id", mode="before" - )(null_or_none_str_to_none_validator) + _null_or_none_str_to_none_validator = field_validator("folder_id", mode="before")( + null_or_none_str_to_none_validator + ) @routes.put( diff --git a/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py index bf612944d4b7..d507a2b1eff9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_groups_handlers.py @@ -21,7 +21,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _groups_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from ._groups_api import ProjectGroupGet from .exceptions import ProjectGroupNotFoundError, ProjectNotFoundError diff --git a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py index 802c13f79378..df139c6fd30b 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_metadata_handlers.py @@ -30,7 +30,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _metadata_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .exceptions import ( NodeNotFoundError, ParentNodeNotFoundError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py index 9ddd88c0df1f..514efafa47d2 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py @@ -74,7 +74,7 @@ from ..utils_aiohttp import envelope_json_response from ..wallets.errors import WalletAccessForbiddenError, WalletNotEnoughCreditsError from . import nodes_utils, projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from ._nodes_api import NodeScreenshot, get_node_screenshots from .exceptions import ( ClustersKeeperNotAvailableError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py index eaacd9c1aa3f..db8be1b9cfd0 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_ports_handlers.py @@ -33,7 +33,7 @@ from ..projects._access_rights_api import check_user_project_permission from ..security.decorators import permission_required from . import _ports_api, projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .db import ProjectDBAPI from .exceptions import ( NodeNotFoundError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py index 05bb2f8e7676..2748d81061e8 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_nodes_pricing_unit_handlers.py @@ -21,7 +21,7 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import projects_api -from ._common_models import RequestContext +from ._common.models import RequestContext from ._nodes_handlers import NodePathParams from .db import ProjectDBAPI from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError diff --git a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py index 8ec0400238cb..67a21b23eceb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py @@ -36,7 +36,7 @@ from ..utils_aiohttp import envelope_json_response from ..wallets.errors import WalletNotEnoughCreditsError from . import projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .exceptions import ( DefaultPricingUnitNotFoundError, ProjectInvalidRightsError, diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py index 963b81c49008..b20636121322 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py @@ -16,10 +16,9 @@ ) from ..login.decorators import get_user_id, login_required from ..products.api import get_product_name -from ..projects._common_models import ProjectPathParams from ..security.decorators import permission_required from . import _trash_api -from ._common_models import RemoveQueryParams +from ._common.models import ProjectPathParams, RemoveQueryParams from .exceptions import ProjectRunningConflictError, ProjectStoppingError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py index 56e7136d299d..dd66c65cb8df 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_wallets_handlers.py @@ -20,7 +20,7 @@ from ..wallets.errors import WalletAccessForbiddenError from . import _wallets_api as wallets_api from . import projects_api -from ._common_models import ProjectPathParams, RequestContext +from ._common.models import ProjectPathParams, RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py index ef3d20b3c5ac..b5a6082cb506 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_workspaces_handlers.py @@ -17,7 +17,7 @@ from ..security.decorators import permission_required from ..workspaces.errors import WorkspaceAccessForbiddenError, WorkspaceNotFoundError from . import _workspaces_api -from ._common_models import RequestContext +from ._common.models import RequestContext from .exceptions import ProjectInvalidRightsError, ProjectNotFoundError _logger = logging.getLogger(__name__) From ec92781233fd52f6ac3be73e89d7c7b159604b6c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:31:39 +0100 Subject: [PATCH 16/71] env --- .env-devel | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env-devel b/.env-devel index cc6609460dad..9ba17baebb7a 100644 --- a/.env-devel +++ b/.env-devel @@ -348,10 +348,10 @@ SIMCORE_VCS_RELEASE_TAG=latest STUDIES_ACCESS_ANONYMOUS_ALLOWED=0 STUDIES_DEFAULT_SERVICE_THUMBNAIL=https://via.placeholder.com/170x120.png TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE=2 -TRACING_OPENTELEMETRY_COLLECTOR_PORT=4318 TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT=http://opentelemetry-collector -TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=100 TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT=http://jaeger:4318 +TRACING_OPENTELEMETRY_COLLECTOR_PORT=4318 +TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE=100 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack TWILIO_ACCOUNT_SID=DUMMY TWILIO_AUTH_TOKEN=DUMMY @@ -365,8 +365,8 @@ WEBSERVER_DEV_FEATURES_ENABLED=0 WEBSERVER_DIAGNOSTICS={} WEBSERVER_EMAIL={} WEBSERVER_EXPORTER={} -WEBSERVER_FRONTEND={} WEBSERVER_FOLDERS=1 +WEBSERVER_FRONTEND={} WEBSERVER_GARBAGE_COLLECTOR=null WEBSERVER_GROUPS=1 WEBSERVER_GUNICORN_CMD_ARGS=--timeout=180 From 76ee8a6a758172422e3706d4afef18a8217aa388 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:33:04 +0100 Subject: [PATCH 17/71] test_activity --- services/web/server/tests/conftest.py | 2 + .../tests/data/test_activity_config.yml | 53 --------------- .../tests/unit/isolated/test_activity.py | 68 ++++++++++++++----- 3 files changed, 54 insertions(+), 69 deletions(-) delete mode 100644 services/web/server/tests/data/test_activity_config.yml diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index 26bd1d6f5dde..c474c48ab572 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -292,6 +292,8 @@ async def _creator( parent_project_uuid: ProjectID | None = None, parent_node_id: NodeID | None = None, ) -> ProjectDict: + assert client.app + url, project_data, expected_data, headers = await _setup( client, project=project, diff --git a/services/web/server/tests/data/test_activity_config.yml b/services/web/server/tests/data/test_activity_config.yml deleted file mode 100644 index 7fc30d838681..000000000000 --- a/services/web/server/tests/data/test_activity_config.yml +++ /dev/null @@ -1,53 +0,0 @@ -version: "1.0" -main: - host: 127.0.0.1 - log_level: DEBUG - port: 8080 - testing: true - studies_access_enabled: True -director: - host: director - port: 8001 - version: v0 -db: - postgres: - database: simcoredb - endpoint: postgres:5432 - host: postgres - maxsize: 10 - minsize: 10 - password: simcore - port: 5432 - user: simcore -# s3: -# access_key: 'Q3AM3UQ867SPQQA43P2F' -# bucket_name: simcore -# endpoint: play.minio.io:9000 -# secret_key: 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG' -login: - enabled: False - registration_invitation_required: False - registration_confirmation_required: True -smtp: - sender: "OSPARC support " - host: mail.foo.com - port: 25 - tls: False - username: Null - password: Null -storage: - host: storage - port: 11111 - version: v0 -rest: - version: v0 -projects: - enabled: True -session: - secret_key: "REPLACE_ME_with_result__Fernet_generate_key=" -activity: - enabled: True - prometheus_url: http://prometheus:9090 - prometheus_username: fake - prometheus_password: fake - prometheus_api_version: v1 diff --git a/services/web/server/tests/unit/isolated/test_activity.py b/services/web/server/tests/unit/isolated/test_activity.py index b8d97b92c67d..108e99a403f3 100644 --- a/services/web/server/tests/unit/isolated/test_activity.py +++ b/services/web/server/tests/unit/isolated/test_activity.py @@ -3,21 +3,25 @@ # pylint:disable=redefined-outer-name import asyncio +import os from collections.abc import Callable -from pathlib import Path from typing import Any from unittest.mock import MagicMock import pytest -import yaml from aiohttp.client_exceptions import ClientConnectionError from aiohttp.test_utils import TestClient from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status from servicelib.aiohttp.application import create_safe_application from simcore_service_webserver.activity.plugin import setup_activity -from simcore_service_webserver.application_settings import setup_settings +from simcore_service_webserver.application_settings import ( + PrometheusSettings, + setup_settings, +) from simcore_service_webserver.rest.plugin import setup_rest from simcore_service_webserver.security.plugin import setup_security from simcore_service_webserver.session.plugin import setup_session @@ -55,29 +59,61 @@ def mocked_monitoring_down(mocker: MockerFixture) -> None: @pytest.fixture -def app_config(fake_data_dir: Path, osparc_simcore_root_dir: Path) -> dict[str, Any]: - with Path.open(fake_data_dir / "test_activity_config.yml") as fh: - content = fh.read() - config = content.replace( - "${OSPARC_SIMCORE_REPO_ROOTDIR}", str(osparc_simcore_root_dir) - ) +def app_environment( + mock_env_devel_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +) -> EnvVarsDict: + + envs = mock_env_devel_environment | setenvs_from_dict( + monkeypatch, + { + "LOGIN_REGISTRATION_CONFIRMATION_REQUIRED": "True", + "LOGIN_REGISTRATION_INVITATION_REQUIRED": "False", + "POSTGRES_DB": "simcoredb", + "POSTGRES_HOST": "postgres", + "POSTGRES_MAXSIZE": "10", + "POSTGRES_MINSIZE": "10", + "POSTGRES_PASSWORD": "simcore", + "POSTGRES_PORT": "5432", + "POSTGRES_USER": "simcore", + "PROMETHEUS_PASSWORD": "fake", + "PROMETHEUS_URL": "http://prometheus:9090", + "PROMETHEUS_USERNAME": "fake", + "PROMETHEUS_VTAG": "v1", + "SESSION_SECRET_KEY": "REPLACE_ME_with_result__Fernet_generate_key=", + "SMTP_HOST": "mail.foo.com", + "SMTP_PORT": "25", + "STORAGE_HOST": "storage", + "STORAGE_PORT": "11111", + "STORAGE_VTAG": "v0", + "WEBSERVER_LOGIN": "null", + "WEBSERVER_LOGLEVEL": "DEBUG", + "WEBSERVER_PORT": "8080", + "WEBSERVER_STUDIES_ACCESS_ENABLED": "True", + }, + ) + + monkeypatch.delenv("WEBSERVER_ACTIVITY") + envs.pop("WEBSERVER_ACTIVITY") - return yaml.safe_load(config) + return envs @pytest.fixture def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, - app_config: dict[str, Any], mock_orphaned_services: MagicMock, - monkeypatch_setenv_from_app_config: Callable, + app_environment: EnvVarsDict, ): - monkeypatch_setenv_from_app_config(app_config) + # app_environment are in place + assert {key: os.environ[key] for key in app_environment} == app_environment + expected_activity_settings = PrometheusSettings.create_from_envs() + + app = create_safe_application() - app = create_safe_application(app_config) + settings = setup_settings(app) + assert expected_activity_settings == settings.WEBSERVER_ACTIVITY - assert setup_settings(app) setup_session(app) setup_security(app) setup_rest(app) @@ -92,7 +128,7 @@ async def test_has_login_required(client: TestClient): async def test_monitoring_up( - mocked_login_required: None, mocked_monitoring: None, client + mocked_login_required: None, mocked_monitoring: None, client: TestClient ): RUNNING_NODE_ID = "894dd8d5-de3b-4767-950c-7c3ed8f51d8c" From cab674f52eabfaf0951b4d1951bf662795449ae6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:39:10 +0100 Subject: [PATCH 18/71] rm app_cfg and monkeypatch_setenv_from_app_config --- .../01/test_groups_handlers_classifers.py | 87 ++++++++--------- .../tests/unit/with_dbs/01/test_statics.py | 14 +-- .../with_dbs/03/meta_modeling/conftest.py | 95 ++++++------------- .../tests/unit/with_dbs/03/tags/conftest.py | 60 ++++++++---- .../with_dbs/03/version_control/conftest.py | 93 ++++++------------ .../test_version_control_handlers.py | 16 ++-- .../test_resource_manager.py | 50 +++++----- .../server/tests/unit/with_dbs/conftest.py | 49 +++++----- 8 files changed, 194 insertions(+), 270 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_groups_handlers_classifers.py b/services/web/server/tests/unit/with_dbs/01/test_groups_handlers_classifers.py index c7367b03b942..02a0ddf581ba 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_groups_handlers_classifers.py +++ b/services/web/server/tests/unit/with_dbs/01/test_groups_handlers_classifers.py @@ -1,61 +1,50 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments import re -from copy import deepcopy import pytest from aiohttp import web_exceptions from aioresponses.core import aioresponses -from simcore_service_webserver.application_settings_utils import AppConfigDict +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict @pytest.fixture -def app_cfg(default_app_cfg: AppConfigDict, unused_tcp_port_factory): - """App's configuration used for every test in this module - - NOTE: Overrides services/web/server/tests/unit/with_dbs/conftest.py::app_cfg to influence app setup - """ - cfg = deepcopy(default_app_cfg) - - cfg["main"]["port"] = unused_tcp_port_factory() - cfg["main"]["studies_access_enabled"] = True - - exclude = { - "tracing", - "smtp", - "storage", - "activity", - "diagnostics", - "tags", - "publications", - "catalog", - "computation", - "products", - "socketio", - "resource_manager", - "projects", - "login", - "users", - } - include = { - "db", - "rest", - "groups", - } - - assert include.intersection(exclude) == set() - - for section in include: - cfg[section]["enabled"] = True - for section in exclude: - cfg[section]["enabled"] = False - - # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG - ## setup_logging(level=logging.DEBUG) - - return cfg +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + + monkeypatch.delenv("WEBSERVER_STUDIES_DISPATCHER", raising=False) + app_environment.pop("WEBSERVER_STUDIES_DISPATCHER", None) + + return app_environment | setenvs_from_dict( + monkeypatch, + { + # exclude + "WEBSERVER_ACTIVITY": "null", + "WEBSERVER_CATALOG": "null", + "WEBSERVER_CLUSTERS": "null", + "WEBSERVER_COMPUTATION": "null", + "WEBSERVER_DIAGNOSTICS": "null", + "WEBSERVER_EMAIL": "null", + "WEBSERVER_GARBAGE_COLLECTOR": "null", + "WEBSERVER_GROUPS": "0", + "WEBSERVER_LOGIN": "null", + "WEBSERVER_PRODUCTS": "0", + "WEBSERVER_PROJECTS": "null", + "WEBSERVER_PUBLICATIONS": "0", + "WEBSERVER_SOCKETIO": "0", + "WEBSERVER_STORAGE": "null", + "WEBSERVER_RESOURCE_MANAGER": "null", + "WEBSERVER_TAGS": "0", + "WEBSERVER_TRACING": "null", + "WEBSERVER_USERS": "null", + }, + ) @pytest.mark.skip(reason="UNDER DEV: test_group_handlers") diff --git a/services/web/server/tests/unit/with_dbs/01/test_statics.py b/services/web/server/tests/unit/with_dbs/01/test_statics.py index 1eb8212d986f..33a666b06b54 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_statics.py +++ b/services/web/server/tests/unit/with_dbs/01/test_statics.py @@ -16,7 +16,6 @@ from servicelib.aiohttp import status from servicelib.aiohttp.application import create_safe_application from simcore_postgres_database.models.products import products -from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.application_settings import setup_settings from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.db.plugin import setup_db @@ -55,25 +54,18 @@ def client( aiohttp_client: Callable, app_cfg: AppConfigDict, postgres_db: sa.engine.Engine, - monkeypatch_setenv_from_app_config: Callable, ) -> TestClient: cfg = deepcopy(app_cfg) - port = cfg["main"]["port"] - assert cfg["rest"]["version"] == API_VTAG - monkeypatch_setenv_from_app_config(cfg) - - # fake config - app = create_safe_application(cfg) + app = create_safe_application() settings = setup_settings(app) - print(settings.model_dump_json(indent=1)) - + assert settings.WEBSERVER_STATICWEB setup_rest(app) setup_db(app) setup_products(app) - setup_statics(app) + assert setup_statics(app) return event_loop.run_until_complete( aiohttp_client(app, server_kwargs={"port": port, "host": "localhost"}) diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py index 3b965a09e46b..d7e3dc7529e0 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/conftest.py @@ -1,80 +1,43 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable - -import logging -from copy import deepcopy -from typing import Any - +# pylint: disable=too-many-arguments import pytest +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from simcore_postgres_database.models.users import UserRole -from simcore_service_webserver.log import setup_logging @pytest.fixture def user_role() -> UserRole: - # TODO: user rights still not in place return UserRole.TESTER @pytest.fixture -def app_cfg(default_app_cfg, unused_tcp_port_factory, monkeypatch) -> dict[str, Any]: - """App's configuration used for every test in this module - - NOTE: Overrides services/web/server/tests/unit/with_dbs/conftest.py::app_cfg to influence app setup - """ - cfg = deepcopy(default_app_cfg) - - monkeypatch.setenv("WEBSERVER_DEV_FEATURES_ENABLED", "1") - - cfg["main"]["port"] = unused_tcp_port_factory() - cfg["main"]["studies_access_enabled"] = True - - exclude = { - "activity", - "clusters", - "computation", - "diagnostics", - "garbage_collector", - "groups", - "publications", - "smtp", - "socketio", - "storage", - "studies_dispatcher", - "tags", - "tracing", - } - include = { - "catalog", - "db", - "login", - "meta_modeling", # MODULE UNDER TEST - "products", - "projects", - "redis", - "resource_manager", - "rest", - "users", - "version_control", - } - - assert include.intersection(exclude) == set() - - for section in include: - cfg[section]["enabled"] = True - for section in exclude: - cfg[section]["enabled"] = False - - # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG - setup_logging( - level=logging.DEBUG, - log_format_local_dev_enabled=True, - logger_filter_mapping={}, - tracing_settings=None, +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + return app_environment | setenvs_from_dict( + monkeypatch, + { + # exclude + "WEBSERVER_ACTIVITY": "null", + "WEBSERVER_CLUSTERS": "null", + "WEBSERVER_COMPUTATION": "null", + "WEBSERVER_DIAGNOSTICS": "null", + "WEBSERVER_GROUPS": "0", + "WEBSERVER_PUBLICATIONS": "0", + "WEBSERVER_GARBAGE_COLLECTOR": "null", + "WEBSERVER_EMAIL": "null", + "WEBSERVER_SOCKETIO": "0", + "WEBSERVER_STORAGE": "null", + "WEBSERVER_STUDIES_DISPATCHER": "null", + "WEBSERVER_TAGS": "0", + "WEBSERVER_TRACING": "null", + # Module under test + "WEBSERVER_DEV_FEATURES_ENABLED": "1", + "WEBSERVER_VERSION_CONTROL": "1", + "WEBSERVER_META_MODELING": "1", + }, ) - - # Enforces smallest GC in the background task - cfg["resource_manager"]["garbage_collection_interval_seconds"] = 1 - - return cfg diff --git a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py index 45682405bfe8..43c6da4f3447 100644 --- a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py @@ -2,16 +2,20 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import asyncio from collections.abc import AsyncIterator, Callable from copy import deepcopy from pathlib import Path import pytest from aioresponses import aioresponses +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects from servicelib.aiohttp.application import create_safe_application from simcore_service_webserver.application_settings import setup_settings +from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.db.plugin import setup_db from simcore_service_webserver.director_v2.plugin import setup_director_v2 from simcore_service_webserver.login.plugin import setup_login @@ -34,33 +38,45 @@ DEFAULT_GARBAGE_COLLECTOR_DELETION_TIMEOUT_SECONDS: int = 3 +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + # NOTE: undos some app_environment settings + monkeypatch.delenv("WEBSERVER_GARBAGE_COLLECTOR", raising=False) + app_environment.pop("WEBSERVER_GARBAGE_COLLECTOR", None) + + return app_environment | setenvs_from_dict( + monkeypatch, + { + # reduce deletion delay + "RESOURCE_MANAGER_RESOURCE_TTL_S": f"{DEFAULT_GARBAGE_COLLECTOR_INTERVAL_SECONDS}", + # increase speed of garbage collection + "GARBAGE_COLLECTOR_INTERVAL_S": f"{DEFAULT_GARBAGE_COLLECTOR_DELETION_TIMEOUT_SECONDS}", + }, + ) + + @pytest.fixture def client( - event_loop, - aiohttp_client, - app_cfg, + event_loop: asyncio.AbstractEventLoop, + aiohttp_client: Callable, + app_cfg: AppConfigDict, + app_environment: EnvVarsDict, postgres_db, mocked_dynamic_services_interface, mock_orphaned_services, redis_client, # this ensure redis is properly cleaned - monkeypatch_setenv_from_app_config: Callable, ): - # config app - cfg = deepcopy(app_cfg) - port = cfg["main"]["port"] - cfg["projects"]["enabled"] = True - cfg["resource_manager"][ - "garbage_collection_interval_seconds" - ] = DEFAULT_GARBAGE_COLLECTOR_INTERVAL_SECONDS # increase speed of garbage collection - cfg["resource_manager"][ - "resource_deletion_timeout_seconds" - ] = DEFAULT_GARBAGE_COLLECTOR_DELETION_TIMEOUT_SECONDS # reduce deletion delay - - monkeypatch_setenv_from_app_config(cfg) + app = create_safe_application() - app = create_safe_application(cfg) + assert "WEBSERVER_GARBAGE_COLLECTOR" not in app_environment - assert setup_settings(app) + settings = setup_settings(app) + assert settings.WEBSERVER_GARBAGE_COLLECTOR is not None + assert settings.WEBSERVER_PROJECTS is not None + assert settings.WEBSERVER_TAGS is not None # setup app setup_db(app) @@ -71,14 +87,16 @@ def client( setup_resource_manager(app) setup_socketio(app) setup_director_v2(app) - setup_tags(app) - assert setup_projects(app) + assert setup_tags(app) + setup_projects(app) setup_products(app) setup_wallets(app) # server and client return event_loop.run_until_complete( - aiohttp_client(app, server_kwargs={"port": port, "host": "localhost"}) + aiohttp_client( + app, server_kwargs={"port": app_cfg["main"]["port"], "host": "localhost"} + ) ) # teardown here ... diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index e9a6244886c7..00f35441f8d4 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -2,11 +2,8 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -import logging from collections.abc import AsyncIterator, Awaitable, Callable -from copy import deepcopy from pathlib import Path -from typing import Any from unittest import mock from uuid import UUID @@ -21,6 +18,8 @@ from models_library.utils.fastapi_encoders import jsonable_encoder from pytest_mock import MockerFixture from pytest_simcore.helpers.faker_factories import random_project +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_projects import NewProject from servicelib.aiohttp import status @@ -31,7 +30,6 @@ from simcore_service_webserver._meta import API_VTAG as VX from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.db.plugin import APP_AIOPG_ENGINE_KEY -from simcore_service_webserver.log import setup_logging from simcore_service_webserver.projects.db import ProjectDBAPI from simcore_service_webserver.projects.models import ProjectDict from tenacity.asyncio import AsyncRetrying @@ -69,70 +67,35 @@ def catalog_subsystem_mock_override( @pytest.fixture -def app_cfg( - default_app_cfg, - unused_tcp_port_factory, +def app_environment( catalog_subsystem_mock_override: None, - monkeypatch, -) -> dict[str, Any]: - """App's configuration used for every test in this module - - NOTE: Overrides services/web/server/tests/unit/with_dbs/conftest.py::app_cfg to influence app setup - """ - cfg = deepcopy(default_app_cfg) - - monkeypatch.setenv("WEBSERVER_DEV_FEATURES_ENABLED", "1") - - cfg["main"]["port"] = unused_tcp_port_factory() - cfg["main"]["studies_access_enabled"] = True - - exclude = { - "activity", - "clusters", - "computation", - "diagnostics", - "groups", - "publications", - "garbage_collector", - "smtp", - "socketio", - "storage", - "studies_dispatcher", - "tags", - "tracing", - } - include = { - "catalog", - "db", - "login", - "products", - "projects", - "resource_manager", - "rest", - "users", - "version_control", # MODULE UNDER TEST - } - - assert include.intersection(exclude) == set() - - for section in include: - cfg[section]["enabled"] = True - for section in exclude: - cfg[section]["enabled"] = False - - # NOTE: To see logs, use pytest -s --log-cli-level=DEBUG - setup_logging( - level=logging.DEBUG, - log_format_local_dev_enabled=True, - logger_filter_mapping={}, - tracing_settings=None, + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + + return app_environment | setenvs_from_dict( + monkeypatch, + { + # exclude + "WEBSERVER_ACTIVITY": "null", + "WEBSERVER_CLUSTERS": "null", + "WEBSERVER_COMPUTATION": "null", + "WEBSERVER_DIAGNOSTICS": "null", + "WEBSERVER_GROUPS": "0", + "WEBSERVER_PUBLICATIONS": "0", + "WEBSERVER_GARBAGE_COLLECTOR": "null", + "WEBSERVER_EMAIL": "null", + "WEBSERVER_SOCKETIO": "0", + "WEBSERVER_STORAGE": "null", + "WEBSERVER_STUDIES_DISPATCHER": "null", + "WEBSERVER_TAGS": "0", + "WEBSERVER_TRACING": "null", + # Module under test + "WEBSERVER_DEV_FEATURES_ENABLED": "1", + "WEBSERVER_VERSION_CONTROL": "1", + }, ) - # Enforces smallest GC in the background task - cfg["resource_manager"]["garbage_collection_interval_seconds"] = 1 - - return cfg - @pytest.fixture async def user_id(logged_user: UserInfoDict) -> UserID: diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py index 325eb63e353e..ac229a3b410d 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py @@ -1,6 +1,8 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument # pylint: disable=unused-variable +# pylint: disable=too-many-arguments + from collections.abc import Awaitable, Callable from http import HTTPStatus @@ -23,7 +25,7 @@ ) -async def assert_resp_page( +async def _assert_resp_page( resp: aiohttp.ClientResponse, expected_page_cls: type[Page], expected_total: int, @@ -38,7 +40,7 @@ async def assert_resp_page( return page -async def assert_status_and_body( +async def _assert_status_and_body( resp, expected_cls: HTTPStatus, expected_model: type[BaseModel] ) -> BaseModel: data, _ = await assert_status(resp, expected_cls) @@ -84,7 +86,7 @@ async def test_workflow( # # this project now has a repo resp = await client.get(f"/{VX}/repos/projects") - page = await assert_resp_page( + page = await _assert_resp_page( resp, expected_page_cls=Page[ProjectDict], expected_total=1, expected_count=1 ) @@ -97,8 +99,8 @@ async def test_workflow( assert CheckpointApiModel.model_validate(data) == checkpoint1 # TODO: GET checkpoint with tag + resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints/v1") with pytest.raises(aiohttp.ClientResponseError) as excinfo: - resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints/v1") resp.raise_for_status() assert CheckpointApiModel.model_validate(data) == checkpoint1 @@ -114,7 +116,7 @@ async def test_workflow( # LIST checkpoints resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints") - page = await assert_resp_page( + page = await _assert_resp_page( resp, expected_page_cls=Page[CheckpointApiModel], expected_total=1, @@ -226,7 +228,7 @@ async def test_delete_project_and_repo( # LIST resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints") - await assert_resp_page( + await _assert_resp_page( resp, expected_page_cls=Page[CheckpointApiModel], expected_total=1, @@ -247,7 +249,7 @@ async def test_delete_project_and_repo( # LIST empty resp = await client.get(f"/{VX}/repos/projects/{project_uuid}/checkpoints") - await assert_resp_page( + await _assert_resp_page( resp, expected_page_cls=Page[CheckpointApiModel], expected_total=0, diff --git a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py index 18cba7971775..ac1866f4ee13 100644 --- a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py @@ -8,7 +8,6 @@ import asyncio from asyncio import Future from collections.abc import AsyncIterator, Awaitable, Callable -from copy import deepcopy from pathlib import Path from typing import Any from unittest import mock @@ -28,6 +27,7 @@ from pytest_mock import MockerFixture from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import UserInfoDict from pytest_simcore.helpers.webserver_parametrizations import MockedStorageSubsystem from pytest_simcore.helpers.webserver_projects import NewProject @@ -37,8 +37,8 @@ from servicelib.aiohttp.application_setup import is_setup_completed from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_postgres_database.models.users import UserRole -from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.application_settings import setup_settings +from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.db.plugin import setup_db from simcore_service_webserver.director_v2.plugin import setup_director_v2 from simcore_service_webserver.garbage_collector import _core as gc_core @@ -104,45 +104,44 @@ async def _open_project(client, project_uuid: str, client_session_id: str) -> No @pytest.fixture def app_environment( - app_environment: dict[str, str], monkeypatch: pytest.MonkeyPatch -) -> dict[str, str]: - overrides = setenvs_from_dict( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + + # NOTE: undos some app_environment settings + monkeypatch.delenv("WEBSERVER_GARBAGE_COLLECTOR", raising=False) + app_environment.pop("WEBSERVER_GARBAGE_COLLECTOR", None) + + return app_environment | setenvs_from_dict( monkeypatch, { "WEBSERVER_COMPUTATION": "1", "WEBSERVER_NOTIFICATIONS": "1", + # sets TTL of a resource after logout + "RESOURCE_MANAGER_RESOURCE_TTL_S": f"{SERVICE_DELETION_DELAY}", + "GARBAGE_COLLECTOR_INTERVAL_S": "30", }, ) - return app_environment | overrides @pytest.fixture def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, - app_cfg: dict[str, Any], + app_cfg: AppConfigDict, + app_environment: EnvVarsDict, postgres_db: sa.engine.Engine, mock_orphaned_services, redis_client: Redis, - monkeypatch_setenv_from_app_config: Callable, mock_dynamic_scheduler_rabbitmq: None, ) -> TestClient: - cfg = deepcopy(app_cfg) - assert cfg["rest"]["version"] == API_VTAG - assert cfg["rest"]["enabled"] - cfg["projects"]["enabled"] = True - - # sets TTL of a resource after logout - cfg["resource_manager"][ - "resource_deletion_timeout_seconds" - ] = SERVICE_DELETION_DELAY - - monkeypatch_setenv_from_app_config(cfg) - app = create_safe_application(cfg) + app = create_safe_application() - # activates only security+restAPI sub-modules + assert "WEBSERVER_GARBAGE_COLLECTOR" not in app_environment - assert setup_settings(app) + settings = setup_settings(app) + assert settings.WEBSERVER_GARBAGE_COLLECTOR is not None + assert settings.WEBSERVER_PROJECTS is not None setup_db(app) setup_session(app) @@ -151,7 +150,7 @@ def client( setup_login(app) setup_users(app) setup_socketio(app) - setup_projects(app) + assert setup_projects(app) setup_director_v2(app) assert setup_resource_manager(app) setup_rabbitmq(app) @@ -167,7 +166,10 @@ def client( return event_loop.run_until_complete( aiohttp_client( app, - server_kwargs={"port": cfg["main"]["port"], "host": cfg["main"]["host"]}, + server_kwargs={ + "port": app_cfg["main"]["port"], + "host": app_cfg["main"]["host"], + }, ) ) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 6661af40d5e3..955a824d5c1e 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -118,25 +118,13 @@ def docker_compose_file(docker_compose_env: pytest.MonkeyPatch) -> str: # WEB SERVER/CLIENT FIXTURES ------------------------------------------------ -@pytest.fixture -def app_cfg(default_app_cfg: AppConfigDict, unused_tcp_port_factory) -> AppConfigDict: - """ - NOTE: SHOULD be overriden in any test module to configure the app accordingly - """ - cfg = deepcopy(default_app_cfg) - # fills ports on the fly - cfg["main"]["port"] = unused_tcp_port_factory() - cfg["storage"]["port"] = unused_tcp_port_factory() - - # this fixture can be safely modified during test since it is renovated on every call - return cfg - - @pytest.fixture def app_environment( monkeypatch: pytest.MonkeyPatch, - app_cfg: AppConfigDict, - monkeypatch_setenv_from_app_config: Callable[[AppConfigDict], dict[str, str]], + default_app_cfg: AppConfigDict, + unused_tcp_port_factory: Callable, + mock_env_devel_environment: EnvVarsDict, + monkeypatch_setenv_from_app_config: Callable[[AppConfigDict], EnvVarsDict], ) -> EnvVarsDict: # WARNING: this fixture is commonly overriden. Check before renaming. """overridable fixture that defines the ENV for the webserver application @@ -144,20 +132,27 @@ def app_environment( override like so: @pytest.fixture - def app_environment(app_environment: dict[str, str], monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: + def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: monkeypatch.setenv("MODIFIED_ENV", "VALUE") return app_environment | {"MODIFIED_ENV":"VALUE"} """ - print("+ web_server:") - cfg = deepcopy(app_cfg) - envs = monkeypatch_setenv_from_app_config(cfg) - - # - # NOTE: this emulates hostname: "wb-{{.Node.Hostname}}-{{.Task.Slot}}" in docker-compose that - # affects PostgresSettings.POSTGRES_CLIENT_NAME - # - extra = setenvs_from_dict(monkeypatch, {"HOSTNAME": "wb-test_host.0"}) - return envs | extra + # NOTE: remains from from old cfg + cfg = deepcopy(default_app_cfg) + cfg["main"]["port"] = unused_tcp_port_factory() + cfg["storage"]["port"] = unused_tcp_port_factory() + + return ( + mock_env_devel_environment + | monkeypatch_setenv_from_app_config(cfg) + | setenvs_from_dict( + monkeypatch, + { + # this emulates hostname: "wb-{{.Node.Hostname}}-{{.Task.Slot}}" in docker-compose that + # affects PostgresSettings.POSTGRES_CLIENT_NAME + "HOSTNAME": "wb-test_host.0" + }, + ) + ) @pytest.fixture From 3f5ba170bfbb76b0e2497dabf9b3fa123d00dc97 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:45:26 +0100 Subject: [PATCH 19/71] deprecate tests --- services/web/server/setup.cfg | 5 +++-- .../isolated/test_application_settings_utils.py | 14 +++++--------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 9a74fe46b3d6..ed37f550fc71 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -12,13 +12,14 @@ commit_args = --no-verify [tool:pytest] addopts = --strict-markers asyncio_mode = auto -markers = +markers = slow: marks tests as slow (deselect with '-m "not slow"') acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." testit: "marks test to run during development" heavy_load: "mark tests that require large amount of data" + deprecated: "will be removed soon" [mypy] -plugins = +plugins = pydantic.mypy sqlalchemy.ext.mypy.plugin diff --git a/services/web/server/tests/unit/isolated/test_application_settings_utils.py b/services/web/server/tests/unit/isolated/test_application_settings_utils.py index 77195f3d02a1..086038879e1a 100644 --- a/services/web/server/tests/unit/isolated/test_application_settings_utils.py +++ b/services/web/server/tests/unit/isolated/test_application_settings_utils.py @@ -9,19 +9,15 @@ ) -@pytest.mark.skip(reason="UNDER DEV") +@pytest.mark.deprecated def test_settings_infered_from_default_tests_config( default_app_cfg: AppConfigDict, monkeypatch_setenv_from_app_config: Callable ): - # TODO: use app_config_for_production_legacy envs = monkeypatch_setenv_from_app_config(default_app_cfg) - assert envs == convert_to_environ_vars(default_app_cfg) + assert envs == { + k: f"{v}" for k, v in convert_to_environ_vars(default_app_cfg).items() + } settings = ApplicationSettings.create_from_envs() - print("settings=\n", settings.model_dump_json(indent=1)) - - infered_config = convert_to_app_config(settings) - - assert default_app_cfg == infered_config - assert set(default_app_cfg.keys()) == set(infered_config.keys()) + assert convert_to_app_config(settings) From 474ec43ca2169f003b7f2ec7b11e29864615abdc Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:46:29 +0100 Subject: [PATCH 20/71] deprecate tests --- services/web/server/setup.cfg | 1 - .../tests/unit/isolated/test_application_settings_utils.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index ed37f550fc71..486fe83406d6 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -17,7 +17,6 @@ markers = acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." testit: "marks test to run during development" heavy_load: "mark tests that require large amount of data" - deprecated: "will be removed soon" [mypy] plugins = diff --git a/services/web/server/tests/unit/isolated/test_application_settings_utils.py b/services/web/server/tests/unit/isolated/test_application_settings_utils.py index 086038879e1a..9db5ed0dd485 100644 --- a/services/web/server/tests/unit/isolated/test_application_settings_utils.py +++ b/services/web/server/tests/unit/isolated/test_application_settings_utils.py @@ -1,6 +1,5 @@ from collections.abc import Callable -import pytest from simcore_service_webserver.application_settings import ApplicationSettings from simcore_service_webserver.application_settings_utils import ( AppConfigDict, @@ -9,7 +8,6 @@ ) -@pytest.mark.deprecated def test_settings_infered_from_default_tests_config( default_app_cfg: AppConfigDict, monkeypatch_setenv_from_app_config: Callable ): From 37e1d19f4d1f7644c1443a2776965a8e5158e28d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:56:35 +0100 Subject: [PATCH 21/71] fixes --- services/web/server/tests/unit/with_dbs/conftest.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 955a824d5c1e..6ebb1b0c2065 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -138,9 +138,6 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc """ # NOTE: remains from from old cfg cfg = deepcopy(default_app_cfg) - cfg["main"]["port"] = unused_tcp_port_factory() - cfg["storage"]["port"] = unused_tcp_port_factory() - return ( mock_env_devel_environment | monkeypatch_setenv_from_app_config(cfg) @@ -186,7 +183,7 @@ async def _print_mail_to_stdout( @pytest.fixture def web_server( event_loop: asyncio.AbstractEventLoop, - app_cfg: AppConfigDict, + unused_tcp_port_factory: Callable, app_environment: EnvVarsDict, postgres_db: sa.engine.Engine, # tools @@ -200,7 +197,7 @@ def web_server( disable_static_webserver(app) server = event_loop.run_until_complete( - aiohttp_server(app, port=app_cfg["main"]["port"]) + aiohttp_server(app, port=unused_tcp_port_factory()) ) assert isinstance(postgres_db, sa.engine.Engine) From 1cbf4dcb5b396e962fefdcbe255780d10d282305 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 19:23:54 +0100 Subject: [PATCH 22/71] rm unused app --- .../tests/unit/with_dbs/01/test_statics.py | 8 +------ .../tests/unit/with_dbs/01/test_storage.py | 21 ++++++++++++------- .../tests/unit/with_dbs/03/tags/conftest.py | 6 +----- .../tests/unit/with_dbs/03/test_session.py | 12 +---------- .../test_resource_manager.py | 6 ------ .../server/tests/unit/with_dbs/conftest.py | 5 ++++- 6 files changed, 20 insertions(+), 38 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/01/test_statics.py b/services/web/server/tests/unit/with_dbs/01/test_statics.py index 33a666b06b54..d3ba4448061d 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_statics.py +++ b/services/web/server/tests/unit/with_dbs/01/test_statics.py @@ -5,7 +5,6 @@ import json import re from collections.abc import Callable -from copy import deepcopy import pytest import sqlalchemy as sa @@ -17,7 +16,6 @@ from servicelib.aiohttp.application import create_safe_application from simcore_postgres_database.models.products import products from simcore_service_webserver.application_settings import setup_settings -from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.db.plugin import setup_db from simcore_service_webserver.products.plugin import setup_products from simcore_service_webserver.rest.plugin import setup_rest @@ -52,12 +50,8 @@ def client( app_environment: EnvVarsDict, event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, - app_cfg: AppConfigDict, postgres_db: sa.engine.Engine, ) -> TestClient: - cfg = deepcopy(app_cfg) - port = cfg["main"]["port"] - app = create_safe_application() settings = setup_settings(app) @@ -68,7 +62,7 @@ def client( assert setup_statics(app) return event_loop.run_until_complete( - aiohttp_client(app, server_kwargs={"port": port, "host": "localhost"}) + aiohttp_client(app, server_kwargs={"host": "localhost"}) ) diff --git a/services/web/server/tests/unit/with_dbs/01/test_storage.py b/services/web/server/tests/unit/with_dbs/01/test_storage.py index e6977b67c6d3..32e469bcc0ea 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_storage.py +++ b/services/web/server/tests/unit/with_dbs/01/test_storage.py @@ -2,12 +2,15 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import asyncio from urllib.parse import quote import pytest from aiohttp import web +from aiohttp.test_utils import TestServer from faker import Faker from pytest_simcore.helpers.assert_checks import assert_status +from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status from servicelib.aiohttp.application import create_safe_application from simcore_postgres_database.models.users import UserRole @@ -17,10 +20,11 @@ # TODO: create a fake storage service here @pytest.fixture() -def storage_server(event_loop, aiohttp_server, app_cfg): - cfg = app_cfg["storage"] - app = create_safe_application(cfg) - +def storage_server( + event_loop: asyncio.AbstractEventLoop, + aiohttp_server: TestServer, + app_environment: EnvVarsDict, +): async def _get_locs(request: web.Request): assert not request.can_read_body @@ -119,11 +123,14 @@ async def _get_datasets_meta(request: web.Request): } ) - storage_api_version = cfg["version"] + storage_api_version = app_environment["STORAGE_VTAG"] + storage_port = int(app_environment["STORAGE_PORT"]) + assert ( storage_api_version != API_VERSION ), "backend service w/ different version as webserver entrypoint" + app = create_safe_application() app.router.add_get(f"/{storage_api_version}/locations", _get_locs) app.router.add_post( f"/{storage_api_version}/locations/0:sync", _post_sync_meta_data @@ -140,9 +147,7 @@ async def _get_datasets_meta(request: web.Request): _get_datasets_meta, ) - assert cfg["host"] == "localhost" - - server = event_loop.run_until_complete(aiohttp_server(app, port=cfg["port"])) + server = event_loop.run_until_complete(aiohttp_server(app, port=storage_port)) return server diff --git a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py index 43c6da4f3447..ea8907aeab38 100644 --- a/services/web/server/tests/unit/with_dbs/03/tags/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/tags/conftest.py @@ -15,7 +15,6 @@ from pytest_simcore.helpers.webserver_projects import NewProject, delete_all_projects from servicelib.aiohttp.application import create_safe_application from simcore_service_webserver.application_settings import setup_settings -from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.db.plugin import setup_db from simcore_service_webserver.director_v2.plugin import setup_director_v2 from simcore_service_webserver.login.plugin import setup_login @@ -62,7 +61,6 @@ def app_environment( def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, - app_cfg: AppConfigDict, app_environment: EnvVarsDict, postgres_db, mocked_dynamic_services_interface, @@ -94,9 +92,7 @@ def client( # server and client return event_loop.run_until_complete( - aiohttp_client( - app, server_kwargs={"port": app_cfg["main"]["port"], "host": "localhost"} - ) + aiohttp_client(app, server_kwargs={"host": "localhost"}) ) # teardown here ... diff --git a/services/web/server/tests/unit/with_dbs/03/test_session.py b/services/web/server/tests/unit/with_dbs/03/test_session.py index c3684acb3265..36a4f693c037 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_session.py +++ b/services/web/server/tests/unit/with_dbs/03/test_session.py @@ -15,7 +15,6 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import NewUser from simcore_service_webserver.application import create_application -from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.session._cookie_storage import ( SharedCookieEncryptedCookieStorage, ) @@ -34,7 +33,6 @@ def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, disable_static_webserver: Callable, - app_cfg: AppConfigDict, app_environment: EnvVarsDict, postgres_db, mock_orphaned_services, # disables gc @@ -52,15 +50,7 @@ async def _get_user_session(request: web.Request): app.add_routes(extra_routes) - return event_loop.run_until_complete( - aiohttp_client( - app, - server_kwargs={ - "port": app_cfg["main"]["port"], - "host": app_cfg["main"]["host"], - }, - ) - ) + return event_loop.run_until_complete(aiohttp_client(app)) async def test_security_identity_is_email_and_product( diff --git a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py index ac1866f4ee13..288e19562ec6 100644 --- a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py @@ -38,7 +38,6 @@ from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_postgres_database.models.users import UserRole from simcore_service_webserver.application_settings import setup_settings -from simcore_service_webserver.application_settings_utils import AppConfigDict from simcore_service_webserver.db.plugin import setup_db from simcore_service_webserver.director_v2.plugin import setup_director_v2 from simcore_service_webserver.garbage_collector import _core as gc_core @@ -128,7 +127,6 @@ def app_environment( def client( event_loop: asyncio.AbstractEventLoop, aiohttp_client: Callable, - app_cfg: AppConfigDict, app_environment: EnvVarsDict, postgres_db: sa.engine.Engine, mock_orphaned_services, @@ -166,10 +164,6 @@ def client( return event_loop.run_until_complete( aiohttp_client( app, - server_kwargs={ - "port": app_cfg["main"]["port"], - "host": app_cfg["main"]["host"], - }, ) ) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 6ebb1b0c2065..463a54c849a5 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -138,9 +138,12 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc """ # NOTE: remains from from old cfg cfg = deepcopy(default_app_cfg) + cfg["storage"]["port"] = unused_tcp_port_factory() + envs_app_cfg = monkeypatch_setenv_from_app_config(cfg) + return ( mock_env_devel_environment - | monkeypatch_setenv_from_app_config(cfg) + | envs_app_cfg | setenvs_from_dict( monkeypatch, { From 842eee355bb3e47b56669eacb420f352c13142f3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 19:27:13 +0100 Subject: [PATCH 23/71] fix static --- .../server/tests/unit/with_dbs/03/test_session.py | 15 +++++++++++++++ .../web/server/tests/unit/with_dbs/conftest.py | 1 + 2 files changed, 16 insertions(+) diff --git a/services/web/server/tests/unit/with_dbs/03/test_session.py b/services/web/server/tests/unit/with_dbs/03/test_session.py index 36a4f693c037..060959ee44d6 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_session.py +++ b/services/web/server/tests/unit/with_dbs/03/test_session.py @@ -12,6 +12,7 @@ from aiohttp import web from aiohttp.test_utils import TestClient from cryptography.fernet import Fernet +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from pytest_simcore.helpers.webserver_login import NewUser from simcore_service_webserver.application import create_application @@ -27,6 +28,20 @@ def session_url_path() -> str: return "/v0/test-session" +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + return app_environment | setenvs_from_dict( + monkeypatch, + { + # do not include static entrypoint + "WEBSERVER_STATICWEB": "null", + }, + ) + + @pytest.fixture def client( session_url_path: str, diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 463a54c849a5..cb1edb11460b 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -194,6 +194,7 @@ def web_server( mocked_send_email: None, disable_static_webserver: Callable, ) -> TestServer: + # original APP app = create_application() From b729c226ac1ceff4d5074694c4068be071cf4518 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 19:41:19 +0100 Subject: [PATCH 24/71] ports --- .../tests/unit/isolated/test__redirections.py | 9 +++- .../tests/unit/isolated/test_activity.py | 4 +- .../isolated/test_diagnostics_healthcheck.py | 18 ++++--- .../server/tests/unit/isolated/test_rest.py | 4 +- .../tests/unit/isolated/test_security_api.py | 4 +- .../unit/isolated/test_utils_rate_limiting.py | 6 ++- .../with_dbs/01/test_guests_management.py | 54 ------------------- .../tests/unit/with_dbs/01/test_storage.py | 36 ++++++++----- .../server/tests/unit/with_dbs/conftest.py | 20 ++++++- 9 files changed, 71 insertions(+), 84 deletions(-) delete mode 100644 services/web/server/tests/unit/with_dbs/01/test_guests_management.py diff --git a/services/web/server/tests/unit/isolated/test__redirections.py b/services/web/server/tests/unit/isolated/test__redirections.py index 3025ac8edb16..978a41195be4 100644 --- a/services/web/server/tests/unit/isolated/test__redirections.py +++ b/services/web/server/tests/unit/isolated/test__redirections.py @@ -5,10 +5,13 @@ import asyncio import textwrap +from collections.abc import Callable from pathlib import Path +from typing import Awaitable import pytest from aiohttp import web +from aiohttp.test_utils import TestClient @pytest.fixture @@ -34,7 +37,11 @@ def index_static_path(tmpdir): @pytest.fixture -def client(event_loop: asyncio.AbstractEventLoop, aiohttp_client, index_static_path): +def client( + event_loop: asyncio.AbstractEventLoop, + aiohttp_client: Callable[..., Awaitable[TestClient]], + index_static_path, +): routes = web.RouteTableDef() diff --git a/services/web/server/tests/unit/isolated/test_activity.py b/services/web/server/tests/unit/isolated/test_activity.py index 108e99a403f3..b08543a99965 100644 --- a/services/web/server/tests/unit/isolated/test_activity.py +++ b/services/web/server/tests/unit/isolated/test_activity.py @@ -4,7 +4,7 @@ import asyncio import os -from collections.abc import Callable +from collections.abc import Awaitable, Callable from typing import Any from unittest.mock import MagicMock @@ -101,7 +101,7 @@ def app_environment( @pytest.fixture def client( event_loop: asyncio.AbstractEventLoop, - aiohttp_client: Callable, + aiohttp_client: Callable[..., Awaitable[TestClient]], mock_orphaned_services: MagicMock, app_environment: EnvVarsDict, ): diff --git a/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py b/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py index b35b2b378f4d..01626715b485 100644 --- a/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py +++ b/services/web/server/tests/unit/isolated/test_diagnostics_healthcheck.py @@ -7,8 +7,8 @@ import asyncio import json import logging -from collections.abc import Callable, Coroutine import time +from collections.abc import Awaitable, Callable, Coroutine import pytest import simcore_service_webserver @@ -89,12 +89,14 @@ def mock_environment( { **mock_env_devel_environment, "AIODEBUG_SLOW_DURATION_SECS": f"{SLOW_HANDLER_DELAY_SECS / 10}", - "WEBSERVER_DIAGNOSTICS": json.dumps({ - "DIAGNOSTICS_MAX_AVG_LATENCY": "2.0", - "DIAGNOSTICS_MAX_TASK_DELAY": f"{SLOW_HANDLER_DELAY_SECS}", - "DIAGNOSTICS_START_SENSING_DELAY": f"{0}", - "DIAGNOSTICS_HEALTHCHECK_ENABLED": "1", - }), + "WEBSERVER_DIAGNOSTICS": json.dumps( + { + "DIAGNOSTICS_MAX_AVG_LATENCY": "2.0", + "DIAGNOSTICS_MAX_TASK_DELAY": f"{SLOW_HANDLER_DELAY_SECS}", + "DIAGNOSTICS_START_SENSING_DELAY": f"{0}", + "DIAGNOSTICS_HEALTHCHECK_ENABLED": "1", + } + ), "SC_HEALTHCHECK_TIMEOUT": "2m", }, ) @@ -104,7 +106,7 @@ def mock_environment( def client( event_loop: asyncio.AbstractEventLoop, unused_tcp_port_factory: Callable, - aiohttp_client: Callable, + aiohttp_client: Callable[..., Awaitable[TestClient]], api_version_prefix: str, mock_environment: EnvVarsDict, ) -> TestClient: diff --git a/services/web/server/tests/unit/isolated/test_rest.py b/services/web/server/tests/unit/isolated/test_rest.py index a10592b77572..12d4ed95fe99 100644 --- a/services/web/server/tests/unit/isolated/test_rest.py +++ b/services/web/server/tests/unit/isolated/test_rest.py @@ -3,7 +3,7 @@ # pylint:disable=no-name-in-module import asyncio -from collections.abc import Callable +from collections.abc import Awaitable, Callable from http import HTTPStatus from unittest.mock import MagicMock @@ -24,7 +24,7 @@ def client( event_loop: asyncio.AbstractEventLoop, unused_tcp_port_factory: Callable, - aiohttp_client: Callable, + aiohttp_client: Callable[..., Awaitable[TestClient]], api_version_prefix: str, mock_env_devel_environment: EnvVarsDict, mock_env_deployer_pipeline: EnvVarsDict, diff --git a/services/web/server/tests/unit/isolated/test_security_api.py b/services/web/server/tests/unit/isolated/test_security_api.py index 079fa68e529b..b913d95e5a7f 100644 --- a/services/web/server/tests/unit/isolated/test_security_api.py +++ b/services/web/server/tests/unit/isolated/test_security_api.py @@ -6,7 +6,7 @@ import asyncio import statistics from collections import OrderedDict -from collections.abc import Callable +from collections.abc import Awaitable, Callable from typing import Any from unittest.mock import MagicMock @@ -205,7 +205,7 @@ async def _logout(request: web.Request): @pytest.fixture def client( event_loop: asyncio.AbstractEventLoop, - aiohttp_client: Callable, + aiohttp_client: Callable[..., Awaitable[TestClient]], mocker: MockerFixture, app_products: OrderedDict[str, Product], set_products_in_app_state: Callable[ diff --git a/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py b/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py index 6568b1b7db47..fed37d1ba4e7 100644 --- a/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py +++ b/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py @@ -4,6 +4,7 @@ import asyncio import time +from collections.abc import Awaitable from typing import Callable import pytest @@ -28,7 +29,10 @@ async def get_ok_handler(_request: web.Request): @pytest.fixture -def client(event_loop, aiohttp_client: Callable) -> TestClient: +def client( + event_loop, + aiohttp_client: Callable[..., Awaitable[TestClient]], +) -> TestClient: app = web.Application() app.router.add_get("/", get_ok_handler) diff --git a/services/web/server/tests/unit/with_dbs/01/test_guests_management.py b/services/web/server/tests/unit/with_dbs/01/test_guests_management.py deleted file mode 100644 index dae525ca89a8..000000000000 --- a/services/web/server/tests/unit/with_dbs/01/test_guests_management.py +++ /dev/null @@ -1,54 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name - -from copy import deepcopy - -import pytest -from simcore_service_webserver import application - - -@pytest.fixture -def client(event_loop, aiohttp_client, app_cfg, postgres_db): - - # config app - cfg = deepcopy(app_cfg) - port = cfg["main"]["port"] - cfg["projects"]["enabled"] = True - - app = application.create_application() - - # server and client - return event_loop.run_until_complete( - aiohttp_client(app, server_kwargs={"port": port, "host": "localhost"}) - ) - - -@pytest.mark.skip(reason="Under dev") -def test_users_projects_db(client): - # given schema, an easy way to produce projects? - # a language to emulate UI?? - # See https://github.com/cwacek/python-jsonschema-objects - # - # api/specs/webserver/v0/components/schemas/project-v0.0.1.json - # create_project(client.app, ) - pass - - -@pytest.mark.skip(reason="Under dev") -def test_cleanup_expired_guest_users(client): - pass - # Guests users expire - - # Shall delete all guest users and non-shared owned projects - # that expired. - - # Shall remove all resources (data and running services) of expired users - - -@pytest.mark.skip(reason="Under dev -> resource management ") -def test_limit_guest_users(client): - # a guest user has limited access to resources - # also limited time and amount - # - pass diff --git a/services/web/server/tests/unit/with_dbs/01/test_storage.py b/services/web/server/tests/unit/with_dbs/01/test_storage.py index 32e469bcc0ea..1e3814eb74a7 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_storage.py +++ b/services/web/server/tests/unit/with_dbs/01/test_storage.py @@ -1,13 +1,15 @@ -# pylint:disable=unused-import -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable +# pylint: disable=too-many-arguments import asyncio +from collections.abc import Awaitable, Callable from urllib.parse import quote import pytest from aiohttp import web -from aiohttp.test_utils import TestServer +from aiohttp.test_utils import TestClient, TestServer from faker import Faker from pytest_simcore.helpers.assert_checks import assert_status from pytest_simcore.helpers.typing_env import EnvVarsDict @@ -22,9 +24,10 @@ @pytest.fixture() def storage_server( event_loop: asyncio.AbstractEventLoop, - aiohttp_server: TestServer, + aiohttp_server: Callable[..., Awaitable[TestServer]], app_environment: EnvVarsDict, -): + storage_test_server_port: int, +) -> TestServer: async def _get_locs(request: web.Request): assert not request.can_read_body @@ -125,6 +128,7 @@ async def _get_datasets_meta(request: web.Request): storage_api_version = app_environment["STORAGE_VTAG"] storage_port = int(app_environment["STORAGE_PORT"]) + assert storage_port == storage_test_server_port assert ( storage_api_version != API_VERSION @@ -164,7 +168,9 @@ async def _get_datasets_meta(request: web.Request): (UserRole.TESTER, status.HTTP_200_OK), ], ) -async def test_get_storage_locations(client, storage_server, logged_user, expected): +async def test_get_storage_locations( + client: TestClient, storage_server: TestServer, logged_user, expected +): url = "/v0/storage/locations" assert url.startswith(PREFIX) @@ -186,7 +192,9 @@ async def test_get_storage_locations(client, storage_server, logged_user, expect (UserRole.ADMIN, status.HTTP_200_OK), ], ) -async def test_sync_file_meta_table(client, storage_server, logged_user, expected): +async def test_sync_file_meta_table( + client: TestClient, storage_server: TestServer, logged_user, expected +): url = "/v0/storage/locations/0:sync" assert url.startswith(PREFIX) @@ -208,7 +216,9 @@ async def test_sync_file_meta_table(client, storage_server, logged_user, expecte (UserRole.TESTER, status.HTTP_200_OK), ], ) -async def test_get_datasets_metadata(client, storage_server, logged_user, expected): +async def test_get_datasets_metadata( + client: TestClient, storage_server: TestServer, logged_user, expected +): url = "/v0/storage/locations/0/datasets" assert url.startswith(PREFIX) @@ -234,7 +244,7 @@ async def test_get_datasets_metadata(client, storage_server, logged_user, expect ], ) async def test_get_files_metadata_dataset( - client, storage_server, logged_user, expected + client: TestClient, storage_server: TestServer, logged_user, expected ): url = "/v0/storage/locations/0/datasets/N:asdfsdf/metadata" assert url.startswith(PREFIX) @@ -263,7 +273,7 @@ async def test_get_files_metadata_dataset( ], ) async def test_storage_file_meta( - client, storage_server, logged_user, expected, faker: Faker + client: TestClient, storage_server: TestServer, logged_user, expected, faker: Faker ): # tests redirect of path with quotes in path file_id = f"{faker.uuid4()}/{faker.uuid4()}/a/b/c/d/e/dat" @@ -289,7 +299,9 @@ async def test_storage_file_meta( (UserRole.TESTER, status.HTTP_200_OK), ], ) -async def test_storage_list_filter(client, storage_server, logged_user, expected): +async def test_storage_list_filter( + client: TestClient, storage_server: TestServer, logged_user, expected +): # tests composition of 2 queries file_id = "a/b/c/d/e/dat" url = "/v0/storage/locations/0/files/metadata?uuid_filter={}".format( diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index cb1edb11460b..a608a59ef92e 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -118,12 +118,27 @@ def docker_compose_file(docker_compose_env: pytest.MonkeyPatch) -> str: # WEB SERVER/CLIENT FIXTURES ------------------------------------------------ +@pytest.fixture +def web_test_server_port(unused_tcp_port_factory: Callable): + return unused_tcp_port_factory() + + +@pytest.fixture +def storage_test_server_port( + unused_tcp_port_factory: Callable, web_test_server_port: int +): + port = unused_tcp_port_factory() + assert port != web_test_server_port + return port + + @pytest.fixture def app_environment( monkeypatch: pytest.MonkeyPatch, default_app_cfg: AppConfigDict, unused_tcp_port_factory: Callable, mock_env_devel_environment: EnvVarsDict, + storage_test_server_port: int, monkeypatch_setenv_from_app_config: Callable[[AppConfigDict], EnvVarsDict], ) -> EnvVarsDict: # WARNING: this fixture is commonly overriden. Check before renaming. @@ -138,7 +153,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc """ # NOTE: remains from from old cfg cfg = deepcopy(default_app_cfg) - cfg["storage"]["port"] = unused_tcp_port_factory() + cfg["storage"]["port"] = storage_test_server_port envs_app_cfg = monkeypatch_setenv_from_app_config(cfg) return ( @@ -189,6 +204,7 @@ def web_server( unused_tcp_port_factory: Callable, app_environment: EnvVarsDict, postgres_db: sa.engine.Engine, + web_test_server_port: int, # tools aiohttp_server: Callable, mocked_send_email: None, @@ -201,7 +217,7 @@ def web_server( disable_static_webserver(app) server = event_loop.run_until_complete( - aiohttp_server(app, port=unused_tcp_port_factory()) + aiohttp_server(app, port=web_test_server_port) ) assert isinstance(postgres_db, sa.engine.Engine) From bee1a67f4602fb9883d7dab86cf65df10ca213e7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 9 Jan 2025 23:40:30 +0100 Subject: [PATCH 25/71] fixes test --- .../server/tests/unit/with_dbs/03/version_control/conftest.py | 4 +--- .../with_dbs/03/version_control/test_version_control_core.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index 00f35441f8d4..53bdeb18c5b0 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -81,12 +81,10 @@ def app_environment( "WEBSERVER_CLUSTERS": "null", "WEBSERVER_COMPUTATION": "null", "WEBSERVER_DIAGNOSTICS": "null", + "WEBSERVER_GARBAGE_COLLECTOR": "null", "WEBSERVER_GROUPS": "0", "WEBSERVER_PUBLICATIONS": "0", - "WEBSERVER_GARBAGE_COLLECTOR": "null", - "WEBSERVER_EMAIL": "null", "WEBSERVER_SOCKETIO": "0", - "WEBSERVER_STORAGE": "null", "WEBSERVER_STUDIES_DISPATCHER": "null", "WEBSERVER_TAGS": "0", "WEBSERVER_TRACING": "null", diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py index fa05653b87ca..a0172815f8e1 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py @@ -2,13 +2,12 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -from typing import Awaitable, Callable +from collections.abc import Awaitable, Callable from uuid import UUID import pytest from aiohttp import web from aiohttp.test_utils import TestClient, make_mocked_request -from faker import Faker from simcore_service_webserver._constants import RQT_USERID_KEY from simcore_service_webserver.projects import projects_api from simcore_service_webserver.projects.models import ProjectDict @@ -32,7 +31,6 @@ def aiohttp_mocked_request(client: TestClient, user_id: int) -> web.Request: async def test_workflow( client: TestClient, project_uuid: UUID, - faker: Faker, user_id: int, user_project: ProjectDict, aiohttp_mocked_request: web.Request, From 2efe9a1e4fec9c191e12d32d4017f81735959d69 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 09:19:20 +0100 Subject: [PATCH 26/71] ruffed --- .../notifications/test_rabbitmq_consumers.py | 3 --- .../02/scicrunch/test_scicrunch__rest.py | 4 ---- .../tests/integration/02/test_computation.py | 2 +- .../test_exporter_formatter_archive.py | 2 +- .../tests/unit/isolated/test__configs.py | 7 +++--- .../tests/unit/isolated/test__redirections.py | 21 ++++++++-------- .../unit/isolated/test_login_settings.py | 2 +- .../unit/isolated/test_projects__db_utils.py | 7 +++--- .../server/tests/unit/isolated/test_rest.py | 3 +-- .../unit/isolated/test_security__authz.py | 7 ++---- ..._studies_dispatcher_projects_permalinks.py | 13 +++++----- .../test_studies_dispatcher_settings.py | 3 +-- .../unit/isolated/test_utils_rate_limiting.py | 5 ++-- .../01/groups/test_groups_handlers_users.py | 3 +-- .../tests/unit/with_dbs/01/test_storage.py | 3 +-- .../unit/with_dbs/02/test_project_lock.py | 3 ++- .../02/test_projects_crud_handlers__list.py | 6 ++--- .../02/test_projects_groups_handlers.py | 18 +++++++------- .../02/test_projects_metadata_handlers.py | 6 ----- ...st_projects_nodes_pricing_unit_handlers.py | 4 ++-- .../02/test_projects_ports_handlers.py | 2 +- .../02/test_projects_states_handlers.py | 6 ++--- .../02/test_projects_wallet_handlers.py | 6 ++--- .../unit/with_dbs/03/invitations/conftest.py | 4 ++-- .../test_products__invitations_handlers.py | 6 ++--- .../03/login/test_login_2fa_resend.py | 2 +- .../03/login/test_login_reset_password.py | 2 +- .../03/login/test_login_utils_emails.py | 6 ++--- .../test_meta_modeling_results.py | 2 +- ...t_list_osparc_credits_aggregated_usages.py | 8 +++---- .../tests/unit/with_dbs/03/test_project_db.py | 2 +- .../03/test_session_access_policies.py | 3 ++- .../tests/unit/with_dbs/03/test_socketio.py | 2 +- .../with_dbs/03/test_users__notifications.py | 8 +++---- .../unit/with_dbs/03/test_users__tokens.py | 2 +- .../test_version_control_core.py | 2 +- .../04/folders/test_folders__full_search.py | 2 +- .../test_resource_manager.py | 2 +- ...fications__db_comp_tasks_listening_task.py | 5 ++-- .../test_studies_dispatcher_handlers.py | 2 +- .../test_studies_dispatcher_projects.py | 3 +-- .../test_studies_dispatcher_studies_access.py | 7 +++--- .../with_dbs/04/workspaces/test_workspaces.py | 4 ++-- ...t_workspaces__folders_and_projects_crud.py | 4 ++-- .../04/workspaces/test_workspaces_groups.py | 24 +++++++++---------- 45 files changed, 107 insertions(+), 131 deletions(-) diff --git a/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py b/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py index f1d4aa621875..b59ddd49a7b5 100644 --- a/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py +++ b/services/web/server/tests/integration/02/notifications/test_rabbitmq_consumers.py @@ -220,7 +220,6 @@ async def test_log_workflow( # project random_node_id_in_user_project: NodeID, user_project_id: ProjectID, - # faker: Faker, mocker: MockerFixture, ): @@ -264,7 +263,6 @@ async def test_log_workflow_only_receives_messages_if_subscribed( # project random_node_id_in_user_project: NodeID, user_project_id: ProjectID, - # faker: Faker, mocker: MockerFixture, ): @@ -333,7 +331,6 @@ async def test_progress_non_computational_workflow( # project random_node_id_in_user_project: NodeID, user_project_id: ProjectID, - # mocker: MockerFixture, ): """ diff --git a/services/web/server/tests/integration/02/scicrunch/test_scicrunch__rest.py b/services/web/server/tests/integration/02/scicrunch/test_scicrunch__rest.py index 4ae3ca6a3e12..d42e8c42e90d 100644 --- a/services/web/server/tests/integration/02/scicrunch/test_scicrunch__rest.py +++ b/services/web/server/tests/integration/02/scicrunch/test_scicrunch__rest.py @@ -14,7 +14,6 @@ """ import os -from pprint import pprint from typing import Any import pytest @@ -48,7 +47,6 @@ async def test_scicrunch_openapi_specs(settings: SciCrunchSettings): async with ClientSession() as client: resp = await client.get(f"{SCICRUNCH_DEFAULT_URL}/swagger-docs/swagger.json") openapi_specs = await resp.json() - pprint(openapi_specs["info"]) expected_api_version = 1 assert openapi_specs["info"]["version"] == expected_api_version @@ -65,7 +63,6 @@ async def test_scicrunch_get_all_versions( ): async with ClientSession() as client: versions = await get_all_versions(rrid, client, settings) - pprint(versions) assert versions @@ -85,7 +82,6 @@ async def test_scicrunch_get_all_versions_with_invalid_rrids( ): async with ClientSession() as client: versions = await get_all_versions(rrid, client, settings) - pprint(versions) # invalid keys return success but an empty list of versions! assert isinstance(versions, list) diff --git a/services/web/server/tests/integration/02/test_computation.py b/services/web/server/tests/integration/02/test_computation.py index 36cb5f972f47..75c22ef3ac75 100644 --- a/services/web/server/tests/integration/02/test_computation.py +++ b/services/web/server/tests/integration/02/test_computation.py @@ -199,7 +199,7 @@ def _assert_db_contents( for task_db in tasks_db: assert task_db.project_id == project_id - assert task_db.node_id in mock_pipeline.keys() + assert task_db.node_id in mock_pipeline assert task_db.inputs == mock_pipeline[task_db.node_id].get("inputs") diff --git a/services/web/server/tests/unit/isolated/exporter/test_exporter_formatter_archive.py b/services/web/server/tests/unit/isolated/exporter/test_exporter_formatter_archive.py index 906b465a3a67..5d468b043a4b 100644 --- a/services/web/server/tests/unit/isolated/exporter/test_exporter_formatter_archive.py +++ b/services/web/server/tests/unit/isolated/exporter/test_exporter_formatter_archive.py @@ -1,8 +1,8 @@ # pylint:disable=redefined-outer-name import tempfile +from collections.abc import Iterator from pathlib import Path -from typing import Iterator import pytest from faker import Faker diff --git a/services/web/server/tests/unit/isolated/test__configs.py b/services/web/server/tests/unit/isolated/test__configs.py index ccd4247ea775..c811b9a0b920 100644 --- a/services/web/server/tests/unit/isolated/test__configs.py +++ b/services/web/server/tests/unit/isolated/test__configs.py @@ -16,7 +16,8 @@ @pytest.fixture(scope="session") def app_config_schema(): - raise RuntimeError("DEPRECATED. MUST NOT BE USED") + msg = "DEPRECATED. MUST NOT BE USED" + raise RuntimeError(msg) @pytest.fixture(scope="session") @@ -33,7 +34,7 @@ def service_webserver_environ( ), # defined if pip install --edit (but not in travis!) } - webserver_environ = eval_service_environ( + return eval_service_environ( services_docker_compose_file, "webserver", host_environ, @@ -41,8 +42,6 @@ def service_webserver_environ( use_env_devel=True, ) - return webserver_environ - @pytest.fixture(scope="session") def app_submodules_with_setup_funs(package_dir: Path) -> set[ModuleType]: diff --git a/services/web/server/tests/unit/isolated/test__redirections.py b/services/web/server/tests/unit/isolated/test__redirections.py index 978a41195be4..6274087326f5 100644 --- a/services/web/server/tests/unit/isolated/test__redirections.py +++ b/services/web/server/tests/unit/isolated/test__redirections.py @@ -5,9 +5,8 @@ import asyncio import textwrap -from collections.abc import Callable +from collections.abc import Awaitable, Callable from pathlib import Path -from typing import Awaitable import pytest from aiohttp import web @@ -47,30 +46,32 @@ def client( @routes.get("/") async def get_root(_request): - raise web.HTTPOk() + raise web.HTTPOk @routes.get("/other") async def get_other(_request): - raise web.HTTPOk() + raise web.HTTPOk @routes.get("/redirect-to-other") async def get_redirect_to_other(request): - raise web.HTTPFound("/other") + msg = "/other" + raise web.HTTPFound(msg) @routes.get("/redirect-to-static") async def get_redirect_to_static(_request): - raise web.HTTPFound("/statics/index.html") + msg = "/statics/index.html" + raise web.HTTPFound(msg) @routes.get("/redirect-to-root") async def get_redirect_to_root(_request): - raise web.HTTPFound("/") + msg = "/" + raise web.HTTPFound(msg) routes.static("/statics", index_static_path) app = web.Application() app.add_routes(routes) - cli = event_loop.run_until_complete(aiohttp_client(app)) - return cli + return event_loop.run_until_complete(aiohttp_client(app)) @pytest.mark.parametrize("test_path", ["/", "/other"]) @@ -80,7 +81,7 @@ async def test_preserves_fragments(client, test_path): assert resp.real_url.fragment == "this/is/a/fragment" -@pytest.mark.xfail +@pytest.mark.xfail() @pytest.mark.parametrize( "test_path,expected_redirected_path", [ diff --git a/services/web/server/tests/unit/isolated/test_login_settings.py b/services/web/server/tests/unit/isolated/test_login_settings.py index 0bfff4469119..9a92dad75427 100644 --- a/services/web/server/tests/unit/isolated/test_login_settings.py +++ b/services/web/server/tests/unit/isolated/test_login_settings.py @@ -34,7 +34,7 @@ def twilio_config(monkeypatch: pytest.MonkeyPatch) -> dict[str, str]: "TWILIO_MESSAGING_SID": "x" * 34, } # NOTE: enforces DELETE-ENV since apparently some session-based fixtures are settings these envs - for key in TWILO_CONFIG.keys(): + for key in TWILO_CONFIG: monkeypatch.delenv(key, raising=False) return TWILO_CONFIG diff --git a/services/web/server/tests/unit/isolated/test_projects__db_utils.py b/services/web/server/tests/unit/isolated/test_projects__db_utils.py index c8c4da57eda3..2a5203ae137b 100644 --- a/services/web/server/tests/unit/isolated/test_projects__db_utils.py +++ b/services/web/server/tests/unit/isolated/test_projects__db_utils.py @@ -5,9 +5,10 @@ import datetime import json import re +from collections.abc import Callable from copy import deepcopy from dataclasses import dataclass -from typing import Any, Callable +from typing import Any import pytest from faker import Faker @@ -95,7 +96,7 @@ def test_convert_to_schema_names(fake_project: dict[str, Any]): assert col is not None # test date time conversion - date = datetime.datetime.now(datetime.timezone.utc) + date = datetime.datetime.now(datetime.UTC) db_entries["creation_date"] = date schema_entries = convert_to_schema_names(db_entries, fake_project["prjOwner"]) assert "creationDate" in schema_entries @@ -110,7 +111,7 @@ def test_convert_to_schema_names_camel_casing(fake_db_dict): assert "anEntryThatUsesSnakeCase" in db_entries assert "anotherEntryThatUsesSnakeCase" in db_entries # test date time conversion - date = datetime.datetime.now(datetime.timezone.utc) + date = datetime.datetime.now(datetime.UTC) fake_db_dict["time_entry"] = date db_entries = convert_to_schema_names(fake_db_dict, fake_email) assert "timeEntry" in db_entries diff --git a/services/web/server/tests/unit/isolated/test_rest.py b/services/web/server/tests/unit/isolated/test_rest.py index 12d4ed95fe99..31fdba39eac3 100644 --- a/services/web/server/tests/unit/isolated/test_rest.py +++ b/services/web/server/tests/unit/isolated/test_rest.py @@ -48,10 +48,9 @@ async def slow_handler(request: web.Request): app.router.add_get("/slow", slow_handler) - cli = event_loop.run_until_complete( + return event_loop.run_until_complete( aiohttp_client(app, server_kwargs=server_kwargs) ) - return cli async def test_frontend_config( diff --git a/services/web/server/tests/unit/isolated/test_security__authz.py b/services/web/server/tests/unit/isolated/test_security__authz.py index 5ba406db5109..2fb280cf25f9 100644 --- a/services/web/server/tests/unit/isolated/test_security__authz.py +++ b/services/web/server/tests/unit/isolated/test_security__authz.py @@ -97,10 +97,7 @@ def test_unique_permissions(): for permission in can: assert ( permission not in used - ), "'{}' in {} is repeated in security_roles.ROLES_PERMISSIONS".format( - permission, - role, - ) + ), f"'{permission}' in {role} is repeated in security_roles.ROLES_PERMISSIONS" used.append(permission) @@ -263,7 +260,7 @@ async def _fake_db(engine, email): raise DatabaseError # inactive user or not found - return copy.deepcopy(users_db.get(email, None)) + return copy.deepcopy(users_db.get(email)) mock_db_fun = mocker.patch( "simcore_service_webserver.security._authz_policy.get_active_user_or_none", diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py index 6858ac07ad9b..f42834051083 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_projects_permalinks.py @@ -61,7 +61,6 @@ def app_environment( "WEBSERVER_TRACING": "null", "WEBSERVER_VERSION_CONTROL": "0", "WEBSERVER_WALLETS": "0", - # "STUDIES_ACCESS_ANONYMOUS_ALLOWED": "1", }, ) @@ -112,12 +111,12 @@ def test_create_permalink(fake_get_project_request: web.Request, is_public: bool def valid_project_kwargs( request: pytest.FixtureRequest, fake_get_project_request: web.Request ): - return dict( - project_uuid=fake_get_project_request.match_info["project_uuid"], - project_type=ProjectType.TEMPLATE, - project_access_rights={"1": {"read": True, "write": False, "delete": False}}, - project_is_public=request.param, - ) + return { + "project_uuid": fake_get_project_request.match_info["project_uuid"], + "project_type": ProjectType.TEMPLATE, + "project_access_rights": {"1": {"read": True, "write": False, "delete": False}}, + "project_is_public": request.param, + } def test_permalink_only_for_template_projects( diff --git a/services/web/server/tests/unit/isolated/test_studies_dispatcher_settings.py b/services/web/server/tests/unit/isolated/test_studies_dispatcher_settings.py index 5c4377d56fdf..c2d9a8b14991 100644 --- a/services/web/server/tests/unit/isolated/test_studies_dispatcher_settings.py +++ b/services/web/server/tests/unit/isolated/test_studies_dispatcher_settings.py @@ -19,11 +19,10 @@ @pytest.fixture def environment(monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: - envs = setenvs_from_dict( + return setenvs_from_dict( monkeypatch, envs=StudiesDispatcherSettings.model_config["json_schema_extra"]["example"], ) - return envs def test_studies_dispatcher_settings(environment: EnvVarsDict): diff --git a/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py b/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py index fed37d1ba4e7..c6da8adf7367 100644 --- a/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py +++ b/services/web/server/tests/unit/isolated/test_utils_rate_limiting.py @@ -4,8 +4,8 @@ import asyncio import time -from collections.abc import Awaitable -from typing import Callable +from collections.abc import Awaitable, Callable +from typing import Annotated import pytest from aiohttp import web @@ -13,7 +13,6 @@ from aiohttp.web_exceptions import HTTPOk, HTTPTooManyRequests from pydantic import Field, TypeAdapter, ValidationError from simcore_service_webserver.utils_rate_limiting import global_rate_limit_route -from typing_extensions import Annotated TOTAL_TEST_TIME = 1 # secs MAX_NUM_REQUESTS = 3 diff --git a/services/web/server/tests/unit/with_dbs/01/groups/test_groups_handlers_users.py b/services/web/server/tests/unit/with_dbs/01/groups/test_groups_handlers_users.py index 0575ae5a4ff8..7661d74443e3 100644 --- a/services/web/server/tests/unit/with_dbs/01/groups/test_groups_handlers_users.py +++ b/services/web/server/tests/unit/with_dbs/01/groups/test_groups_handlers_users.py @@ -4,9 +4,8 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -from collections.abc import AsyncIterator +from collections.abc import AsyncIterable, AsyncIterator from contextlib import AsyncExitStack -from typing import AsyncIterable import pytest from aiohttp.test_utils import TestClient diff --git a/services/web/server/tests/unit/with_dbs/01/test_storage.py b/services/web/server/tests/unit/with_dbs/01/test_storage.py index 1e3814eb74a7..e03c838cd0aa 100644 --- a/services/web/server/tests/unit/with_dbs/01/test_storage.py +++ b/services/web/server/tests/unit/with_dbs/01/test_storage.py @@ -151,8 +151,7 @@ async def _get_datasets_meta(request: web.Request): _get_datasets_meta, ) - server = event_loop.run_until_complete(aiohttp_server(app, port=storage_port)) - return server + return event_loop.run_until_complete(aiohttp_server(app, port=storage_port)) # -------------------------------------------------------------------------- diff --git a/services/web/server/tests/unit/with_dbs/02/test_project_lock.py b/services/web/server/tests/unit/with_dbs/02/test_project_lock.py index 5c33586e151f..71d4602a1f8c 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_project_lock.py +++ b/services/web/server/tests/unit/with_dbs/02/test_project_lock.py @@ -122,7 +122,8 @@ async def test_raise_exception_while_locked_release_lock( ) assert redis_value # now raising an exception - raise ValueError("pytest exception") + msg = "pytest exception" + raise ValueError(msg) # now the lock shall be released redis_value = await redis_locks_client.get( PROJECT_REDIS_LOCK_KEY.format(project_uuid) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list.py index f0147c7eb023..d16058179be6 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__list.py @@ -32,7 +32,7 @@ def _extract(dikt, keys): modified = [ "lastChangeDate", ] - keep = [k for k in update_data.keys() if k not in modified] + keep = [k for k in update_data if k not in modified] assert _extract(current_project, keep) == _extract(update_data, keep) @@ -86,7 +86,7 @@ async def _list_projects( ) ) if exp_offset <= 0: - assert links["prev"] == None + assert links["prev"] is None else: assert links["prev"] == str( URL(complete_url).update_query( @@ -94,7 +94,7 @@ async def _list_projects( ) ) if exp_offset >= (exp_last_page * exp_limit): - assert links["next"] == None + assert links["next"] is None else: assert links["next"] == str( URL(complete_url).update_query( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_groups_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_groups_handlers.py index fe3ecfa8c3ae..765691510683 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_groups_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_groups_handlers.py @@ -111,9 +111,9 @@ async def test_projects_groups_full_workflow( data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 2 assert data[1]["gid"] == new_user["primary_gid"] - assert data[1]["read"] == True - assert data[1]["write"] == False - assert data[1]["delete"] == False + assert data[1]["read"] is True + assert data[1]["write"] is False + assert data[1]["delete"] is False # Get the project endpoint and check the permissions url = client.app.router["get_project"].url_for( @@ -163,9 +163,9 @@ async def test_projects_groups_full_workflow( ) data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["gid"] == new_user["primary_gid"] - assert data["read"] == True - assert data["write"] == True - assert data["delete"] == False + assert data["read"] is True + assert data["write"] is True + assert data["delete"] is False # List the project groups url = client.app.router["list_project_groups"].url_for( @@ -175,9 +175,9 @@ async def test_projects_groups_full_workflow( data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 2 assert data[1]["gid"] == new_user["primary_gid"] - assert data[1]["read"] == True - assert data[1]["write"] == True - assert data[1]["delete"] == False + assert data[1]["read"] is True + assert data[1]["write"] is True + assert data[1]["delete"] is False # Get the project endpoint and check the permissions url = client.app.router["get_project"].url_for( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py index ca1935443028..ce87970f75c6 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_metadata_handlers.py @@ -43,7 +43,6 @@ async def test_custom_metadata_handlers( # for deletion mocked_dynamic_services_interface: dict[str, MagicMock], storage_subsystem_mock: MockedStorageSubsystem, - # client: TestClient, faker: Faker, logged_user: UserInfoDict, @@ -118,7 +117,6 @@ async def test_new_project_with_parent_project_node( # for deletion mocked_dynamic_services_interface: dict[str, MagicMock], storage_subsystem_mock: MockedStorageSubsystem, - # client: TestClient, logged_user: UserInfoDict, primary_group: dict[str, str], @@ -196,7 +194,6 @@ async def test_new_project_with_invalid_parent_project_node( # for deletion mocked_dynamic_services_interface: dict[str, MagicMock], storage_subsystem_mock: MockedStorageSubsystem, - # client: TestClient, logged_user: UserInfoDict, primary_group: dict[str, str], @@ -280,7 +277,6 @@ async def test_set_project_parent_backward_compatibility( # for deletion mocked_dynamic_services_interface: dict[str, MagicMock], storage_subsystem_mock: MockedStorageSubsystem, - # client: TestClient, logged_user: UserInfoDict, primary_group: dict[str, str], @@ -344,7 +340,6 @@ async def test_update_project_metadata_backward_compatibility_with_same_project_ # for deletion mocked_dynamic_services_interface: dict[str, MagicMock], storage_subsystem_mock: MockedStorageSubsystem, - # client: TestClient, faker: Faker, logged_user: UserInfoDict, @@ -400,7 +395,6 @@ async def test_update_project_metadata_s4lacad_backward_compatibility_passing_ni # for deletion mocked_dynamic_services_interface: dict[str, MagicMock], storage_subsystem_mock: MockedStorageSubsystem, - # client: TestClient, logged_user: UserInfoDict, primary_group: dict[str, str], diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py index 5812190c354e..1ebd06d73929 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_pricing_unit_handlers.py @@ -73,10 +73,10 @@ async def test_project_node_pricing_unit_user_project_access( ) resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, expected) - assert data == None + assert data is None # Now we will log as a different user who doesnt have access to the project - async with LoggedUser(client) as new_logged_user: + async with LoggedUser(client): base_url = client.app.router["get_project_node_pricing_unit"].url_for( project_id=user_project["uuid"], node_id=node_id ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py index db4596f06ec1..95c18aca7c89 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_ports_handlers.py @@ -236,7 +236,7 @@ async def test_io_workflow( }, "76f607b4-8761-4f96-824d-cab670bc45f5": { "key": "76f607b4-8761-4f96-824d-cab670bc45f5", - "value": 6, # + "value": 6, "label": "Random sleep interval", }, } diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index f3b91131b1a3..6fcd177f37b2 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -163,7 +163,7 @@ async def _open_project( try: data, error = await assert_status(resp, e) return data, error - except AssertionError: # noqa: PERF203 + except AssertionError: # re-raise if last item if e == expected[-1]: raise @@ -1235,8 +1235,8 @@ async def test_open_shared_project_2_users_locked( # now the expected result is that the project is locked and opened by client 1 owner1 = Owner( user_id=logged_user["id"], - first_name=logged_user.get("first_name", None), - last_name=logged_user.get("last_name", None), + first_name=logged_user.get("first_name"), + last_name=logged_user.get("last_name"), ) expected_project_state_client_1.locked.value = True expected_project_state_client_1.locked.status = ProjectStatus.OPENED diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_wallet_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_wallet_handlers.py index 30aaa89abbc8..132c6f21252c 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_wallet_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_wallet_handlers.py @@ -63,10 +63,10 @@ async def test_project_wallets_user_project_access( ) resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, expected) - assert data == None + assert data is None # Now we will log as a different user who doesnt have access to the project - async with LoggedUser(client) as new_logged_user: + async with LoggedUser(client): base_url = client.app.router["get_project_wallet"].url_for( project_id=user_project["uuid"] ) @@ -112,7 +112,7 @@ async def test_project_wallets_full_workflow( ) resp = await client.get(f"{base_url}") data, _ = await assert_status(resp, expected) - assert data == None + assert data is None # Now we will connect the wallet base_url = client.app.router["connect_wallet_to_project"].url_for( diff --git a/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py b/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py index 247289ca322b..48d8f1ac41fc 100644 --- a/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/invitations/conftest.py @@ -8,7 +8,7 @@ import json from contextlib import suppress from copy import deepcopy -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -155,7 +155,7 @@ def _generate(url, **kwargs): **example, **body, "invitation_url": f"https://osparc-simcore.test/#/registration?invitation={fake_code}", - "created": datetime.now(tz=timezone.utc), + "created": datetime.now(tz=UTC), } ) ), diff --git a/services/web/server/tests/unit/with_dbs/03/invitations/test_products__invitations_handlers.py b/services/web/server/tests/unit/with_dbs/03/invitations/test_products__invitations_handlers.py index 64aec0a93d95..9f347239acd7 100644 --- a/services/web/server/tests/unit/with_dbs/03/invitations/test_products__invitations_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/invitations/test_products__invitations_handlers.py @@ -4,7 +4,7 @@ # pylint: disable=too-many-arguments -from datetime import datetime, timezone +from datetime import UTC, datetime from http import HTTPStatus from typing import Final @@ -81,7 +81,7 @@ async def test_product_owner_generates_invitation( trial_account_days: PositiveInt | None, extra_credits_in_usd: PositiveInt | None, ): - before_dt = datetime.now(tz=timezone.utc) + before_dt = datetime.now(tz=UTC) request_model = GenerateInvitation( guest=guest_email, @@ -109,7 +109,7 @@ async def test_product_owner_generates_invitation( product_base_url = f"{client.make_url('/')}" assert f"{got.invitation_link}".startswith(product_base_url) assert before_dt < got.created - assert got.created < datetime.now(tz=timezone.utc) + assert got.created < datetime.now(tz=UTC) MANY_TIMES: Final = 2 diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa_resend.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa_resend.py index 2cf5b63eb240..1f3c76c2ea84 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa_resend.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_2fa_resend.py @@ -78,7 +78,7 @@ async def test_resend_2fa_workflow( assert client.app # spy send functions - mock_send_sms_code1 = mocker.patch( + mocker.patch( "simcore_service_webserver.login._2fa_handlers.send_sms_code", autospec=True ) mock_send_sms_code2 = mocker.patch( diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_reset_password.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_reset_password.py index dd4a49a698aa..a5c95ba7c3bb 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_reset_password.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_reset_password.py @@ -176,7 +176,7 @@ async def test_reset_and_confirm( assert ( response.url.path_qs == URL(login_options.LOGIN_REDIRECT) - .with_fragment("reset-password?code=%s" % code) + .with_fragment(f"reset-password?code={code}") .path_qs ) diff --git a/services/web/server/tests/unit/with_dbs/03/login/test_login_utils_emails.py b/services/web/server/tests/unit/with_dbs/03/login/test_login_utils_emails.py index 42d803d67102..e5e417bb8fc9 100644 --- a/services/web/server/tests/unit/with_dbs/03/login/test_login_utils_emails.py +++ b/services/web/server/tests/unit/with_dbs/03/login/test_login_utils_emails.py @@ -56,18 +56,16 @@ async def print_mail(*, message, settings): print(message) print("---------------") - mock = mocker.patch( + return mocker.patch( "simcore_service_webserver.email._core._do_send_mail", spec=True, side_effect=print_mail, ) - return mock @pytest.fixture def destination_email(faker: Faker) -> str: - email = faker.email() - return email + return faker.email() @pytest.mark.parametrize("product_name", FRONTEND_APPS_AVAILABLE) diff --git a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_results.py b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_results.py index 8c86119bfc1b..fdec06806eda 100644 --- a/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_results.py +++ b/services/web/server/tests/unit/with_dbs/03/meta_modeling/test_meta_modeling_results.py @@ -127,7 +127,7 @@ def test_extract_project_results(fake_workbench: dict[str, Any]): @pytest.mark.parametrize( "model_cls", - (ExtractedResults,), + [ExtractedResults], ) def test_models_examples( model_cls: type[BaseModel], model_cls_examples: dict[str, Any] diff --git a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_list_osparc_credits_aggregated_usages.py b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_list_osparc_credits_aggregated_usages.py index 5f7f7c41b180..bb6f2123276a 100644 --- a/services/web/server/tests/unit/with_dbs/03/resource_usage/test_list_osparc_credits_aggregated_usages.py +++ b/services/web/server/tests/unit/with_dbs/03/resource_usage/test_list_osparc_credits_aggregated_usages.py @@ -27,11 +27,9 @@ _SERVICE_RUN_GET = OsparcCreditsAggregatedUsagesPage( items=[ OsparcCreditsAggregatedByServiceGet( - **{ - "osparc_credits": Decimal(-50), - "service_key": "simcore/services/comp/itis/sleeper", - "running_time_in_hours": Decimal(0.5), - } + osparc_credits=Decimal(-50), + service_key="simcore/services/comp/itis/sleeper", + running_time_in_hours=Decimal(0.5), ) ], total=1, diff --git a/services/web/server/tests/unit/with_dbs/03/test_project_db.py b/services/web/server/tests/unit/with_dbs/03/test_project_db.py index 1ab6ca802f3d..098a4a783888 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_project_db.py +++ b/services/web/server/tests/unit/with_dbs/03/test_project_db.py @@ -504,7 +504,7 @@ async def test_patch_user_project_workbench_concurrently( # patch all the nodes concurrently randomly_created_outputs = [ { - "outputs": {f"out_{k}": f"{k}"} # noqa: RUF011 + "outputs": {f"out_{k}": f"{k}"} # noqa: B035 for k in range(randint(1, 10)) # noqa: S311 } for n in range(_NUMBER_OF_NODES) diff --git a/services/web/server/tests/unit/with_dbs/03/test_session_access_policies.py b/services/web/server/tests/unit/with_dbs/03/test_session_access_policies.py index d19d2dca6ae4..dd79e1a9b6c6 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_session_access_policies.py +++ b/services/web/server/tests/unit/with_dbs/03/test_session_access_policies.py @@ -5,7 +5,8 @@ from asyncio import AbstractEventLoop -from typing import Callable, Protocol +from collections.abc import Callable +from typing import Protocol import pytest from aiohttp import ClientResponse, web diff --git a/services/web/server/tests/unit/with_dbs/03/test_socketio.py b/services/web/server/tests/unit/with_dbs/03/test_socketio.py index 699ff0ccef94..5b63d8f3c84f 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_socketio.py +++ b/services/web/server/tests/unit/with_dbs/03/test_socketio.py @@ -54,7 +54,7 @@ def app_environment(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatc @pytest.mark.skip( reason="Pending https://github.com/ITISFoundation/osparc-simcore/issues/5332" ) -@pytest.mark.parametrize("user_role", (UserRole.USER,)) +@pytest.mark.parametrize("user_role", [UserRole.USER]) async def test_socketio_session_client_to_server( logged_user: UserInfoDict, client: TestClient, diff --git a/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py b/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py index ccf246540bd0..a1d99e48268f 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py +++ b/services/web/server/tests/unit/with_dbs/03/test_users__notifications.py @@ -10,7 +10,7 @@ from collections.abc import AsyncIterable, AsyncIterator from contextlib import asynccontextmanager from copy import deepcopy -from datetime import datetime, timezone +from datetime import UTC, datetime from http import HTTPStatus from typing import Any @@ -76,7 +76,7 @@ def _create_notification( "actionable_path": "a/path", "title": "test_title", "text": "text_text", - "date": datetime.now(timezone.utc).isoformat(), + "date": datetime.now(UTC).isoformat(), "product": product_name, } ) @@ -154,7 +154,7 @@ async def test_list_user_notifications( result = TypeAdapter(list[UserNotification]).validate_python( json_response["data"] - ) # noqa: F821 + ) assert len(result) <= MAX_NOTIFICATIONS_FOR_USER_TO_SHOW assert result == list( reversed(created_notifications[:MAX_NOTIFICATIONS_FOR_USER_TO_SHOW]) @@ -229,7 +229,7 @@ async def test_create_user_notification( # these are always generated and overwritten, even if provided by the user, since # we do not want to overwrite existing ones assert user_notifications[0].read is False - assert user_notifications[0].id != notification_dict.get("id", None) + assert user_notifications[0].id != notification_dict.get("id") else: assert error is not None diff --git a/services/web/server/tests/unit/with_dbs/03/test_users__tokens.py b/services/web/server/tests/unit/with_dbs/03/test_users__tokens.py index fd040e1d88a7..76481526d96e 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_users__tokens.py +++ b/services/web/server/tests/unit/with_dbs/03/test_users__tokens.py @@ -183,7 +183,7 @@ async def test_delete_token( sid = fake_tokens[0]["service"] url = client.app.router["delete_token"].url_for(service=sid) - assert "/v0/me/tokens/%s" % sid == str(url) + assert f"/v0/me/tokens/{sid}" == str(url) resp = await client.delete(url.path) diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py index a0172815f8e1..c5ca46bf52c1 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_core.py @@ -27,7 +27,7 @@ def aiohttp_mocked_request(client: TestClient, user_id: int) -> web.Request: return req -@pytest.mark.acceptance_test +@pytest.mark.acceptance_test() async def test_workflow( client: TestClient, project_uuid: UUID, diff --git a/services/web/server/tests/unit/with_dbs/04/folders/test_folders__full_search.py b/services/web/server/tests/unit/with_dbs/04/folders/test_folders__full_search.py index e9bde5d9ec52..6fdfe372c6b2 100644 --- a/services/web/server/tests/unit/with_dbs/04/folders/test_folders__full_search.py +++ b/services/web/server/tests/unit/with_dbs/04/folders/test_folders__full_search.py @@ -112,7 +112,7 @@ async def test_folders_full_search( assert len(data) == 1 # Create new user - async with LoggedUser(client) as new_logged_user: + async with LoggedUser(client): # list full folder search url = client.app.router["list_folders_full_search"].url_for() resp = await client.get(f"{url}") diff --git a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py index 288e19562ec6..c55b2e38e136 100644 --- a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py @@ -347,7 +347,7 @@ async def test_websocket_multiple_connections( clients.append(sio) resource_keys.append(resource_key) - for sio, resource_key in zip(clients, resource_keys): + for sio, resource_key in zip(clients, resource_keys, strict=False): sid = sio.get_sid() await sio.disconnect() await sio.wait() diff --git a/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py b/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py index 49989b3fa313..c17c5aa1aa69 100644 --- a/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py +++ b/services/web/server/tests/unit/with_dbs/04/notifications/test_notifications__db_comp_tasks_listening_task.py @@ -6,7 +6,8 @@ import json import logging -from typing import Any, AsyncIterator, Awaitable, Callable, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable, Iterator +from typing import Any from unittest import mock import aiopg.sa @@ -53,7 +54,7 @@ async def mock_project_subsystem( return_value="", ) - yield mocked_project_calls + return mocked_project_calls @pytest.fixture diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py index 86ed849075fa..29c3251bbad0 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_handlers.py @@ -317,7 +317,7 @@ async def assert_redirected_to_study( assert resp.url.path == "/" assert ( "OSPARC-SIMCORE" in content - ), "Expected front-end rendering workbench's study, got %s" % str(content) + ), f"Expected front-end rendering workbench's study, got {content!s}" # Expects auth cookie for current user assert DEFAULT_SESSION_COOKIE_NAME in [c.key for c in session.cookie_jar] diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py index cd9bc502089e..1488ea3bbc09 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_projects.py @@ -73,8 +73,7 @@ def viewer_id(faker: Faker) -> NodeID: @pytest.fixture def viewer_info(view: dict[str, Any]) -> ViewerInfo: view.setdefault("label", view.pop("display_name", "Undefined")) - viewer_ = ViewerInfo(**view) - return viewer_ + return ViewerInfo(**view) @pytest.mark.parametrize("only_service", [True, False]) diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py index 366f10dba16f..582376778fa9 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -207,7 +207,7 @@ async def _assert_redirected_to_study( assert response.url.path == "/" assert ( "OSPARC-SIMCORE" in content - ), "Expected front-end rendering workbench's study, got %s" % str(content) + ), f"Expected front-end rendering workbench's study, got {content!s}" # Expects fragment to indicate client where to find newly created project m = re.match(r"/study/([\d\w-]+)", response.real_url.fragment) @@ -217,8 +217,7 @@ async def _assert_redirected_to_study( assert _is_user_authenticated(session) # returns newly created project - redirected_project_id = m.group(1) - return redirected_project_id + return m.group(1) # ----------------------------------------------------------- @@ -333,7 +332,7 @@ async def test_access_study_by_logged_user( user_project = projects[0] # heck redirects to /#/study/{uuid} - assert resp.real_url.fragment.endswith("/study/%s" % user_project["uuid"]) + assert resp.real_url.fragment.endswith("/study/{}".format(user_project["uuid"])) _assert_same_projects(user_project, published_project) assert user_project["prjOwner"] == logged_user["email"] diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py index 7e45b93400a7..56753822a6df 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py @@ -169,7 +169,7 @@ async def test_list_workspaces_with_text_search( }, ) data, _ = await assert_status(resp, status.HTTP_201_CREATED) - added_workspace = WorkspaceGet.model_validate(data) + WorkspaceGet.model_validate(data) # CREATE a new workspace url = client.app.router["create_workspace"].url_for() @@ -182,7 +182,7 @@ async def test_list_workspaces_with_text_search( }, ) data, _ = await assert_status(resp, status.HTTP_201_CREATED) - added_workspace = WorkspaceGet.model_validate(data) + WorkspaceGet.model_validate(data) # LIST user workspaces url = client.app.router["list_workspaces"].url_for() diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py index 3d5c2d7991a7..d8053ab82640 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces__folders_and_projects_crud.py @@ -418,7 +418,7 @@ async def test_listing_folders_and_projects_in_workspace__multiple_workspaces_cr # Create project in workspace project_data = deepcopy(fake_project) project_data["workspace_id"] = f"{added_workspace_1['workspaceId']}" - project = await create_project( + await create_project( client.app, project_data, user_id=logged_user["id"], @@ -451,7 +451,7 @@ async def test_listing_folders_and_projects_in_workspace__multiple_workspaces_cr # Create project in workspace project_data = deepcopy(fake_project) project_data["workspace_id"] = f"{added_workspace_2['workspaceId']}" - project = await create_project( + await create_project( client.app, project_data, user_id=logged_user["id"], diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces_groups.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces_groups.py index 3bb41d3bd374..af19129d88ca 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces_groups.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces_groups.py @@ -40,9 +40,9 @@ async def test_workspaces_groups_full_workflow( data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 1 assert data[0]["gid"] == logged_user["primary_gid"] - assert data[0]["read"] == True - assert data[0]["write"] == True - assert data[0]["delete"] == True + assert data[0]["read"] is True + assert data[0]["write"] is True + assert data[0]["delete"] is True async with NewUser( app=client.app, @@ -65,9 +65,9 @@ async def test_workspaces_groups_full_workflow( data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 2 assert data[1]["gid"] == new_user["primary_gid"] - assert data[1]["read"] == True - assert data[1]["write"] == False - assert data[1]["delete"] == False + assert data[1]["read"] is True + assert data[1]["write"] is False + assert data[1]["delete"] is False # Update the workspace permissions of the added user url = client.app.router["replace_workspace_group"].url_for( @@ -79,9 +79,9 @@ async def test_workspaces_groups_full_workflow( ) data, _ = await assert_status(resp, status.HTTP_200_OK) assert data["gid"] == new_user["primary_gid"] - assert data["read"] == True - assert data["write"] == True - assert data["delete"] == False + assert data["read"] is True + assert data["write"] is True + assert data["delete"] is False # List the workspace groups url = client.app.router["list_workspace_groups"].url_for( @@ -91,9 +91,9 @@ async def test_workspaces_groups_full_workflow( data, _ = await assert_status(resp, status.HTTP_200_OK) assert len(data) == 2 assert data[1]["gid"] == new_user["primary_gid"] - assert data[1]["read"] == True - assert data[1]["write"] == True - assert data[1]["delete"] == False + assert data[1]["read"] is True + assert data[1]["write"] is True + assert data[1]["delete"] is False # Delete the workspace group url = client.app.router["delete_workspace_group"].url_for( From 7d9271521babf646397e2a6e369f29cdc322398d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 09:32:51 +0100 Subject: [PATCH 27/71] minor --- .../web/server/tests/unit/with_dbs/conftest.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index a608a59ef92e..b4c4c8752c7e 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -66,6 +66,7 @@ get_default_product_name, get_or_create_product_group, ) +from simcore_service_director.core.settings import get_application_settings from simcore_service_webserver._constants import INDEX_RESOURCE_NAME from simcore_service_webserver.application import create_application from simcore_service_webserver.application_settings_utils import AppConfigDict @@ -97,6 +98,7 @@ def disable_swagger_doc_generation( def docker_compose_env(default_app_cfg: AppConfigDict) -> Iterator[pytest.MonkeyPatch]: postgres_cfg = default_app_cfg["db"]["postgres"] redis_cfg = default_app_cfg["resource_manager"]["redis"] + # docker-compose reads these environs with pytest.MonkeyPatch().context() as patcher: patcher.setenv("TEST_POSTGRES_DB", postgres_cfg["database"]) @@ -119,16 +121,18 @@ def docker_compose_file(docker_compose_env: pytest.MonkeyPatch) -> str: @pytest.fixture -def web_test_server_port(unused_tcp_port_factory: Callable): +def webserver_test_server_port(unused_tcp_port_factory: Callable): + # used to create a TestServer that emulates web-server service return unused_tcp_port_factory() @pytest.fixture def storage_test_server_port( - unused_tcp_port_factory: Callable, web_test_server_port: int + unused_tcp_port_factory: Callable, webserver_test_server_port: int ): + # used to create a TestServer that emulates storage service port = unused_tcp_port_factory() - assert port != web_test_server_port + assert port != webserver_test_server_port return port @@ -136,7 +140,6 @@ def storage_test_server_port( def app_environment( monkeypatch: pytest.MonkeyPatch, default_app_cfg: AppConfigDict, - unused_tcp_port_factory: Callable, mock_env_devel_environment: EnvVarsDict, storage_test_server_port: int, monkeypatch_setenv_from_app_config: Callable[[AppConfigDict], EnvVarsDict], @@ -201,15 +204,15 @@ async def _print_mail_to_stdout( @pytest.fixture def web_server( event_loop: asyncio.AbstractEventLoop, - unused_tcp_port_factory: Callable, app_environment: EnvVarsDict, postgres_db: sa.engine.Engine, - web_test_server_port: int, + webserver_test_server_port: int, # tools aiohttp_server: Callable, mocked_send_email: None, disable_static_webserver: Callable, ) -> TestServer: + assert app_environment # original APP app = create_application() @@ -217,7 +220,7 @@ def web_server( disable_static_webserver(app) server = event_loop.run_until_complete( - aiohttp_server(app, port=web_test_server_port) + aiohttp_server(app, port=webserver_test_server_port) ) assert isinstance(postgres_db, sa.engine.Engine) From d9af2e2884582b28982da46d7468e50e78f1ef8f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 09:50:54 +0100 Subject: [PATCH 28/71] fixes imports --- services/web/server/tests/unit/with_dbs/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index b4c4c8752c7e..12d34d5be78c 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -66,7 +66,6 @@ get_default_product_name, get_or_create_product_group, ) -from simcore_service_director.core.settings import get_application_settings from simcore_service_webserver._constants import INDEX_RESOURCE_NAME from simcore_service_webserver.application import create_application from simcore_service_webserver.application_settings_utils import AppConfigDict From e1712c137701b56affe8d702df4b81cedbb0e334 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 11:41:53 +0100 Subject: [PATCH 29/71] @mrnicegyu11 review: wrong env --- .../src/simcore_service_webserver/application_settings.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/application_settings.py b/services/web/server/src/simcore_service_webserver/application_settings.py index f88de60ff097..676725f65b8d 100644 --- a/services/web/server/src/simcore_service_webserver/application_settings.py +++ b/services/web/server/src/simcore_service_webserver/application_settings.py @@ -448,9 +448,7 @@ def to_client_statics(self) -> dict[str, Any]: "SWARM_STACK_NAME": True, "WEBSERVER_PROJECTS": { "PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES", - }, - "WEBSERVER_TRASH": { - "TRASH_RETENTION_DAYS", + "PROJECTS_TRASH_RETENTION_DAYS", }, "WEBSERVER_LOGIN": { "LOGIN_ACCOUNT_DELETION_RETENTION_DAYS", From 79c9e3309c0521646fe9049738e5d22862a610fc Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 13:10:55 +0100 Subject: [PATCH 30/71] @GitHK review: strict argument --- .../unit/with_dbs/04/garbage_collector/test_resource_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py index c55b2e38e136..d4aee7eb58f5 100644 --- a/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/04/garbage_collector/test_resource_manager.py @@ -347,7 +347,7 @@ async def test_websocket_multiple_connections( clients.append(sio) resource_keys.append(resource_key) - for sio, resource_key in zip(clients, resource_keys, strict=False): + for sio, resource_key in zip(clients, resource_keys, strict=True): sid = sio.get_sid() await sio.disconnect() await sio.wait() From f6f3ed5b6ee3a4c59e05de5eee07b73e1024edf3 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 15:16:03 +0100 Subject: [PATCH 31/71] rm unused --- .../projects/_common_models.py | 26 - .../src/simcore_service_webserver/tree.md | 562 ------------------ 2 files changed, 588 deletions(-) delete mode 100644 services/web/server/src/simcore_service_webserver/projects/_common_models.py delete mode 100644 services/web/server/src/simcore_service_webserver/tree.md diff --git a/services/web/server/src/simcore_service_webserver/projects/_common_models.py b/services/web/server/src/simcore_service_webserver/projects/_common_models.py deleted file mode 100644 index bb98b168aea3..000000000000 --- a/services/web/server/src/simcore_service_webserver/projects/_common_models.py +++ /dev/null @@ -1,26 +0,0 @@ -""" Handlers for STANDARD methods on /projects colletions - -Standard methods or CRUD that states for Create+Read(Get&List)+Update+Delete - -""" - -from models_library.projects import ProjectID -from pydantic import BaseModel, ConfigDict, Field - -from ..models import RequestContext - -assert RequestContext.__name__ # nosec - - -class ProjectPathParams(BaseModel): - project_id: ProjectID - model_config = ConfigDict(populate_by_name=True, extra="forbid") - - -class RemoveQueryParams(BaseModel): - force: bool = Field( - default=False, description="Force removal (even if resource is active)" - ) - - -__all__: tuple[str, ...] = ("RequestContext",) diff --git a/services/web/server/src/simcore_service_webserver/tree.md b/services/web/server/src/simcore_service_webserver/tree.md deleted file mode 100644 index 0117a6c851ee..000000000000 --- a/services/web/server/src/simcore_service_webserver/tree.md +++ /dev/null @@ -1,562 +0,0 @@ -This is a tree view of my app. It is built in python's aiohttp. - - -├── activity -│   ├── _api.py -│   ├── _handlers.py -│   ├── plugin.py -│   └── settings.py -├── announcements -│   ├── _api.py -│   ├── _handlers.py -│   ├── _models.py -│   ├── plugin.py -│   └── _redis.py -├── api_keys -│   ├── api.py -│   ├── errors.py -│   ├── _exceptions_handlers.py -│   ├── _models.py -│   ├── plugin.py -│   ├── _repository.py -│   ├── _rest.py -│   ├── _rpc.py -│   └── _service.py -├── application.py -├── application_settings.py -├── application_settings_utils.py -├── catalog -│   ├── _api.py -│   ├── _api_units.py -│   ├── client.py -│   ├── _constants.py -│   ├── exceptions.py -│   ├── _handlers_errors.py -│   ├── _handlers.py -│   ├── licenses -│   │   ├── api.py -│   │   ├── errors.py -│   │   ├── _exceptions_handlers.py -│   │   ├── _licensed_items_api.py -│   │   ├── _licensed_items_db.py -│   │   ├── _licensed_items_handlers.py -│   │   ├── _models.py -│   │   └── plugin.py -│   ├── _models.py -│   ├── plugin.py -│   ├── settings.py -│   └── _tags_handlers.py -├── cli.py -├── _constants.py -├── db -│   ├── _aiopg.py -│   ├── _asyncpg.py -│   ├── base_repository.py -│   ├── models.py -│   ├── plugin.py -│   └── settings.py -├── db_listener -│   ├── _db_comp_tasks_listening_task.py -│   ├── plugin.py -│   └── _utils.py -├── diagnostics -│   ├── _handlers.py -│   ├── _healthcheck.py -│   ├── _monitoring.py -│   ├── plugin.py -│   └── settings.py -├── director_v2 -│   ├── _abc.py -│   ├── api.py -│   ├── _api_utils.py -│   ├── _core_base.py -│   ├── _core_computations.py -│   ├── _core_dynamic_services.py -│   ├── _core_utils.py -│   ├── exceptions.py -│   ├── _handlers.py -│   ├── plugin.py -│   └── settings.py -├── dynamic_scheduler -│   ├── api.py -│   ├── plugin.py -│   └── settings.py -├── email -│   ├── _core.py -│   ├── _handlers.py -│   ├── plugin.py -│   ├── settings.py -│   └── utils.py -├── errors.py -├── exception_handling -│   ├── _base.py -│   └── _factory.py -├── exporter -│   ├── exceptions.py -│   ├── _formatter -│   │   ├── archive.py -│   │   ├── _sds.py -│   │   ├── template_json.py -│   │   └── xlsx -│   │   ├── code_description.py -│   │   ├── core -│   │   │   ├── styling_components.py -│   │   │   └── xlsx_base.py -│   │   ├── dataset_description.py -│   │   ├── manifest.py -│   │   ├── utils.py -│   │   └── writer.py -│   ├── _handlers.py -│   ├── plugin.py -│   ├── settings.py -│   └── utils.py -├── folders -│   ├── api.py -│   ├── errors.py -│   ├── _exceptions_handlers.py -│   ├── _folders_api.py -│   ├── _folders_db.py -│   ├── _folders_handlers.py -│   ├── _models.py -│   ├── plugin.py -│   ├── _trash_api.py -│   ├── _trash_handlers.py -│   ├── _workspaces_api.py -│   └── _workspaces_handlers.py -├── garbage_collector -│   ├── _core_disconnected.py -│   ├── _core_guests.py -│   ├── _core_orphans.py -│   ├── _core.py -│   ├── _core_utils.py -│   ├── plugin.py -│   ├── settings.py -│   ├── _tasks_api_keys.py -│   ├── _tasks_core.py -│   ├── _tasks_trash.py -│   └── _tasks_users.py -├── groups -│   ├── api.py -│   ├── _classifiers_api.py -│   ├── _classifiers_handlers.py -│   ├── _common -│   │   ├── exceptions_handlers.py -│   │   └── schemas.py -│   ├── exceptions.py -│   ├── _groups_api.py -│   ├── _groups_db.py -│   ├── _groups_handlers.py -│   └── plugin.py -├── invitations -│   ├── api.py -│   ├── _client.py -│   ├── _core.py -│   ├── errors.py -│   ├── plugin.py -│   └── settings.py -├── login -│   ├── _2fa_api.py -│   ├── _2fa_handlers.py -│   ├── _auth_api.py -│   ├── _auth_handlers.py -│   ├── cli.py -│   ├── _confirmation.py -│   ├── _constants.py -│   ├── decorators.py -│   ├── errors.py -│   ├── handlers_change.py -│   ├── handlers_confirmation.py -│   ├── handlers_registration.py -│   ├── _models.py -│   ├── plugin.py -│   ├── _registration_api.py -│   ├── _registration_handlers.py -│   ├── _registration.py -│   ├── _security.py -│   ├── settings.py -│   ├── _sql.py -│   ├── storage.py -│   ├── utils_email.py -│   └── utils.py -├── log.py -├── long_running_tasks.py -├── __main__.py -├── meta_modeling -│   ├── _function_nodes.py -│   ├── _handlers.py -│   ├── _iterations.py -│   ├── plugin.py -│   ├── _projects.py -│   ├── _results.py -│   └── _version_control.py -├── _meta.py -├── models.py -├── notifications -│   ├── plugin.py -│   ├── project_logs.py -│   ├── _rabbitmq_consumers_common.py -│   ├── _rabbitmq_exclusive_queue_consumers.py -│   ├── _rabbitmq_nonexclusive_queue_consumers.py -│   └── wallet_osparc_credits.py -├── payments -│   ├── api.py -│   ├── _autorecharge_api.py -│   ├── _autorecharge_db.py -│   ├── errors.py -│   ├── _events.py -│   ├── _methods_api.py -│   ├── _methods_db.py -│   ├── _onetime_api.py -│   ├── _onetime_db.py -│   ├── plugin.py -│   ├── _rpc_invoice.py -│   ├── _rpc.py -│   ├── settings.py -│   ├── _socketio.py -│   └── _tasks.py -├── products -│   ├── _api.py -│   ├── api.py -│   ├── _db.py -│   ├── errors.py -│   ├── _events.py -│   ├── _handlers.py -│   ├── _invitations_handlers.py -│   ├── _middlewares.py -│   ├── _model.py -│   ├── plugin.py -│   └── _rpc.py -├── projects -│   ├── _access_rights_api.py -│   ├── _access_rights_db.py -│   ├── api.py -│   ├── _comments_api.py -│   ├── _comments_db.py -│   ├── _comments_handlers.py -│   ├── _common_models.py -│   ├── _crud_api_create.py -│   ├── _crud_api_delete.py -│   ├── _crud_api_read.py -│   ├── _crud_handlers_models.py -│   ├── _crud_handlers.py -│   ├── db.py -│   ├── _db_utils.py -│   ├── exceptions.py -│   ├── _folders_api.py -│   ├── _folders_db.py -│   ├── _folders_handlers.py -│   ├── _groups_api.py -│   ├── _groups_db.py -│   ├── _groups_handlers.py -│   ├── lock.py -│   ├── _metadata_api.py -│   ├── _metadata_db.py -│   ├── _metadata_handlers.py -│   ├── models.py -│   ├── _nodes_api.py -│   ├── _nodes_handlers.py -│   ├── _nodes_utils.py -│   ├── nodes_utils.py -│   ├── _observer.py -│   ├── _permalink_api.py -│   ├── plugin.py -│   ├── _ports_api.py -│   ├── _ports_handlers.py -│   ├── _projects_access.py -│   ├── projects_api.py -│   ├── _projects_db.py -│   ├── _projects_nodes_pricing_unit_handlers.py -│   ├── settings.py -│   ├── _states_handlers.py -│   ├── _tags_api.py -│   ├── _tags_handlers.py -│   ├── _trash_api.py -│   ├── _trash_handlers.py -│   ├── utils.py -│   ├── _wallets_api.py -│   ├── _wallets_handlers.py -│   ├── _workspaces_api.py -│   └── _workspaces_handlers.py -├── publications -│   ├── _handlers.py -│   └── plugin.py -├── rabbitmq.py -├── rabbitmq_settings.py -├── redis.py -├── resource_manager -│   ├── _constants.py -│   ├── plugin.py -│   ├── registry.py -│   ├── settings.py -│   └── user_sessions.py -├── _resources.py -├── resource_usage -│   ├── api.py -│   ├── _client.py -│   ├── _constants.py -│   ├── errors.py -│   ├── _observer.py -│   ├── plugin.pyf -│   ├── _pricing_plans_admin_api.py -│   ├── _pricing_plans_admin_handlers.py -│   ├── _pricing_plans_api.py -│   ├── _pricing_plans_handlers.py -│   ├── _service_runs_api.py -│   ├── _service_runs_handlers.py -│   ├── settings.py -│   └── _utils.py -├── rest -│   ├── _handlers.py -│   ├── healthcheck.py -│   ├── plugin.py -│   ├── settings.py -│   └── _utils.py -├── scicrunch -│   ├── db.py -│   ├── errors.py -│   ├── models.py -│   ├── plugin.py -│   ├── _resolver.py -│   ├── _rest.py -│   ├── service_client.py -│   └── settings.py -├── security -│   ├── api.py -│   ├── _authz_access_model.py -│   ├── _authz_access_roles.py -│   ├── _authz_db.py -│   ├── _authz_policy.py -│   ├── _constants.py -│   ├── decorators.py -│   ├── _identity_api.py -│   ├── _identity_policy.py -│   └── plugin.py -├── session -│   ├── access_policies.py -│   ├── api.py -│   ├── _cookie_storage.py -│   ├── errors.py -│   ├── plugin.py -│   └── settings.py -├── socketio -│   ├── _handlers.py -│   ├── messages.py -│   ├── models.py -│   ├── _observer.py -│   ├── plugin.py -│   ├── server.py -│   └── _utils.py -├── statics -│   ├── _constants.py -│   ├── _events.py -│   ├── _handlers.py -│   ├── plugin.py -│   └── settings.py -├── storage -│   ├── api.py -│   ├── _handlers.py -│   ├── plugin.py -│   ├── schemas.py -│   └── settings.py -├── studies_dispatcher -│   ├── _catalog.py -│   ├── _constants.py -│   ├── _core.py -│   ├── _errors.py -│   ├── _models.py -│   ├── plugin.py -│   ├── _projects_permalinks.py -│   ├── _projects.py -│   ├── _redirects_handlers.py -│   ├── _rest_handlers.py -│   ├── settings.py -│   ├── _studies_access.py -│   └── _users.py -├── tags -│   ├── _api.py -│   ├── _handlers.py -│   ├── plugin.py -│   └── schemas.py -├── tracing.py -├── users -│   ├── _api.py -│   ├── api.py -│   ├── _constants.py -│   ├── _db.py -│   ├── exceptions.py -│   ├── _handlers.py -│   ├── _models.py -│   ├── _notifications_handlers.py -│   ├── _notifications.py -│   ├── plugin.py -│   ├── _preferences_api.py -│   ├── preferences_api.py -│   ├── _preferences_db.py -│   ├── _preferences_handlers.py -│   ├── _preferences_models.py -│   ├── _schemas.py -│   ├── schemas.py -│   ├── settings.py -│   ├── _tokens_handlers.py -│   └── _tokens.py -├── utils_aiohttp.py -├── utils.py -├── utils_rate_limiting.py -├── version_control -│   ├── _core.py -│   ├── db.py -│   ├── errors.py -│   ├── _handlers_base.py -│   ├── _handlers.py -│   ├── models.py -│   ├── plugin.py -│   ├── vc_changes.py -│   └── vc_tags.py -├── wallets -│   ├── _api.py -│   ├── api.py -│   ├── _constants.py -│   ├── _db.py -│   ├── errors.py -│   ├── _events.py -│   ├── _groups_api.py -│   ├── _groups_db.py -│   ├── _groups_handlers.py -│   ├── _handlers.py -│   ├── _payments_handlers.py -│   └── plugin.py -└── workspaces - ├── api.py - ├── errors.py - ├── _exceptions_handlers.py - ├── _groups_api.py - ├── _groups_db.py - ├── _groups_handlers.py - ├── _models.py - ├── plugin.py - ├── _trash_api.py - ├── _trash_handlers.py - ├── _workspaces_api.py - ├── _workspaces_db.py - └── _workspaces_handlers.py - - - - - -The top folders represent plugins that could be interprested as different domains with small compling between each other - -Here are some conventions - -- `plugin` has a setup function to setup the app (e.g. add routes, setup events etc ). Classic `setup_xxx(app)` for aiohttp -- `settings` includes pydantic settings classes specific to the domain -- `exceptions` or `errors` include only exceptions classes - - `_exceptions_handlers` are utils to handle exceptions -- `models` correspond to domain models, i.e. not part of any of the controllers interfaces. Those are denoted `scheme`. - -Then - -- `_handlers` (or _rest) represent the rest handlers (i.e. controller layer) -- `_rpc` contains handlers (server side) to an RPC interface (i.e. controller layer) -- `_api` (or `_service`) represent the business logic of this domain (i.e. service layer) - - the shared service layer for inter-domain logic is called `api` (i.e. without `_`) -- `_db` (or `_repository`) represents the repository layer - - -Could you please apply the rules on this structure and come up with a new tree that follows: -- keeps the domain-drive modular organization -- every domain implements controller-service-repository (CSR) -- highligh the shared service layer for inter-domain logic - - -As an output just recreate the tree adding some comments on it (e.g. with #) but no further explanatio is needed - - - -```plaintext -├── activity -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for activity domain (service layer) -│   ├── repositories -│   │   ├── activity_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for activity domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── announcements -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for announcements domain (service layer) -│   ├── repositories -│   │   ├── announcements_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for announcements domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── api_keys -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for api_keys domain (service layer) -│   ├── repositories -│   │   ├── api_keys_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for api_keys domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── shared_services -│   ├── inter_domain_service.py # Shared service layer for inter-domain logic -├── db -│   ├── connection.py # Database connection logic -│   ├── base_repository.py # Base repository logic -│   ├── plugin.py # Setup function for database -│   ├── settings.py # Database-specific settings -├── catalog -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for catalog domain (service layer) -│   ├── repositories -│   │   ├── catalog_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for catalog domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── users -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for users domain (service layer) -│   ├── repositories -│   │   ├── users_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for users domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── projects -│   ├── controllers -│   │   ├── rest_handlers.py # Rest API handlers (controller layer) -│   │   ├── rpc_handlers.py # RPC handlers (controller layer) -│   ├── services -│   │   ├── domain_service.py # Business logic for projects domain (service layer) -│   ├── repositories -│   │   ├── projects_repository.py # Data access logic (repository layer) -│   ├── plugin.py # Setup function for projects domain -│   ├── settings.py # Domain-specific settings -│   ├── exceptions.py # Domain-specific exceptions -├── shared -│   ├── models -│   │   ├── user.py # Shared user model -│   │   ├── project.py # Shared project model -│   ├── schemas -│   │   ├── user_schema.py # Shared user schemas -│   │   ├── project_schema.py # Shared project schemas -│   ├── utils -│   │   ├── logger.py # Shared logging logic -│   │   ├── validators.py # Shared validation logic -├── application.py # Main application initialization -└── cli.py # Command-line interface logic -``` From 22445844e4259f1a29d8a0e6ff6270b7e1839704 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:25:12 +0100 Subject: [PATCH 32/71] rename layers in projects --- api/specs/web-server/_trash.py | 2 +- .../src/simcore_service_webserver/folders/_trash_api.py | 2 +- .../projects/{_trash_handlers.py => _trash_rest.py} | 6 +++--- .../projects/{_trash_api.py => _trash_service.py} | 0 .../server/src/simcore_service_webserver/projects/plugin.py | 4 ++-- .../src/simcore_service_webserver/workspaces/_trash_api.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) rename services/web/server/src/simcore_service_webserver/projects/{_trash_handlers.py => _trash_rest.py} (96%) rename services/web/server/src/simcore_service_webserver/projects/{_trash_api.py => _trash_service.py} (100%) diff --git a/api/specs/web-server/_trash.py b/api/specs/web-server/_trash.py index 6eb39f6593cf..7a5c075bcb49 100644 --- a/api/specs/web-server/_trash.py +++ b/api/specs/web-server/_trash.py @@ -14,7 +14,7 @@ FoldersPathParams, FolderTrashQueryParams, ) -from simcore_service_webserver.projects._trash_handlers import ProjectPathParams +from simcore_service_webserver.projects._trash_rest import ProjectPathParams from simcore_service_webserver.workspaces._models import ( WorkspacesPathParams, WorkspaceTrashQueryParams, diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_api.py b/services/web/server/src/simcore_service_webserver/folders/_trash_api.py index b3e1823369a2..bbc683cd14fd 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_trash_api.py +++ b/services/web/server/src/simcore_service_webserver/folders/_trash_api.py @@ -11,7 +11,7 @@ from sqlalchemy.ext.asyncio import AsyncConnection from ..db.plugin import get_asyncpg_engine -from ..projects._trash_api import trash_project, untrash_project +from ..projects._trash_service import trash_project, untrash_project from ..workspaces.api import check_user_workspace_access from . import _folders_db diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_trash_rest.py similarity index 96% rename from services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py rename to services/web/server/src/simcore_service_webserver/projects/_trash_rest.py index e555807e2d49..22368285efc6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_rest.py @@ -17,7 +17,7 @@ from ..login.decorators import get_user_id, login_required from ..products.api import get_product_name from ..security.decorators import permission_required -from . import _trash_api +from . import _trash_service from ._common.models import ProjectPathParams, RemoveQueryParams from .exceptions import ProjectRunningConflictError, ProjectStoppingError @@ -64,7 +64,7 @@ async def trash_project(request: web.Request): RemoveQueryParams, request ) - await _trash_api.trash_project( + await _trash_service.trash_project( request.app, product_name=product_name, user_id=user_id, @@ -85,7 +85,7 @@ async def untrash_project(request: web.Request): product_name = get_product_name(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) - await _trash_api.untrash_project( + await _trash_service.untrash_project( request.app, product_name=product_name, user_id=user_id, diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_api.py b/services/web/server/src/simcore_service_webserver/projects/_trash_service.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/projects/_trash_api.py rename to services/web/server/src/simcore_service_webserver/projects/_trash_service.py diff --git a/services/web/server/src/simcore_service_webserver/projects/plugin.py b/services/web/server/src/simcore_service_webserver/projects/plugin.py index b72c4a90b9ac..5cba65b8a2b6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/plugin.py +++ b/services/web/server/src/simcore_service_webserver/projects/plugin.py @@ -20,7 +20,7 @@ _projects_nodes_pricing_unit_handlers, _states_handlers, _tags_handlers, - _trash_handlers, + _trash_rest, _wallets_handlers, _workspaces_handlers, ) @@ -62,6 +62,6 @@ def setup_projects(app: web.Application) -> bool: app.router.add_routes(_folders_handlers.routes) app.router.add_routes(_projects_nodes_pricing_unit_handlers.routes) app.router.add_routes(_workspaces_handlers.routes) - app.router.add_routes(_trash_handlers.routes) + app.router.add_routes(_trash_rest.routes) return True diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py index 18c3ae93b88a..80807f53caa6 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py @@ -11,7 +11,7 @@ from ..db.plugin import get_asyncpg_engine from ..folders._trash_api import trash_folder, untrash_folder -from ..projects._trash_api import trash_project, untrash_project +from ..projects._trash_service import trash_project, untrash_project from ._workspaces_api import check_user_workspace_access from ._workspaces_db import update_workspace From e79213ac0df3c39cedf5621814d52c498869dff0 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:26:36 +0100 Subject: [PATCH 33/71] rename layers in folders --- .../folders/{_trash_handlers.py => _trash_rest.py} | 6 +++--- .../folders/{_trash_api.py => _trash_service.py} | 0 .../server/src/simcore_service_webserver/folders/plugin.py | 4 ++-- .../src/simcore_service_webserver/workspaces/_trash_api.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename services/web/server/src/simcore_service_webserver/folders/{_trash_handlers.py => _trash_rest.py} (94%) rename services/web/server/src/simcore_service_webserver/folders/{_trash_api.py => _trash_service.py} (100%) diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py similarity index 94% rename from services/web/server/src/simcore_service_webserver/folders/_trash_handlers.py rename to services/web/server/src/simcore_service_webserver/folders/_trash_rest.py index c702d8a70820..df711ce3c35e 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_trash_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py @@ -11,7 +11,7 @@ from ..login.decorators import get_user_id, login_required from ..products.api import get_product_name from ..security.decorators import permission_required -from . import _trash_api +from . import _trash_service from ._exceptions_handlers import handle_plugin_requests_exceptions from ._models import FoldersPathParams, FolderTrashQueryParams @@ -33,7 +33,7 @@ async def trash_folder(request: web.Request): FolderTrashQueryParams, request ) - await _trash_api.trash_folder( + await _trash_service.trash_folder( request.app, product_name=product_name, user_id=user_id, @@ -53,7 +53,7 @@ async def untrash_folder(request: web.Request): product_name = get_product_name(request) path_params = parse_request_path_parameters_as(FoldersPathParams, request) - await _trash_api.untrash_folder( + await _trash_service.untrash_folder( request.app, product_name=product_name, user_id=user_id, diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_api.py b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/folders/_trash_api.py rename to services/web/server/src/simcore_service_webserver/folders/_trash_service.py diff --git a/services/web/server/src/simcore_service_webserver/folders/plugin.py b/services/web/server/src/simcore_service_webserver/folders/plugin.py index 2601962e52f4..dee48921c18d 100644 --- a/services/web/server/src/simcore_service_webserver/folders/plugin.py +++ b/services/web/server/src/simcore_service_webserver/folders/plugin.py @@ -7,7 +7,7 @@ from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _folders_handlers, _trash_handlers, _workspaces_handlers +from . import _folders_handlers, _trash_rest, _workspaces_handlers _logger = logging.getLogger(__name__) @@ -24,5 +24,5 @@ def setup_folders(app: web.Application): # routes app.router.add_routes(_folders_handlers.routes) - app.router.add_routes(_trash_handlers.routes) + app.router.add_routes(_trash_rest.routes) app.router.add_routes(_workspaces_handlers.routes) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py index 80807f53caa6..817b2c41983d 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py @@ -10,7 +10,7 @@ from simcore_postgres_database.utils_repos import transaction_context from ..db.plugin import get_asyncpg_engine -from ..folders._trash_api import trash_folder, untrash_folder +from ..folders._trash_service import trash_folder, untrash_folder from ..projects._trash_service import trash_project, untrash_project from ._workspaces_api import check_user_workspace_access from ._workspaces_db import update_workspace From 26dcf1f106a0cf3175c541c4ae78f62bdba14b8b Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:27:21 +0100 Subject: [PATCH 34/71] rename layers in workspaces --- .../workspaces/{_trash_handlers.py => _trash_rest.py} | 6 +++--- .../workspaces/{_trash_api.py => _trash_services.py} | 0 .../src/simcore_service_webserver/workspaces/plugin.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename services/web/server/src/simcore_service_webserver/workspaces/{_trash_handlers.py => _trash_rest.py} (94%) rename services/web/server/src/simcore_service_webserver/workspaces/{_trash_api.py => _trash_services.py} (100%) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_handlers.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py similarity index 94% rename from services/web/server/src/simcore_service_webserver/workspaces/_trash_handlers.py rename to services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py index eafabb8d0774..99f4e27ddc18 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_handlers.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py @@ -11,7 +11,7 @@ from ..login.decorators import get_user_id, login_required from ..products.api import get_product_name from ..security.decorators import permission_required -from . import _trash_api +from . import _trash_services from ._exceptions_handlers import handle_plugin_requests_exceptions from ._models import WorkspacesPathParams, WorkspaceTrashQueryParams @@ -33,7 +33,7 @@ async def trash_workspace(request: web.Request): WorkspaceTrashQueryParams, request ) - await _trash_api.trash_workspace( + await _trash_services.trash_workspace( request.app, product_name=product_name, user_id=user_id, @@ -53,7 +53,7 @@ async def untrash_workspace(request: web.Request): product_name = get_product_name(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) - await _trash_api.untrash_workspace( + await _trash_services.untrash_workspace( request.app, product_name=product_name, user_id=user_id, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/workspaces/_trash_api.py rename to services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py diff --git a/services/web/server/src/simcore_service_webserver/workspaces/plugin.py b/services/web/server/src/simcore_service_webserver/workspaces/plugin.py index d67a9167c927..65f1866441f4 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/plugin.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/plugin.py @@ -7,7 +7,7 @@ from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _groups_handlers, _trash_handlers, _workspaces_handlers +from . import _groups_handlers, _trash_rest, _workspaces_handlers _logger = logging.getLogger(__name__) @@ -25,4 +25,4 @@ def setup_workspaces(app: web.Application): # routes app.router.add_routes(_workspaces_handlers.routes) app.router.add_routes(_groups_handlers.routes) - app.router.add_routes(_trash_handlers.routes) + app.router.add_routes(_trash_rest.routes) From 7b5a55fff0837d5dbcd51e3df820d2086f7f8f36 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:38:03 +0100 Subject: [PATCH 35/71] unused --- .../api_schemas_webserver/folders.py | 48 ------------------- 1 file changed, 48 deletions(-) delete mode 100644 packages/models-library/src/models_library/api_schemas_webserver/folders.py diff --git a/packages/models-library/src/models_library/api_schemas_webserver/folders.py b/packages/models-library/src/models_library/api_schemas_webserver/folders.py deleted file mode 100644 index dd4647185710..000000000000 --- a/packages/models-library/src/models_library/api_schemas_webserver/folders.py +++ /dev/null @@ -1,48 +0,0 @@ -from datetime import datetime -from typing import NamedTuple - -from models_library.basic_types import IDStr -from models_library.folders import FolderID -from models_library.groups import GroupID -from models_library.projects_access import AccessRights -from models_library.utils.common_validators import null_or_none_str_to_none_validator -from pydantic import ConfigDict, PositiveInt, field_validator - -from ._base import InputSchema, OutputSchema - - -class FolderGet(OutputSchema): - folder_id: FolderID - parent_folder_id: FolderID | None = None - name: str - description: str - created_at: datetime - modified_at: datetime - trashed_at: datetime | None - owner: GroupID - my_access_rights: AccessRights - access_rights: dict[GroupID, AccessRights] - - -class FolderGetPage(NamedTuple): - items: list[FolderGet] - total: PositiveInt - - -class CreateFolderBodyParams(InputSchema): - name: IDStr - description: str - parent_folder_id: FolderID | None = None - - model_config = ConfigDict(extra="forbid") - - _null_or_none_str_to_none_validator = field_validator( - "parent_folder_id", mode="before" - )(null_or_none_str_to_none_validator) - - -class PutFolderBodyParams(InputSchema): - name: IDStr - description: str - - model_config = ConfigDict(extra="forbid") From b2a1e1e68f1dc4797075705a0da2c1e3d0554323 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:43:04 +0100 Subject: [PATCH 36/71] doc --- .../web/server/src/simcore_service_webserver/trash/_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/web/server/src/simcore_service_webserver/trash/_service.py b/services/web/server/src/simcore_service_webserver/trash/_service.py index 874534944962..cc94d680d644 100644 --- a/services/web/server/src/simcore_service_webserver/trash/_service.py +++ b/services/web/server/src/simcore_service_webserver/trash/_service.py @@ -23,6 +23,7 @@ async def empty_trash(app: web.Application, product_name: ProductName, user_id: async def prune_trash(app: web.Application) -> list[str]: + """Deletes expired items in the trash""" settings = get_plugin_settings(app) retention = timedelta(days=settings.TRASH_RETENTION_DAYS) From 4500ed51449b87af62e79fbb69ad9c0ef58793cd Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:50:49 +0100 Subject: [PATCH 37/71] refactor folders subdomain --- api/specs/web-server/_folders.py | 8 ++++---- api/specs/web-server/_trash.py | 2 +- api/specs/web-server/_workspaces.py | 8 +++++--- .../folders/_common/__init__.py | 0 .../exceptions_handlers.py} | 8 ++++---- .../folders/{_models.py => _common/models.py} | 2 +- .../{_folders_db.py => _folders_repository.py} | 0 .../{_folders_handlers.py => _folders_rest.py} | 18 +++++++++--------- .../{_folders_api.py => _folders_service.py} | 4 ++-- .../folders/_trash_rest.py | 4 ++-- .../folders/_trash_service.py | 16 ++++++++-------- ...spaces_api.py => _workspaces_repository.py} | 10 +++++----- ...kspaces_handlers.py => _workspaces_rest.py} | 8 ++++---- .../simcore_service_webserver/folders/api.py | 2 -- .../folders/plugin.py | 6 +++--- .../projects/_crud_api_create.py | 2 +- .../projects/_crud_api_read.py | 2 +- .../projects/_folders_api.py | 2 +- ..._handlers__clone_in_workspace_and_folder.py | 2 +- 19 files changed, 52 insertions(+), 52 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/folders/_common/__init__.py rename services/web/server/src/simcore_service_webserver/folders/{_exceptions_handlers.py => _common/exceptions_handlers.py} (94%) rename services/web/server/src/simcore_service_webserver/folders/{_models.py => _common/models.py} (98%) rename services/web/server/src/simcore_service_webserver/folders/{_folders_db.py => _folders_repository.py} (100%) rename services/web/server/src/simcore_service_webserver/folders/{_folders_handlers.py => _folders_rest.py} (92%) rename services/web/server/src/simcore_service_webserver/folders/{_folders_api.py => _folders_service.py} (96%) rename services/web/server/src/simcore_service_webserver/folders/{_workspaces_api.py => _workspaces_repository.py} (95%) rename services/web/server/src/simcore_service_webserver/folders/{_workspaces_handlers.py => _workspaces_rest.py} (80%) delete mode 100644 services/web/server/src/simcore_service_webserver/folders/api.py diff --git a/api/specs/web-server/_folders.py b/api/specs/web-server/_folders.py index 2aa77e485d4f..226a71dd92c6 100644 --- a/api/specs/web-server/_folders.py +++ b/api/specs/web-server/_folders.py @@ -19,13 +19,13 @@ from models_library.generics import Envelope from models_library.rest_error import EnvelopedError from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.folders._exceptions_handlers import _TO_HTTP_ERROR_MAP -from simcore_service_webserver.folders._models import ( +from simcore_service_webserver.folders._common.exceptions_handlers import ( + _TO_HTTP_ERROR_MAP, +) +from simcore_service_webserver.folders._common.models import ( FolderSearchQueryParams, FoldersListQueryParams, FoldersPathParams, -) -from simcore_service_webserver.folders._workspaces_handlers import ( _FolderWorkspacesPathParams, ) diff --git a/api/specs/web-server/_trash.py b/api/specs/web-server/_trash.py index 7a5c075bcb49..1235d8dcdd24 100644 --- a/api/specs/web-server/_trash.py +++ b/api/specs/web-server/_trash.py @@ -10,7 +10,7 @@ from fastapi import APIRouter, Depends, status from models_library.trash import RemoveQueryParams from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.folders._models import ( +from simcore_service_webserver.folders._common.models import ( FoldersPathParams, FolderTrashQueryParams, ) diff --git a/api/specs/web-server/_workspaces.py b/api/specs/web-server/_workspaces.py index fce290fffb0c..56350f594665 100644 --- a/api/specs/web-server/_workspaces.py +++ b/api/specs/web-server/_workspaces.py @@ -19,14 +19,16 @@ from models_library.generics import Envelope from models_library.rest_error import EnvelopedError from simcore_service_webserver._meta import API_VTAG -from simcore_service_webserver.folders._exceptions_handlers import _TO_HTTP_ERROR_MAP -from simcore_service_webserver.workspaces._groups_api import WorkspaceGroupGet -from simcore_service_webserver.workspaces._models import ( +from simcore_service_webserver.folders._common.exceptions_handlers import ( + _TO_HTTP_ERROR_MAP, +) +from simcore_service_webserver.workspaces._common.models import ( WorkspacesGroupsBodyParams, WorkspacesGroupsPathParams, WorkspacesListQueryParams, WorkspacesPathParams, ) +from simcore_service_webserver.workspaces._groups_api import WorkspaceGroupGet router = APIRouter( prefix=f"/{API_VTAG}", diff --git a/services/web/server/src/simcore_service_webserver/folders/_common/__init__.py b/services/web/server/src/simcore_service_webserver/folders/_common/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/web/server/src/simcore_service_webserver/folders/_exceptions_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_common/exceptions_handlers.py similarity index 94% rename from services/web/server/src/simcore_service_webserver/folders/_exceptions_handlers.py rename to services/web/server/src/simcore_service_webserver/folders/_common/exceptions_handlers.py index 8b571562c8d8..d117b870c970 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_exceptions_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_common/exceptions_handlers.py @@ -2,23 +2,23 @@ from servicelib.aiohttp import status -from ..exception_handling import ( +from ...exception_handling import ( ExceptionToHttpErrorMap, HttpErrorInfo, exception_handling_decorator, to_exceptions_handlers_map, ) -from ..projects.exceptions import ( +from ...projects.exceptions import ( ProjectInvalidRightsError, ProjectRunningConflictError, ProjectStoppingError, ) -from ..workspaces.errors import ( +from ...workspaces.errors import ( WorkspaceAccessForbiddenError, WorkspaceFolderInconsistencyError, WorkspaceNotFoundError, ) -from .errors import ( +from ..errors import ( FolderAccessForbiddenError, FolderNotFoundError, FoldersValueError, diff --git a/services/web/server/src/simcore_service_webserver/folders/_models.py b/services/web/server/src/simcore_service_webserver/folders/_common/models.py similarity index 98% rename from services/web/server/src/simcore_service_webserver/folders/_models.py rename to services/web/server/src/simcore_service_webserver/folders/_common/models.py index 553d43bd64c2..37e03e42636c 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_models.py +++ b/services/web/server/src/simcore_service_webserver/folders/_common/models.py @@ -20,7 +20,7 @@ from models_library.workspaces import WorkspaceID from pydantic import BaseModel, BeforeValidator, ConfigDict, Field -from .._constants import RQ_PRODUCT_KEY, RQT_USERID_KEY +from ..._constants import RQ_PRODUCT_KEY, RQT_USERID_KEY _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_db.py b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/folders/_folders_db.py rename to services/web/server/src/simcore_service_webserver/folders/_folders_repository.py diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py similarity index 92% rename from services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py rename to services/web/server/src/simcore_service_webserver/folders/_folders_rest.py index e1dea38ecae6..b77829867a10 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py @@ -23,9 +23,9 @@ from ..login.decorators import login_required from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _folders_api -from ._exceptions_handlers import handle_plugin_requests_exceptions -from ._models import ( +from . import _folders_service +from ._common.exceptions_handlers import handle_plugin_requests_exceptions +from ._common.models import ( FolderFilters, FolderSearchQueryParams, FoldersListQueryParams, @@ -47,7 +47,7 @@ async def create_folder(request: web.Request): req_ctx = FoldersRequestContext.model_validate(request) body_params = await parse_request_body_as(FolderCreateBodyParams, request) - folder = await _folders_api.create_folder( + folder = await _folders_service.create_folder( request.app, user_id=req_ctx.user_id, name=body_params.name, @@ -72,7 +72,7 @@ async def list_folders(request: web.Request): if not query_params.filters: query_params.filters = FolderFilters() - folders: FolderGetPage = await _folders_api.list_folders( + folders: FolderGetPage = await _folders_service.list_folders( app=request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, @@ -112,7 +112,7 @@ async def list_folders_full_search(request: web.Request): if not query_params.filters: query_params.filters = FolderFilters() - folders: FolderGetPage = await _folders_api.list_folders_full_depth( + folders: FolderGetPage = await _folders_service.list_folders_full_depth( app=request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, @@ -146,7 +146,7 @@ async def get_folder(request: web.Request): req_ctx = FoldersRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(FoldersPathParams, request) - folder: FolderGet = await _folders_api.get_folder( + folder: FolderGet = await _folders_service.get_folder( app=request.app, folder_id=path_params.folder_id, user_id=req_ctx.user_id, @@ -168,7 +168,7 @@ async def replace_folder(request: web.Request): path_params = parse_request_path_parameters_as(FoldersPathParams, request) body_params = await parse_request_body_as(FolderReplaceBodyParams, request) - folder = await _folders_api.update_folder( + folder = await _folders_service.update_folder( app=request.app, user_id=req_ctx.user_id, folder_id=path_params.folder_id, @@ -190,7 +190,7 @@ async def delete_folder_group(request: web.Request): req_ctx = FoldersRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(FoldersPathParams, request) - await _folders_api.delete_folder( + await _folders_service.delete_folder( app=request.app, user_id=req_ctx.user_id, folder_id=path_params.folder_id, diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_api.py b/services/web/server/src/simcore_service_webserver/folders/_folders_service.py similarity index 96% rename from services/web/server/src/simcore_service_webserver/folders/_folders_api.py rename to services/web/server/src/simcore_service_webserver/folders/_folders_service.py index 6cd65316b057..52d37412e32b 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_api.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_service.py @@ -16,7 +16,6 @@ from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from servicelib.utils import fire_and_forget_task -from ..folders.errors import FolderValueNotPermittedError from ..projects.projects_api import submit_delete_project_task from ..users.api import get_user from ..workspaces.api import check_user_workspace_access @@ -24,7 +23,8 @@ WorkspaceAccessForbiddenError, WorkspaceFolderInconsistencyError, ) -from . import _folders_db as folders_db +from . import _folders_repository as folders_db +from .errors import FolderValueNotPermittedError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py b/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py index df711ce3c35e..0e035012adb1 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py +++ b/services/web/server/src/simcore_service_webserver/folders/_trash_rest.py @@ -12,8 +12,8 @@ from ..products.api import get_product_name from ..security.decorators import permission_required from . import _trash_service -from ._exceptions_handlers import handle_plugin_requests_exceptions -from ._models import FoldersPathParams, FolderTrashQueryParams +from ._common.exceptions_handlers import handle_plugin_requests_exceptions +from ._common.models import FoldersPathParams, FolderTrashQueryParams _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py index bbc683cd14fd..468d0a7d4a3a 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py +++ b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py @@ -13,7 +13,7 @@ from ..db.plugin import get_asyncpg_engine from ..projects._trash_service import trash_project, untrash_project from ..workspaces.api import check_user_workspace_access -from . import _folders_db +from . import _folders_repository _logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ async def _check_exists_and_access( # exists? # check whether this folder exists # otherwise raise not-found error - folder_db = await _folders_db.get( + folder_db = await _folders_repository.get( app, folder_id=folder_id, product_name=product_name ) @@ -46,7 +46,7 @@ async def _check_exists_and_access( ) workspace_is_private = False - await _folders_db.get_for_user_or_workspace( + await _folders_repository.get_for_user_or_workspace( app, folder_id=folder_id, product_name=product_name, @@ -65,7 +65,7 @@ async def _folders_db_update( trashed_at: datetime | None, ): # EXPLICIT un/trash - await _folders_db.update( + await _folders_repository.update( app, connection, folders_id_or_ids=folder_id, @@ -77,14 +77,14 @@ async def _folders_db_update( # IMPLICIT un/trash child_folders: set[FolderID] = { f - for f in await _folders_db.get_folders_recursively( + for f in await _folders_repository.get_folders_recursively( app, connection, folder_id=folder_id, product_name=product_name ) if f != folder_id } if child_folders: - await _folders_db.update( + await _folders_repository.update( app, connection, folders_id_or_ids=child_folders, @@ -124,7 +124,7 @@ async def trash_folder( # 2. Trash all child projects that I am an owner child_projects: list[ ProjectID - ] = await _folders_db.get_projects_recursively_only_if_user_is_owner( + ] = await _folders_repository.get_projects_recursively_only_if_user_is_owner( app, connection, folder_id=folder_id, @@ -169,7 +169,7 @@ async def untrash_folder( # 3.2 UNtrash all child projects that I am an owner child_projects: list[ ProjectID - ] = await _folders_db.get_projects_recursively_only_if_user_is_owner( + ] = await _folders_repository.get_projects_recursively_only_if_user_is_owner( app, folder_id=folder_id, private_workspace_user_id_or_none=user_id if workspace_is_private else None, diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_repository.py similarity index 95% rename from services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py rename to services/web/server/src/simcore_service_webserver/folders/_workspaces_repository.py index 115ff2c8d8eb..bac38edb7ca4 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_repository.py @@ -14,7 +14,7 @@ from ..projects._access_rights_api import check_user_project_permission from ..users.api import get_user from ..workspaces.api import check_user_workspace_access -from . import _folders_db +from . import _folders_repository _logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ async def move_folder_into_workspace( product_name: ProductName, ) -> None: # 1. User needs to have delete permission on source folder - folder_db = await _folders_db.get( + folder_db = await _folders_repository.get( app, folder_id=folder_id, product_name=product_name ) workspace_is_private = True @@ -56,7 +56,7 @@ async def move_folder_into_workspace( ( folder_ids, project_ids, - ) = await _folders_db.get_all_folders_and_projects_ids_recursively( + ) = await _folders_repository.get_all_folders_and_projects_ids_recursively( app, connection=None, folder_id=folder_id, @@ -86,7 +86,7 @@ async def move_folder_into_workspace( ) # 5. BATCH update of folders with workspace_id - await _folders_db.update( + await _folders_repository.update( app, connection=conn, folders_id_or_ids=set(folder_ids), @@ -96,7 +96,7 @@ async def move_folder_into_workspace( ) # 6. Update source folder parent folder ID with NULL (it will appear in the root directory) - await _folders_db.update( + await _folders_repository.update( app, connection=conn, folders_id_or_ids=folder_id, diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_rest.py similarity index 80% rename from services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py rename to services/web/server/src/simcore_service_webserver/folders/_workspaces_rest.py index faa505ecd316..2f6b8b1cda24 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_rest.py @@ -7,9 +7,9 @@ from .._meta import api_version_prefix as VTAG from ..login.decorators import login_required from ..security.decorators import permission_required -from . import _workspaces_api -from ._exceptions_handlers import handle_plugin_requests_exceptions -from ._models import FoldersRequestContext, _FolderWorkspacesPathParams +from . import _workspaces_repository +from ._common.exceptions_handlers import handle_plugin_requests_exceptions +from ._common.models import FoldersRequestContext, _FolderWorkspacesPathParams _logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ async def move_folder_to_workspace(request: web.Request): req_ctx = FoldersRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(_FolderWorkspacesPathParams, request) - await _workspaces_api.move_folder_into_workspace( + await _workspaces_repository.move_folder_into_workspace( app=request.app, user_id=req_ctx.user_id, folder_id=path_params.folder_id, diff --git a/services/web/server/src/simcore_service_webserver/folders/api.py b/services/web/server/src/simcore_service_webserver/folders/api.py deleted file mode 100644 index 2c5be8fdf520..000000000000 --- a/services/web/server/src/simcore_service_webserver/folders/api.py +++ /dev/null @@ -1,2 +0,0 @@ -__all__: tuple[str, ...] = () -# nopycln: file diff --git a/services/web/server/src/simcore_service_webserver/folders/plugin.py b/services/web/server/src/simcore_service_webserver/folders/plugin.py index dee48921c18d..ec1e3f80ffe4 100644 --- a/services/web/server/src/simcore_service_webserver/folders/plugin.py +++ b/services/web/server/src/simcore_service_webserver/folders/plugin.py @@ -7,7 +7,7 @@ from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _folders_handlers, _trash_rest, _workspaces_handlers +from . import _folders_rest, _trash_rest, _workspaces_rest _logger = logging.getLogger(__name__) @@ -23,6 +23,6 @@ def setup_folders(app: web.Application): assert app[APP_SETTINGS_KEY].WEBSERVER_FOLDERS # nosec # routes - app.router.add_routes(_folders_handlers.routes) + app.router.add_routes(_folders_rest.routes) app.router.add_routes(_trash_rest.routes) - app.router.add_routes(_workspaces_handlers.routes) + app.router.add_routes(_workspaces_rest.routes) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 9953914f5d04..3d276c9f7184 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -28,7 +28,7 @@ from ..catalog import client as catalog_client from ..director_v2 import api as director_v2_api from ..dynamic_scheduler import api as dynamic_scheduler_api -from ..folders import _folders_db as folders_db +from ..folders import _folders_repository as folders_db from ..storage.api import ( copy_data_folders_from_project, get_project_total_size_simcore_s3, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index 55a9b7c6429b..77cdd5cf7d49 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -19,7 +19,7 @@ from simcore_postgres_database.webserver_models import ProjectType as ProjectTypeDB from ..catalog.client import get_services_for_user_in_product -from ..folders import _folders_db as folders_db +from ..folders import _folders_repository as folders_db from ..workspaces._workspaces_api import check_user_workspace_access from . import projects_api from ._permalink_api import update_or_pop_permalink_in_project diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_api.py b/services/web/server/src/simcore_service_webserver/projects/_folders_api.py index 4b465edf0e9d..865339a91658 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_api.py @@ -6,7 +6,7 @@ from models_library.projects import ProjectID from models_library.users import UserID -from ..folders import _folders_db as folders_db +from ..folders import _folders_repository as folders_db from ..projects._access_rights_api import get_user_project_access_rights from . import _folders_db as project_to_folders_db from .db import APP_PROJECT_DBAPI, ProjectDBAPI diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py index 8ba05ea870c1..6fef8ab7870a 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py @@ -20,7 +20,7 @@ from simcore_postgres_database.models.folders_v2 import folders_v2 from simcore_postgres_database.models.workspaces import workspaces from simcore_service_webserver.db.models import UserRole -from simcore_service_webserver.folders._folders_api import create_folder +from simcore_service_webserver.folders._folders_service import create_folder from simcore_service_webserver.projects._folders_api import move_project_into_folder from simcore_service_webserver.projects.models import ProjectDict from simcore_service_webserver.workspaces._workspaces_api import create_workspace From 89e33bfb63d7b266ac572cecd326fd836d8bb95a Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:55:51 +0100 Subject: [PATCH 38/71] common --- api/specs/web-server/_trash.py | 2 +- .../workspaces/_common/__init__.py | 0 .../workspaces/{ => _common}/_exceptions_handlers.py | 6 +++--- .../workspaces/{ => _common}/_models.py | 2 +- .../workspaces/_groups_handlers.py | 6 +++--- .../src/simcore_service_webserver/workspaces/_trash_rest.py | 4 ++-- .../workspaces/_workspaces_handlers.py | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) create mode 100644 services/web/server/src/simcore_service_webserver/workspaces/_common/__init__.py rename services/web/server/src/simcore_service_webserver/workspaces/{ => _common}/_exceptions_handlers.py (90%) rename services/web/server/src/simcore_service_webserver/workspaces/{ => _common}/_models.py (98%) diff --git a/api/specs/web-server/_trash.py b/api/specs/web-server/_trash.py index 1235d8dcdd24..dd6774b7944a 100644 --- a/api/specs/web-server/_trash.py +++ b/api/specs/web-server/_trash.py @@ -15,7 +15,7 @@ FolderTrashQueryParams, ) from simcore_service_webserver.projects._trash_rest import ProjectPathParams -from simcore_service_webserver.workspaces._models import ( +from simcore_service_webserver.workspaces._common._models import ( WorkspacesPathParams, WorkspaceTrashQueryParams, ) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_common/__init__.py b/services/web/server/src/simcore_service_webserver/workspaces/_common/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_exceptions_handlers.py b/services/web/server/src/simcore_service_webserver/workspaces/_common/_exceptions_handlers.py similarity index 90% rename from services/web/server/src/simcore_service_webserver/workspaces/_exceptions_handlers.py rename to services/web/server/src/simcore_service_webserver/workspaces/_common/_exceptions_handlers.py index 1bb16355b80f..32bb81224a79 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_exceptions_handlers.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_common/_exceptions_handlers.py @@ -2,14 +2,14 @@ from servicelib.aiohttp import status -from ..exception_handling import ( +from ...exception_handling import ( ExceptionToHttpErrorMap, HttpErrorInfo, exception_handling_decorator, to_exceptions_handlers_map, ) -from ..projects.exceptions import ProjectRunningConflictError, ProjectStoppingError -from .errors import ( +from ...projects.exceptions import ProjectRunningConflictError, ProjectStoppingError +from ..errors import ( WorkspaceAccessForbiddenError, WorkspaceGroupNotFoundError, WorkspaceNotFoundError, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_models.py b/services/web/server/src/simcore_service_webserver/workspaces/_common/_models.py similarity index 98% rename from services/web/server/src/simcore_service_webserver/workspaces/_models.py rename to services/web/server/src/simcore_service_webserver/workspaces/_common/_models.py index d2f22a3c8789..a94ec063f15a 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_models.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_common/_models.py @@ -18,7 +18,7 @@ from pydantic import BaseModel, BeforeValidator, ConfigDict, Field from servicelib.request_keys import RQT_USERID_KEY -from .._constants import RQ_PRODUCT_KEY +from ..._constants import RQ_PRODUCT_KEY _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py index 599305c3f81f..ed755dba35d0 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py @@ -12,14 +12,14 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _groups_api -from ._exceptions_handlers import handle_plugin_requests_exceptions -from ._groups_api import WorkspaceGroupGet -from ._models import ( +from ._common._exceptions_handlers import handle_plugin_requests_exceptions +from ._common._models import ( WorkspacesGroupsBodyParams, WorkspacesGroupsPathParams, WorkspacesPathParams, WorkspacesRequestContext, ) +from ._groups_api import WorkspaceGroupGet _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py index 99f4e27ddc18..4284acacbfa9 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py @@ -12,8 +12,8 @@ from ..products.api import get_product_name from ..security.decorators import permission_required from . import _trash_services -from ._exceptions_handlers import handle_plugin_requests_exceptions -from ._models import WorkspacesPathParams, WorkspaceTrashQueryParams +from ._common._exceptions_handlers import handle_plugin_requests_exceptions +from ._common._models import WorkspacesPathParams, WorkspaceTrashQueryParams _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py index c1f706f259af..b606b5419a61 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py @@ -24,8 +24,8 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _workspaces_api -from ._exceptions_handlers import handle_plugin_requests_exceptions -from ._models import ( +from ._common._exceptions_handlers import handle_plugin_requests_exceptions +from ._common._models import ( WorkspacesFilters, WorkspacesListQueryParams, WorkspacesPathParams, From 14980969794aa4bef44ea1fd0ef1c310502149bc Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 16:59:36 +0100 Subject: [PATCH 39/71] rename layers in workspaces --- api/specs/web-server/_workspaces.py | 2 +- .../projects/_crud_api_create.py | 2 +- .../projects/_crud_api_read.py | 2 +- .../simcore_service_webserver/projects/_tags_api.py | 2 +- .../projects/projects_api.py | 2 +- .../{_groups_db.py => _groups_repository.py} | 0 .../{_groups_handlers.py => _groups_rest.py} | 12 ++++++------ .../{_groups_api.py => _groups_service.py} | 8 ++++---- .../workspaces/_trash_services.py | 4 ++-- .../{_workspaces_db.py => _workspaces_repository.py} | 0 .../{_workspaces_handlers.py => _workspaces_rest.py} | 12 ++++++------ .../{_workspaces_api.py => _workspaces_service.py} | 2 +- .../src/simcore_service_webserver/workspaces/api.py | 2 +- .../simcore_service_webserver/workspaces/plugin.py | 6 +++--- ...s_crud_handlers__clone_in_workspace_and_folder.py | 2 +- .../unit/with_dbs/04/workspaces/test_workspaces.py | 2 +- 16 files changed, 30 insertions(+), 30 deletions(-) rename services/web/server/src/simcore_service_webserver/workspaces/{_groups_db.py => _groups_repository.py} (100%) rename services/web/server/src/simcore_service_webserver/workspaces/{_groups_handlers.py => _groups_rest.py} (91%) rename services/web/server/src/simcore_service_webserver/workspaces/{_groups_api.py => _groups_service.py} (96%) rename services/web/server/src/simcore_service_webserver/workspaces/{_workspaces_db.py => _workspaces_repository.py} (100%) rename services/web/server/src/simcore_service_webserver/workspaces/{_workspaces_handlers.py => _workspaces_rest.py} (92%) rename services/web/server/src/simcore_service_webserver/workspaces/{_workspaces_api.py => _workspaces_service.py} (99%) diff --git a/api/specs/web-server/_workspaces.py b/api/specs/web-server/_workspaces.py index 56350f594665..a86f5ceaaae6 100644 --- a/api/specs/web-server/_workspaces.py +++ b/api/specs/web-server/_workspaces.py @@ -28,7 +28,7 @@ WorkspacesListQueryParams, WorkspacesPathParams, ) -from simcore_service_webserver.workspaces._groups_api import WorkspaceGroupGet +from simcore_service_webserver.workspaces._groups_service import WorkspaceGroupGet router = APIRouter( prefix=f"/{API_VTAG}", diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 3d276c9f7184..f3c547e2e2e1 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -34,7 +34,7 @@ get_project_total_size_simcore_s3, ) from ..users.api import get_user_fullname -from ..workspaces import _workspaces_db as workspaces_db +from ..workspaces import _workspaces_repository as workspaces_db from ..workspaces.api import check_user_workspace_access from ..workspaces.errors import WorkspaceAccessForbiddenError from . import _folders_db as project_to_folders_db diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index 77cdd5cf7d49..0b772d592e11 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -20,7 +20,7 @@ from ..catalog.client import get_services_for_user_in_product from ..folders import _folders_repository as folders_db -from ..workspaces._workspaces_api import check_user_workspace_access +from ..workspaces._workspaces_service import check_user_workspace_access from . import projects_api from ._permalink_api import update_or_pop_permalink_in_project from .db import ProjectDBAPI diff --git a/services/web/server/src/simcore_service_webserver/projects/_tags_api.py b/services/web/server/src/simcore_service_webserver/projects/_tags_api.py index c8e0937dbdbb..1fa9bdcb5900 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_tags_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_tags_api.py @@ -9,7 +9,7 @@ from models_library.users import UserID from models_library.workspaces import UserWorkspaceAccessRightsDB -from ..workspaces import _workspaces_db as workspaces_db +from ..workspaces import _workspaces_repository as workspaces_db from ._access_rights_api import check_user_project_permission from .db import ProjectDBAPI from .models import ProjectDict diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 3edd4c50e391..6919567c466d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -120,7 +120,7 @@ ) from ..wallets import api as wallets_api from ..wallets.errors import WalletNotEnoughCreditsError -from ..workspaces import _workspaces_db as workspaces_db +from ..workspaces import _workspaces_repository as workspaces_db from . import _crud_api_delete, _nodes_api, _projects_db from ._access_rights_api import ( check_user_project_permission, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_db.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_repository.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/workspaces/_groups_db.py rename to services/web/server/src/simcore_service_webserver/workspaces/_groups_repository.py diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_rest.py similarity index 91% rename from services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py rename to services/web/server/src/simcore_service_webserver/workspaces/_groups_rest.py index ed755dba35d0..487e937acb6d 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_groups_handlers.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_groups_rest.py @@ -11,7 +11,7 @@ from ..login.decorators import login_required from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _groups_api +from . import _groups_service from ._common._exceptions_handlers import handle_plugin_requests_exceptions from ._common._models import ( WorkspacesGroupsBodyParams, @@ -19,7 +19,7 @@ WorkspacesPathParams, WorkspacesRequestContext, ) -from ._groups_api import WorkspaceGroupGet +from ._groups_service import WorkspaceGroupGet _logger = logging.getLogger(__name__) @@ -43,7 +43,7 @@ async def create_workspace_group(request: web.Request): path_params = parse_request_path_parameters_as(WorkspacesGroupsPathParams, request) body_params = await parse_request_body_as(WorkspacesGroupsBodyParams, request) - workspace_groups: WorkspaceGroupGet = await _groups_api.create_workspace_group( + workspace_groups: WorkspaceGroupGet = await _groups_service.create_workspace_group( request.app, user_id=req_ctx.user_id, workspace_id=path_params.workspace_id, @@ -67,7 +67,7 @@ async def list_workspace_groups(request: web.Request): workspaces_groups: list[ WorkspaceGroupGet - ] = await _groups_api.list_workspace_groups_by_user_and_workspace( + ] = await _groups_service.list_workspace_groups_by_user_and_workspace( request.app, user_id=req_ctx.user_id, workspace_id=path_params.workspace_id, @@ -89,7 +89,7 @@ async def replace_workspace_group(request: web.Request): path_params = parse_request_path_parameters_as(WorkspacesGroupsPathParams, request) body_params = await parse_request_body_as(WorkspacesGroupsBodyParams, request) - workspace_group = await _groups_api.update_workspace_group( + workspace_group = await _groups_service.update_workspace_group( app=request.app, user_id=req_ctx.user_id, workspace_id=path_params.workspace_id, @@ -113,7 +113,7 @@ async def delete_workspace_group(request: web.Request): req_ctx = WorkspacesRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesGroupsPathParams, request) - await _groups_api.delete_workspace_group( + await _groups_service.delete_workspace_group( app=request.app, user_id=req_ctx.user_id, workspace_id=path_params.workspace_id, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_api.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py similarity index 96% rename from services/web/server/src/simcore_service_webserver/workspaces/_groups_api.py rename to services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py index 2ca935c89671..5ff4449a57a4 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_groups_api.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py @@ -9,10 +9,10 @@ from pydantic import BaseModel, ConfigDict from ..users import api as users_api -from . import _groups_db as workspaces_groups_db -from . import _workspaces_db as workspaces_db -from ._groups_db import WorkspaceGroupGetDB -from ._workspaces_api import check_user_workspace_access +from . import _groups_repository as workspaces_groups_db +from . import _workspaces_repository as workspaces_db +from ._groups_repository import WorkspaceGroupGetDB +from ._workspaces_service import check_user_workspace_access from .errors import WorkspaceAccessForbiddenError log = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py index 817b2c41983d..667ca4841a7e 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py @@ -12,8 +12,8 @@ from ..db.plugin import get_asyncpg_engine from ..folders._trash_service import trash_folder, untrash_folder from ..projects._trash_service import trash_project, untrash_project -from ._workspaces_api import check_user_workspace_access -from ._workspaces_db import update_workspace +from ._workspaces_repository import update_workspace +from ._workspaces_service import check_user_workspace_access _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_db.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/workspaces/_workspaces_db.py rename to services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py similarity index 92% rename from services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py rename to services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py index b606b5419a61..e5268d245a33 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_handlers.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py @@ -23,7 +23,7 @@ from ..login.decorators import login_required from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response -from . import _workspaces_api +from . import _workspaces_service from ._common._exceptions_handlers import handle_plugin_requests_exceptions from ._common._models import ( WorkspacesFilters, @@ -46,7 +46,7 @@ async def create_workspace(request: web.Request): req_ctx = WorkspacesRequestContext.model_validate(request) body_params = await parse_request_body_as(WorkspaceCreateBodyParams, request) - workspace: WorkspaceGet = await _workspaces_api.create_workspace( + workspace: WorkspaceGet = await _workspaces_service.create_workspace( request.app, user_id=req_ctx.user_id, name=body_params.name, @@ -72,7 +72,7 @@ async def list_workspaces(request: web.Request): query_params.filters = WorkspacesFilters() assert query_params.filters - workspaces: WorkspaceGetPage = await _workspaces_api.list_workspaces( + workspaces: WorkspaceGetPage = await _workspaces_service.list_workspaces( app=request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, @@ -106,7 +106,7 @@ async def get_workspace(request: web.Request): req_ctx = WorkspacesRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) - workspace: WorkspaceGet = await _workspaces_api.get_workspace( + workspace: WorkspaceGet = await _workspaces_service.get_workspace( app=request.app, workspace_id=path_params.workspace_id, user_id=req_ctx.user_id, @@ -128,7 +128,7 @@ async def replace_workspace(request: web.Request): path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) body_params = await parse_request_body_as(WorkspaceReplaceBodyParams, request) - workspace: WorkspaceGet = await _workspaces_api.update_workspace( + workspace: WorkspaceGet = await _workspaces_service.update_workspace( app=request.app, user_id=req_ctx.user_id, workspace_id=path_params.workspace_id, @@ -149,7 +149,7 @@ async def delete_workspace(request: web.Request): req_ctx = WorkspacesRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) - await _workspaces_api.delete_workspace( + await _workspaces_service.delete_workspace( app=request.app, user_id=req_ctx.user_id, workspace_id=path_params.workspace_id, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_api.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py similarity index 99% rename from services/web/server/src/simcore_service_webserver/workspaces/_workspaces_api.py rename to services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py index b4881c2816c2..0be4a1bac4ab 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_api.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py @@ -19,7 +19,7 @@ from ..projects._db_utils import PermissionStr from ..users.api import get_user -from . import _workspaces_db as db +from . import _workspaces_repository as db from .errors import WorkspaceAccessForbiddenError _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/api.py b/services/web/server/src/simcore_service_webserver/workspaces/api.py index 2b3ca3bbd42f..7a0e2d338bc5 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/api.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/api.py @@ -1,5 +1,5 @@ # mypy: disable-error-code=truthy-function -from ._workspaces_api import check_user_workspace_access, get_workspace +from ._workspaces_service import check_user_workspace_access, get_workspace assert get_workspace # nosec assert check_user_workspace_access # nosec diff --git a/services/web/server/src/simcore_service_webserver/workspaces/plugin.py b/services/web/server/src/simcore_service_webserver/workspaces/plugin.py index 65f1866441f4..b5936e128db4 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/plugin.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/plugin.py @@ -7,7 +7,7 @@ from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup -from . import _groups_handlers, _trash_rest, _workspaces_handlers +from . import _groups_rest, _trash_rest, _workspaces_rest _logger = logging.getLogger(__name__) @@ -23,6 +23,6 @@ def setup_workspaces(app: web.Application): assert app[APP_SETTINGS_KEY].WEBSERVER_WORKSPACES # nosec # routes - app.router.add_routes(_workspaces_handlers.routes) - app.router.add_routes(_groups_handlers.routes) + app.router.add_routes(_workspaces_rest.routes) + app.router.add_routes(_groups_rest.routes) app.router.add_routes(_trash_rest.routes) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py index 6fef8ab7870a..42e9d2cf971d 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py @@ -23,7 +23,7 @@ from simcore_service_webserver.folders._folders_service import create_folder from simcore_service_webserver.projects._folders_api import move_project_into_folder from simcore_service_webserver.projects.models import ProjectDict -from simcore_service_webserver.workspaces._workspaces_api import create_workspace +from simcore_service_webserver.workspaces._workspaces_service import create_workspace from yarl import URL diff --git a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py index 56753822a6df..aa3d8bb367e4 100644 --- a/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py +++ b/services/web/server/tests/unit/with_dbs/04/workspaces/test_workspaces.py @@ -20,7 +20,7 @@ from servicelib.aiohttp import status from simcore_service_webserver.db.models import UserRole from simcore_service_webserver.projects.models import ProjectDict -from simcore_service_webserver.workspaces._workspaces_handlers import ( +from simcore_service_webserver.workspaces._workspaces_rest import ( WorkspacesListQueryParams, ) From a0daa3d22c2c5195c98baea4bffd471c683bb747 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 17:40:34 +0100 Subject: [PATCH 40/71] folders columns --- .../models/folders_v2.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/folders_v2.py b/packages/postgres-database/src/simcore_postgres_database/models/folders_v2.py index 78f3de8bdf9d..eebfd2079f80 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/folders_v2.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/folders_v2.py @@ -1,8 +1,15 @@ import sqlalchemy as sa from sqlalchemy.sql import expression -from ._common import RefActions, column_created_datetime, column_modified_datetime +from ._common import ( + RefActions, + column_created_datetime, + column_modified_datetime, + column_trashed_by_user, + column_trashed_datetime, +) from .base import metadata +from .users import users from .workspaces import workspaces folders_v2 = sa.Table( @@ -75,13 +82,8 @@ ), column_created_datetime(timezone=True), column_modified_datetime(timezone=True), - sa.Column( - "trashed_at", - sa.DateTime(timezone=True), - nullable=True, - comment="The date and time when the folder was marked as trashed." - "Null if the folder has not been trashed [default].", - ), + column_trashed_datetime("folders"), + column_trashed_by_user("folders", users_table=users), sa.Column( "trashed_explicitly", sa.Boolean, From 3f4c25e2bb57d7c88efe4f024858441a0e22a7a8 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 17:42:18 +0100 Subject: [PATCH 41/71] projects columns --- .../src/simcore_postgres_database/models/projects.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/postgres-database/src/simcore_postgres_database/models/projects.py b/packages/postgres-database/src/simcore_postgres_database/models/projects.py index 93ff3a74ea3b..e13f9bc42214 100644 --- a/packages/postgres-database/src/simcore_postgres_database/models/projects.py +++ b/packages/postgres-database/src/simcore_postgres_database/models/projects.py @@ -7,8 +7,9 @@ from sqlalchemy.dialects.postgresql import ARRAY, JSONB from sqlalchemy.sql import expression, func -from ._common import RefActions +from ._common import RefActions, column_trashed_by_user, column_trashed_datetime from .base import metadata +from .users import users class ProjectType(enum.Enum): @@ -142,13 +143,8 @@ class ProjectType(enum.Enum): default=False, doc="If true, the project is by default not listed in the API", ), - sa.Column( - "trashed_at", - sa.DateTime(timezone=True), - nullable=True, - comment="The date and time when the project was marked as trashed. " - "Null if the project has not been trashed [default].", - ), + column_trashed_datetime("projects"), + column_trashed_by_user("projects", users_table=users), sa.Column( "trashed_explicitly", sa.Boolean, From b412011827c8a4b0e4a14839622ea228ae826c50 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 17:58:06 +0100 Subject: [PATCH 42/71] migration --- .../f19905923355_adds_trashed_by_column.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 packages/postgres-database/src/simcore_postgres_database/migration/versions/f19905923355_adds_trashed_by_column.py diff --git a/packages/postgres-database/src/simcore_postgres_database/migration/versions/f19905923355_adds_trashed_by_column.py b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f19905923355_adds_trashed_by_column.py new file mode 100644 index 000000000000..0f7de4587911 --- /dev/null +++ b/packages/postgres-database/src/simcore_postgres_database/migration/versions/f19905923355_adds_trashed_by_column.py @@ -0,0 +1,84 @@ +"""Adds trashed by column + +Revision ID: f19905923355 +Revises: 307017ee1a49 +Create Date: 2025-01-10 16:43:21.559138+00:00 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f19905923355" +down_revision = "307017ee1a49" +branch_labels = None +depends_on = None + + +def upgrade(): + + with op.batch_alter_table("folders_v2") as batch_op: + batch_op.alter_column( + "trashed_at", + new_column_name="trashed", + comment="The date and time when the folders was marked as trashed. Null if the folders has not been trashed [default].", + ) + batch_op.add_column( + sa.Column( + "trashed_by", + sa.BigInteger(), + nullable=True, + comment="User who trashed the folders, or null if not trashed or user is unknown.", + ) + ) + batch_op.create_foreign_key( + "fk_folders_trashed_by_user_id", + "users", + ["trashed_by"], + ["id"], + onupdate="CASCADE", + ondelete="SET NULL", + ) + + with op.batch_alter_table("projects") as batch_op: + batch_op.alter_column( + "trashed_at", + new_column_name="trashed", + comment="The date and time when the projects was marked as trashed. Null if the projects has not been trashed [default].", + ) + batch_op.add_column( + sa.Column( + "trashed_by", + sa.BigInteger(), + nullable=True, + comment="User who trashed the projects, or null if not trashed or user is unknown.", + ) + ) + batch_op.create_foreign_key( + "fk_projects_trashed_by_user_id", + "users", + ["trashed_by"], + ["id"], + onupdate="CASCADE", + ondelete="SET NULL", + ) + + +def downgrade(): + with op.batch_alter_table("projects") as batch_op: + batch_op.drop_constraint("fk_projects_trashed_by_user_id", type_="foreignkey") + batch_op.drop_column("trashed_by") + batch_op.alter_column( + "trashed", + new_column_name="trashed_at", + comment="The date and time when the project was marked as trashed. Null if the project has not been trashed [default].", + ) + + with op.batch_alter_table("folders_v2") as batch_op: + batch_op.drop_constraint("fk_folders_trashed_by_user_id", type_="foreignkey") + batch_op.drop_column("trashed_by") + batch_op.alter_column( + "trashed", + new_column_name="trashed_at", + comment="The date and time when the folder was marked as trashed. Null if the folder has not been trashed [default].", + ) From 1e48e05c1ba1ae857d6f8fc77a68f9d2994b73ce Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Fri, 10 Jan 2025 18:29:29 +0100 Subject: [PATCH 43/71] rename trashed_at in db --- packages/common-library/src/common_library/exclude.py | 4 ++++ .../models_library/api_schemas_webserver/projects.py | 3 +++ packages/models-library/src/models_library/folders.py | 2 +- .../postgres-database/tests/test_utils_projects.py | 9 +++++---- .../folders/_folders_repository.py | 10 +++++----- .../folders/_folders_service.py | 10 +++++----- .../folders/_trash_service.py | 4 ++-- .../simcore_service_webserver/projects/_db_utils.py | 2 +- .../projects/_projects_db.py | 2 +- .../src/simcore_service_webserver/projects/db.py | 4 ++-- .../src/simcore_service_webserver/projects/models.py | 11 +++++++++-- .../projects/projects_api.py | 8 +++----- 12 files changed, 41 insertions(+), 28 deletions(-) diff --git a/packages/common-library/src/common_library/exclude.py b/packages/common-library/src/common_library/exclude.py index 6f635dfe643f..7f2392dec338 100644 --- a/packages/common-library/src/common_library/exclude.py +++ b/packages/common-library/src/common_library/exclude.py @@ -8,6 +8,10 @@ class UnSet: UnSet.VALUE = UnSet() +def is_unset(v: Any) -> bool: + return isinstance(v, UnSet) + + def as_dict_exclude_unset(**params) -> dict[str, Any]: return {k: v for k, v in params.items() if not isinstance(v, UnSet)} diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index a595245e331b..a227ce606fa1 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -160,6 +160,9 @@ class ProjectPatch(InputSchema): ] = Field(default=None) quality: dict[str, Any] | None = Field(default=None) + def to_model(self) -> dict[str, Any]: + return self.model_dump(exclude_unset=True, by_alias=False) + __all__: tuple[str, ...] = ( "EmptyModel", diff --git a/packages/models-library/src/models_library/folders.py b/packages/models-library/src/models_library/folders.py index 554311731119..0a3821fc9873 100644 --- a/packages/models-library/src/models_library/folders.py +++ b/packages/models-library/src/models_library/folders.py @@ -64,7 +64,7 @@ class FolderDB(BaseModel): ..., description="Timestamp of last modification", ) - trashed_at: datetime | None = Field( + trashed: datetime | None = Field( ..., ) diff --git a/packages/postgres-database/tests/test_utils_projects.py b/packages/postgres-database/tests/test_utils_projects.py index c97c822090f6..be4cde5f180d 100644 --- a/packages/postgres-database/tests/test_utils_projects.py +++ b/packages/postgres-database/tests/test_utils_projects.py @@ -54,7 +54,7 @@ async def registered_project( @pytest.mark.parametrize("expected", (datetime.now(tz=UTC), None)) -async def test_get_project_trashed_at_column_can_be_converted_to_datetime( +async def test_get_project_trashed_column_can_be_converted_to_datetime( asyncpg_engine: AsyncEngine, registered_project: dict, expected: datetime | None ): project_id = registered_project["uuid"] @@ -62,15 +62,16 @@ async def test_get_project_trashed_at_column_can_be_converted_to_datetime( async with transaction_context(asyncpg_engine) as conn: result = await conn.execute( projects.update() - .values(trashed_at=expected) + .values(trashed=expected) .where(projects.c.uuid == project_id) .returning(sa.literal_column("*")) ) row = result.fetchone() - trashed_at = TypeAdapter(datetime | None).validate_python(row.trashed_at) - assert trashed_at == expected + assert row + trashed = TypeAdapter(datetime | None).validate_python(row.trashed) + assert trashed == expected async def test_get_project_last_change_date( diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py index 32ff9e4d3a53..98b6bd34fabe 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py @@ -55,7 +55,7 @@ folders_v2.c.created_by_gid, folders_v2.c.created, folders_v2.c.modified, - folders_v2.c.trashed_at, + folders_v2.c.trashed, folders_v2.c.user_id, folders_v2.c.workspace_id, ) @@ -185,11 +185,11 @@ async def list_( # pylint: disable=too-many-arguments,too-many-branches if filter_trashed is not None: attributes_filters.append( ( - (folders_v2.c.trashed_at.is_not(None)) + (folders_v2.c.trashed.is_not(None)) & (folders_v2.c.trashed_explicitly.is_(True)) ) if filter_trashed - else folders_v2.c.trashed_at.is_(None) + else folders_v2.c.trashed.is_(None) ) if folder_query.folder_scope is not FolderScope.ALL: if folder_query.folder_scope == FolderScope.SPECIFIC: @@ -315,7 +315,7 @@ async def update( # updatable columns name: str | UnSet = UnSet.VALUE, parent_folder_id: FolderID | None | UnSet = UnSet.VALUE, - trashed_at: datetime | None | UnSet = UnSet.VALUE, + trashed: datetime | None | UnSet = UnSet.VALUE, trashed_explicitly: bool | UnSet = UnSet.VALUE, workspace_id: WorkspaceID | None | UnSet = UnSet.VALUE, user_id: UserID | None | UnSet = UnSet.VALUE, @@ -327,7 +327,7 @@ async def update( updated = as_dict_exclude_unset( name=name, parent_folder_id=parent_folder_id, - trashed_at=trashed_at, + trashed=trashed, trashed_explicitly=trashed_explicitly, workspace_id=workspace_id, user_id=user_id, diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_service.py b/services/web/server/src/simcore_service_webserver/folders/_folders_service.py index 52d37412e32b..e99605125586 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_service.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_service.py @@ -92,7 +92,7 @@ async def create_folder( name=folder_db.name, created_at=folder_db.created, modified_at=folder_db.modified, - trashed_at=folder_db.trashed_at, + trashed_at=folder_db.trashed, owner=folder_db.created_by_gid, workspace_id=workspace_id, my_access_rights=user_folder_access_rights, @@ -135,7 +135,7 @@ async def get_folder( name=folder_db.name, created_at=folder_db.created, modified_at=folder_db.modified, - trashed_at=folder_db.trashed_at, + trashed_at=folder_db.trashed, owner=folder_db.created_by_gid, workspace_id=folder_db.workspace_id, my_access_rights=user_folder_access_rights, @@ -185,7 +185,7 @@ async def list_folders( name=folder.name, created_at=folder.created, modified_at=folder.modified, - trashed_at=folder.trashed_at, + trashed_at=folder.trashed, owner=folder.created_by_gid, workspace_id=folder.workspace_id, my_access_rights=folder.my_access_rights, @@ -229,7 +229,7 @@ async def list_folders_full_depth( name=folder.name, created_at=folder.created, modified_at=folder.modified, - trashed_at=folder.trashed_at, + trashed_at=folder.trashed, owner=folder.created_by_gid, workspace_id=folder.workspace_id, my_access_rights=folder.my_access_rights, @@ -306,7 +306,7 @@ async def update_folder( name=folder_db.name, created_at=folder_db.created, modified_at=folder_db.modified, - trashed_at=folder_db.trashed_at, + trashed_at=folder_db.trashed, owner=folder_db.created_by_gid, workspace_id=folder_db.workspace_id, my_access_rights=user_folder_access_rights, diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py index 468d0a7d4a3a..a7318e228e66 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py +++ b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py @@ -70,7 +70,7 @@ async def _folders_db_update( connection, folders_id_or_ids=folder_id, product_name=product_name, - trashed_at=trashed_at, + trashed=trashed_at, trashed_explicitly=trashed_at is not None, ) @@ -89,7 +89,7 @@ async def _folders_db_update( connection, folders_id_or_ids=child_folders, product_name=product_name, - trashed_at=trashed_at, + trashed=trashed_at, trashed_explicitly=False, ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py index 2b14c2d1566f..0dc513c1f22f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py @@ -86,7 +86,7 @@ def convert_to_schema_names( if key in DB_EXCLUSIVE_COLUMNS: continue converted_value = value - if isinstance(value, datetime) and key not in {"trashed_at"}: + if isinstance(value, datetime) and key not in {"trashed"}: converted_value = format_datetime(value) elif key == "prj_owner": # this entry has to be converted to the owner e-mail address diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_db.py b/services/web/server/src/simcore_service_webserver/projects/_projects_db.py index 3c94e9e7cdc2..92c73867f779 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_db.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_db.py @@ -34,7 +34,7 @@ projects.c.published, projects.c.hidden, projects.c.workspace_id, - projects.c.trashed_at, + projects.c.trashed, ] diff --git a/services/web/server/src/simcore_service_webserver/projects/db.py b/services/web/server/src/simcore_service_webserver/projects/db.py index 7890537a9843..47704f74e2fd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/db.py +++ b/services/web/server/src/simcore_service_webserver/projects/db.py @@ -567,12 +567,12 @@ async def list_projects( # pylint: disable=too-many-arguments,too-many-statemen attributes_filters.append( # marked explicitly as trashed ( - projects.c.trashed_at.is_not(None) + projects.c.trashed.is_not(None) & projects.c.trashed_explicitly.is_(True) ) if filter_trashed # not marked as trashed - else projects.c.trashed_at.is_(None) + else projects.c.trashed.is_(None) ) if filter_by_text is not None: attributes_filters.append( diff --git a/services/web/server/src/simcore_service_webserver/projects/models.py b/services/web/server/src/simcore_service_webserver/projects/models.py index dca631ba39a1..ee9321600a8f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/models.py +++ b/services/web/server/src/simcore_service_webserver/projects/models.py @@ -3,6 +3,7 @@ from typing import Any, TypeAlias from aiopg.sa.result import RowProxy +from common_library.dict_tools import remap_keys from models_library.api_schemas_webserver.projects import ProjectPatch from models_library.folders import FolderID from models_library.projects import ClassifierID, ProjectID @@ -51,7 +52,7 @@ class ProjectDB(BaseModel): published: bool hidden: bool workspace_id: WorkspaceID | None - trashed_at: datetime | None + trashed: datetime | None trashed_explicitly: bool = False model_config = ConfigDict(from_attributes=True, arbitrary_types_allowed=True) @@ -94,12 +95,18 @@ class UserProjectAccessRightsWithWorkspace(BaseModel): class ProjectPatchExtended(ProjectPatch): - # Only used internally + # ONLY used internally trashed_at: datetime | None trashed_explicitly: bool model_config = ConfigDict(populate_by_name=True, extra="forbid") + def to_model(self) -> dict[str, Any]: + return remap_keys( + self.model_dump(exclude_unset=True, by_alias=False), + rename={"trashed_at": "trash"}, + ) + __all__: tuple[str, ...] = ( "ProjectDict", diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 6919567c466d..99c827943f1c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -254,9 +254,7 @@ async def patch_project( project_patch: ProjectPatch | ProjectPatchExtended, product_name: ProductName, ): - _project_patch_exclude_unset = project_patch.model_dump( - exclude_unset=True, by_alias=False - ) + patch_project_data = project_patch.to_model() db: ProjectDBAPI = app[APP_PROJECT_DBAPI] # 1. Get project @@ -272,7 +270,7 @@ async def patch_project( ) # 3. If patching access rights - if new_prj_access_rights := _project_patch_exclude_unset.get("access_rights"): + if new_prj_access_rights := patch_project_data.get("access_rights"): # 3.1 Check if user is Owner and therefore can modify access rights if not _user_project_access_rights.delete: raise ProjectInvalidRightsError(user_id=user_id, project_uuid=project_uuid) @@ -293,7 +291,7 @@ async def patch_project( await _projects_db.patch_project( app=app, project_uuid=project_uuid, - new_partial_project_data=_project_patch_exclude_unset, + new_partial_project_data=patch_project_data, ) From f8ef4e43afac9b86a1b1224f987d48509fb2e1ab Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:34:40 +0100 Subject: [PATCH 44/71] fixes test --- .../web/server/src/simcore_service_webserver/trash/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/trash/settings.py b/services/web/server/src/simcore_service_webserver/trash/settings.py index 55456f6da2ed..38d4f91fdcb6 100644 --- a/services/web/server/src/simcore_service_webserver/trash/settings.py +++ b/services/web/server/src/simcore_service_webserver/trash/settings.py @@ -7,7 +7,8 @@ class TrashSettings(BaseCustomSettings): TRASH_RETENTION_DAYS: NonNegativeInt = Field( - description="Trashed items will be deleted after this time", + default=7, + description="Number of days that trashed items are kept in the bin before finally deleting them", ) From 001a4dde544536a654301ad62c403791e58aaf88 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 11:33:01 +0100 Subject: [PATCH 45/71] fixing mapping --- .../api_schemas_webserver/projects.py | 18 ++++++++- .../src/models_library/projects.py | 13 ++++--- .../projects/_crud_api_create.py | 2 +- .../projects/_db_utils.py | 37 ++++++++++++++----- .../tests/unit/with_dbs/03/test_project_db.py | 4 +- 5 files changed, 56 insertions(+), 18 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index a227ce606fa1..9c1080263ae5 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -6,8 +6,9 @@ """ from datetime import datetime -from typing import Annotated, Any, Literal, TypeAlias +from typing import Annotated, Any, Literal, Self, TypeAlias +from common_library.dict_tools import remap_keys from models_library.folders import FolderID from models_library.utils._original_fastapi_encoders import jsonable_encoder from models_library.workspaces import WorkspaceID @@ -35,6 +36,7 @@ ) from ..workspaces import WorkspaceID from ._base import EmptyModel, InputSchema, OutputSchema +from .groups import GroupID from .permalinks import ProjectPermalink @@ -95,7 +97,12 @@ class ProjectGet(OutputSchema): permalink: ProjectPermalink | None = None workspace_id: WorkspaceID | None folder_id: FolderID | None + trashed_at: datetime | None + trashed_by: Annotated[ + GroupID | None, + Field(description="Primary group ID of the user who trashed this item"), + ] _empty_description = field_validator("description", mode="before")( none_to_empty_str_pre_validator @@ -103,6 +110,15 @@ class ProjectGet(OutputSchema): model_config = ConfigDict(frozen=False) + @classmethod + def from_model(cls, project: dict[str, Any]) -> Self: + return cls.model_validate( + remap_keys( + project, + rename={"trashed": "trashed_at"}, + ) + ) + TaskProjectGet: TypeAlias = TaskGet diff --git a/packages/models-library/src/models_library/projects.py b/packages/models-library/src/models_library/projects.py index c25309dac3e2..81f230768cc7 100644 --- a/packages/models-library/src/models_library/projects.py +++ b/packages/models-library/src/models_library/projects.py @@ -20,6 +20,7 @@ from .projects_nodes_io import NodeIDStr from .projects_state import ProjectState from .projects_ui import StudyUI +from .users import UserID from .utils.common_validators import ( empty_str_to_none_pre_validator, none_to_empty_str_pre_validator, @@ -182,10 +183,10 @@ class Project(BaseProjectModel): alias="folderId", ) - trashed_at: datetime | None = Field( - default=None, - alias="trashedAt", - ) - trashed_explicitly: bool = Field(default=False, alias="trashedExplicitly") + trashed: datetime | None = None + trashed_by: Annotated[UserID | None, Field(alias="trashedBy")] = None + trashed_explicitly: Annotated[bool, Field(alias="trashedExplicitly")] = False - model_config = ConfigDict(title="osparc-simcore project", extra="forbid") + model_config = ConfigDict( + extra="forbid", + ) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index f3c547e2e2e1..29a1d215c97a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -423,7 +423,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche } # Ensures is like ProjectGet - data = ProjectGet.model_validate(new_project).data(exclude_unset=True) + data = ProjectGet.from_model(new_project).data(exclude_unset=True) raise web.HTTPCreated( text=json_dumps({"data": data}), diff --git a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py index 0dc513c1f22f..da1f06d06dfd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_db_utils.py +++ b/services/web/server/src/simcore_service_webserver/projects/_db_utils.py @@ -82,20 +82,20 @@ def convert_to_schema_names( ) -> dict: # SEE https://github.com/ITISFoundation/osparc-simcore/issues/3516 converted_args = {} - for key, value in project_database_data.items(): - if key in DB_EXCLUSIVE_COLUMNS: + for col_name, col_value in project_database_data.items(): + if col_name in DB_EXCLUSIVE_COLUMNS: continue - converted_value = value - if isinstance(value, datetime) and key not in {"trashed"}: - converted_value = format_datetime(value) - elif key == "prj_owner": + converted_value = col_value + if isinstance(col_value, datetime) and col_name not in {"trashed"}: + converted_value = format_datetime(col_value) + elif col_name == "prj_owner": # this entry has to be converted to the owner e-mail address converted_value = user_email - if key in SCHEMA_NON_NULL_KEYS and value is None: + if col_name in SCHEMA_NON_NULL_KEYS and col_value is None: converted_value = "" - converted_args[snake_to_camel(key)] = converted_value + converted_args[snake_to_camel(col_name)] = converted_value converted_args.update(**kwargs) return converted_args @@ -275,7 +275,26 @@ async def _get_project( query = ( sa.select( - *[col for col in projects.columns if col.name not in ["access_rights"]], + projects.c.id, + projects.c.type, + projects.c.uuid, + projects.c.name, + projects.c.description, + projects.c.thumbnail, + projects.c.prj_owner, # == user.id (who created) + projects.c.creation_date, + projects.c.last_change_date, + projects.c.workbench, + projects.c.ui, + projects.c.classifiers, + projects.c.dev, + projects.c.quality, + projects.c.published, + projects.c.hidden, + projects.c.trashed, + projects.c.trashed_by, # == user.id (who trashed) + projects.c.trashed_explicitly, + projects.c.workspace_id, access_rights_subquery.c.access_rights, ) .select_from(projects.join(access_rights_subquery, isouter=True)) diff --git a/services/web/server/tests/unit/with_dbs/03/test_project_db.py b/services/web/server/tests/unit/with_dbs/03/test_project_db.py index 098a4a783888..d592bc1d8d90 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_project_db.py +++ b/services/web/server/tests/unit/with_dbs/03/test_project_db.py @@ -97,7 +97,8 @@ def _assert_added_project( "creationDate", "lastChangeDate", "accessRights", # NOTE: access rights were moved away from the projects table - "trashedAt", + "trashed", + "trashedBy", "trashedExplicitly", ] assert {k: v for k, v in expected_prj.items() if k in _DIFFERENT_KEYS} != { @@ -184,6 +185,7 @@ async def insert_project_in_db( client: TestClient, ) -> AsyncIterator[Callable[..., Awaitable[dict[str, Any]]]]: inserted_projects = [] + assert client.app async def _inserter(prj: dict[str, Any], **overrides) -> dict[str, Any]: # add project without user id -> by default creates a template From fb16e055df9599dea4663d72c3f832dad123d403 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:30:22 +0100 Subject: [PATCH 46/71] fixing tests --- .../api_schemas_webserver/projects.py | 5 ----- .../projects/_crud_api_read.py | 12 ++++-------- services/web/server/tests/conftest.py | 13 +++++++++---- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 9c1080263ae5..63b7fd337fac 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -36,7 +36,6 @@ ) from ..workspaces import WorkspaceID from ._base import EmptyModel, InputSchema, OutputSchema -from .groups import GroupID from .permalinks import ProjectPermalink @@ -99,10 +98,6 @@ class ProjectGet(OutputSchema): folder_id: FolderID | None trashed_at: datetime | None - trashed_by: Annotated[ - GroupID | None, - Field(description="Primary group ID of the user who trashed this item"), - ] _empty_description = field_validator("description", mode="before")( none_to_empty_str_pre_validator diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index 0b772d592e11..fea86ef0a2f7 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -6,7 +6,6 @@ """ from aiohttp import web -from models_library.api_schemas_webserver._base import OutputSchema from models_library.api_schemas_webserver.projects import ProjectListItem from models_library.folders import FolderID, FolderQuery, FolderScope from models_library.projects import ProjectID @@ -27,13 +26,12 @@ from .models import ProjectDict, ProjectTypeAPI -async def _append_fields( +async def _append_item( request: web.Request, *, user_id: UserID, project: ProjectDict, is_template: bool, - model_schema_cls: type[OutputSchema], ): # state await projects_api.add_project_states_for_user( @@ -47,7 +45,7 @@ async def _append_fields( await update_or_pop_permalink_in_project(request, project) # validate - return model_schema_cls.model_validate(project).data(exclude_unset=True) + return ProjectListItem.from_model(project).data(exclude_unset=True) async def list_projects( # pylint: disable=too-many-arguments @@ -128,12 +126,11 @@ async def list_projects( # pylint: disable=too-many-arguments projects: list[ProjectDict] = await logged_gather( *( - _append_fields( + _append_item( request, user_id=user_id, project=prj, is_template=prj_type == ProjectTypeDB.TEMPLATE, - model_schema_cls=ProjectListItem, ) for prj, prj_type in zip(db_projects, db_project_types, strict=False) ), @@ -182,12 +179,11 @@ async def list_projects_full_depth( projects: list[ProjectDict] = await logged_gather( *( - _append_fields( + _append_item( request, user_id=user_id, project=prj, is_template=prj_type == ProjectTypeDB.TEMPLATE, - model_schema_cls=ProjectListItem, ) for prj, prj_type in zip(db_projects, db_project_types, strict=False) ), diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index c474c48ab572..0f8ab3e09c4e 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -18,6 +18,7 @@ from aiohttp.test_utils import TestClient from common_library.json_serialization import json_dumps from faker import Faker +from models_library.api_schemas_webserver.projects import ProjectGet from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.projects_state import ProjectState @@ -216,7 +217,7 @@ async def _setup( project_data: ProjectDict = {} expected_data: ProjectDict = { "classifiers": [], - "accessRights": [], + "accessRights": {}, "tags": [], "lastChangeDate": None, "creationDate": None, @@ -235,12 +236,16 @@ async def _setup( "trashedAt": None, } if from_study: - # access rights are replaced - expected_data = deepcopy(from_study) - expected_data["accessRights"] = {} + from_study_wo_access_rights = deepcopy(from_study) + from_study_wo_access_rights.pop("accessRights") + expected_data = {**expected_data, **from_study_wo_access_rights} if not as_template: expected_data["name"] = f"{from_study['name']} (Copy)" + expected_data = ProjectGet.from_model(expected_data).model_dump( + mode="json", by_alias=True + ) + if not from_study or project: assert NEW_PROJECT.request_payload project_data = deepcopy(NEW_PROJECT.request_payload) From bf59729119f4949eb276838f9ce187da3bf36373 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:39:08 +0100 Subject: [PATCH 47/71] wrong name --- api/specs/web-server/_trash.py | 2 +- .../{_exceptions_handlers.py => exceptions_handlers.py} | 0 .../workspaces/_common/{_models.py => models.py} | 0 .../src/simcore_service_webserver/workspaces/_groups_rest.py | 4 ++-- .../src/simcore_service_webserver/workspaces/_trash_rest.py | 4 ++-- .../simcore_service_webserver/workspaces/_workspaces_rest.py | 4 ++-- 6 files changed, 7 insertions(+), 7 deletions(-) rename services/web/server/src/simcore_service_webserver/workspaces/_common/{_exceptions_handlers.py => exceptions_handlers.py} (100%) rename services/web/server/src/simcore_service_webserver/workspaces/_common/{_models.py => models.py} (100%) diff --git a/api/specs/web-server/_trash.py b/api/specs/web-server/_trash.py index dd6774b7944a..8f0f5ea6086a 100644 --- a/api/specs/web-server/_trash.py +++ b/api/specs/web-server/_trash.py @@ -15,7 +15,7 @@ FolderTrashQueryParams, ) from simcore_service_webserver.projects._trash_rest import ProjectPathParams -from simcore_service_webserver.workspaces._common._models import ( +from simcore_service_webserver.workspaces._common.models import ( WorkspacesPathParams, WorkspaceTrashQueryParams, ) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_common/_exceptions_handlers.py b/services/web/server/src/simcore_service_webserver/workspaces/_common/exceptions_handlers.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/workspaces/_common/_exceptions_handlers.py rename to services/web/server/src/simcore_service_webserver/workspaces/_common/exceptions_handlers.py diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_common/_models.py b/services/web/server/src/simcore_service_webserver/workspaces/_common/models.py similarity index 100% rename from services/web/server/src/simcore_service_webserver/workspaces/_common/_models.py rename to services/web/server/src/simcore_service_webserver/workspaces/_common/models.py diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_rest.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_rest.py index 487e937acb6d..0ccaf3371396 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_groups_rest.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_groups_rest.py @@ -12,8 +12,8 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _groups_service -from ._common._exceptions_handlers import handle_plugin_requests_exceptions -from ._common._models import ( +from ._common.exceptions_handlers import handle_plugin_requests_exceptions +from ._common.models import ( WorkspacesGroupsBodyParams, WorkspacesGroupsPathParams, WorkspacesPathParams, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py index 4284acacbfa9..fd7b708c1ddf 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_rest.py @@ -12,8 +12,8 @@ from ..products.api import get_product_name from ..security.decorators import permission_required from . import _trash_services -from ._common._exceptions_handlers import handle_plugin_requests_exceptions -from ._common._models import WorkspacesPathParams, WorkspaceTrashQueryParams +from ._common.exceptions_handlers import handle_plugin_requests_exceptions +from ._common.models import WorkspacesPathParams, WorkspaceTrashQueryParams _logger = logging.getLogger(__name__) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py index e5268d245a33..792df1f121f0 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py @@ -24,8 +24,8 @@ from ..security.decorators import permission_required from ..utils_aiohttp import envelope_json_response from . import _workspaces_service -from ._common._exceptions_handlers import handle_plugin_requests_exceptions -from ._common._models import ( +from ._common.exceptions_handlers import handle_plugin_requests_exceptions +from ._common.models import ( WorkspacesFilters, WorkspacesListQueryParams, WorkspacesPathParams, From 48093f1a417d1a1034ec385d46c4a38113478e7e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:40:08 +0100 Subject: [PATCH 48/71] rename --- api/specs/web-server/_folders.py | 4 ++-- .../src/simcore_service_webserver/folders/_common/models.py | 2 +- .../src/simcore_service_webserver/folders/_workspaces_rest.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/api/specs/web-server/_folders.py b/api/specs/web-server/_folders.py index 226a71dd92c6..aa0f88c8d93f 100644 --- a/api/specs/web-server/_folders.py +++ b/api/specs/web-server/_folders.py @@ -26,7 +26,7 @@ FolderSearchQueryParams, FoldersListQueryParams, FoldersPathParams, - _FolderWorkspacesPathParams, + FolderWorkspacesPathParams, ) router = APIRouter( @@ -109,6 +109,6 @@ async def delete_folder( tags=["workspaces"], ) async def move_folder_to_workspace( - _path: Annotated[_FolderWorkspacesPathParams, Depends()], + _path: Annotated[FolderWorkspacesPathParams, Depends()], ): ... diff --git a/services/web/server/src/simcore_service_webserver/folders/_common/models.py b/services/web/server/src/simcore_service_webserver/folders/_common/models.py index 37e03e42636c..551c531d74c1 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_common/models.py +++ b/services/web/server/src/simcore_service_webserver/folders/_common/models.py @@ -89,7 +89,7 @@ class FolderTrashQueryParams(RemoveQueryParams): ... -class _FolderWorkspacesPathParams(BaseModel): +class FolderWorkspacesPathParams(BaseModel): folder_id: FolderID workspace_id: Annotated[ WorkspaceID | None, BeforeValidator(null_or_none_str_to_none_validator) diff --git a/services/web/server/src/simcore_service_webserver/folders/_workspaces_rest.py b/services/web/server/src/simcore_service_webserver/folders/_workspaces_rest.py index 2f6b8b1cda24..b327e84e5747 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_workspaces_rest.py +++ b/services/web/server/src/simcore_service_webserver/folders/_workspaces_rest.py @@ -9,7 +9,7 @@ from ..security.decorators import permission_required from . import _workspaces_repository from ._common.exceptions_handlers import handle_plugin_requests_exceptions -from ._common.models import FoldersRequestContext, _FolderWorkspacesPathParams +from ._common.models import FoldersRequestContext, FolderWorkspacesPathParams _logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ @handle_plugin_requests_exceptions async def move_folder_to_workspace(request: web.Request): req_ctx = FoldersRequestContext.model_validate(request) - path_params = parse_request_path_parameters_as(_FolderWorkspacesPathParams, request) + path_params = parse_request_path_parameters_as(FolderWorkspacesPathParams, request) await _workspaces_repository.move_folder_into_workspace( app=request.app, From 88cc7cbc194d1bd28b4f4d1c498c207f81e2c15f Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:54:59 +0100 Subject: [PATCH 49/71] rest model knows about domain model but not the opposite --- .../api_schemas_webserver/workspaces.py | 23 ++++++++--- .../workspaces/_workspaces_rest.py | 20 ++++----- .../workspaces/_workspaces_service.py | 41 ++++--------------- ...handlers__clone_in_workspace_and_folder.py | 7 ++-- 4 files changed, 39 insertions(+), 52 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py b/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py index de3b0640b981..8552f79308ee 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py @@ -1,13 +1,14 @@ from datetime import datetime -from typing import NamedTuple +from typing import Self from models_library.basic_types import IDStr from models_library.groups import GroupID from models_library.workspaces import WorkspaceID -from pydantic import ConfigDict, PositiveInt +from pydantic import ConfigDict from ..access_rights import AccessRights from ..users import UserID +from ..workspaces import UserWorkspaceAccessRightsDB, WorkspaceID from ._base import InputSchema, OutputSchema @@ -23,10 +24,20 @@ class WorkspaceGet(OutputSchema): my_access_rights: AccessRights access_rights: dict[GroupID, AccessRights] - -class WorkspaceGetPage(NamedTuple): - items: list[WorkspaceGet] - total: PositiveInt + @classmethod + def from_model(cls, workspace_db: UserWorkspaceAccessRightsDB) -> Self: + return cls( + workspace_id=workspace_db.workspace_id, + name=workspace_db.name, + description=workspace_db.description, + thumbnail=workspace_db.thumbnail, + created_at=workspace_db.created, + modified_at=workspace_db.modified, + trashed_at=workspace_db.trashed, + trashed_by=workspace_db.trashed_by if workspace_db.trashed else None, + my_access_rights=workspace_db.my_access_rights, + access_rights=workspace_db.access_rights, + ) class WorkspaceCreateBodyParams(InputSchema): diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py index 792df1f121f0..2fdd08f51ac3 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py @@ -4,12 +4,12 @@ from models_library.api_schemas_webserver.workspaces import ( WorkspaceCreateBodyParams, WorkspaceGet, - WorkspaceGetPage, WorkspaceReplaceBodyParams, ) from models_library.rest_ordering import OrderBy from models_library.rest_pagination import Page from models_library.rest_pagination_utils import paginate_data +from models_library.workspaces import UserWorkspaceAccessRightsDB from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -46,7 +46,7 @@ async def create_workspace(request: web.Request): req_ctx = WorkspacesRequestContext.model_validate(request) body_params = await parse_request_body_as(WorkspaceCreateBodyParams, request) - workspace: WorkspaceGet = await _workspaces_service.create_workspace( + workspace: UserWorkspaceAccessRightsDB = await _workspaces_service.create_workspace( request.app, user_id=req_ctx.user_id, name=body_params.name, @@ -55,7 +55,7 @@ async def create_workspace(request: web.Request): product_name=req_ctx.product_name, ) - return envelope_json_response(workspace, web.HTTPCreated) + return envelope_json_response(WorkspaceGet.from_model(workspace), web.HTTPCreated) @routes.get(f"/{VTAG}/workspaces", name="list_workspaces") @@ -72,7 +72,7 @@ async def list_workspaces(request: web.Request): query_params.filters = WorkspacesFilters() assert query_params.filters - workspaces: WorkspaceGetPage = await _workspaces_service.list_workspaces( + total_count, workspaces = await _workspaces_service.list_workspaces( app=request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name, @@ -85,9 +85,9 @@ async def list_workspaces(request: web.Request): page = Page[WorkspaceGet].model_validate( paginate_data( - chunk=workspaces.items, + chunk=[WorkspaceGet.from_model(w) for w in workspaces], request_url=request.url, - total=workspaces.total, + total=total_count, limit=query_params.limit, offset=query_params.offset, ) @@ -106,14 +106,14 @@ async def get_workspace(request: web.Request): req_ctx = WorkspacesRequestContext.model_validate(request) path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) - workspace: WorkspaceGet = await _workspaces_service.get_workspace( + workspace = await _workspaces_service.get_workspace( app=request.app, workspace_id=path_params.workspace_id, user_id=req_ctx.user_id, product_name=req_ctx.product_name, ) - return envelope_json_response(workspace) + return envelope_json_response(WorkspaceGet.from_model(workspace)) @routes.put( @@ -128,14 +128,14 @@ async def replace_workspace(request: web.Request): path_params = parse_request_path_parameters_as(WorkspacesPathParams, request) body_params = await parse_request_body_as(WorkspaceReplaceBodyParams, request) - workspace: WorkspaceGet = await _workspaces_service.update_workspace( + workspace = await _workspaces_service.update_workspace( app=request.app, user_id=req_ctx.user_id, workspace_id=path_params.workspace_id, product_name=req_ctx.product_name, **body_params.model_dump(), ) - return envelope_json_response(workspace) + return envelope_json_response(WorkspaceGet.from_model(workspace)) @routes.delete( diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py index 0be4a1bac4ab..3d271b6d746a 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py @@ -3,10 +3,6 @@ import logging from aiohttp import web -from models_library.api_schemas_webserver.workspaces import ( - WorkspaceGet, - WorkspaceGetPage, -) from models_library.products import ProductName from models_library.rest_ordering import OrderBy from models_library.users import UserID @@ -25,21 +21,6 @@ _logger = logging.getLogger(__name__) -def _to_api_model(workspace_db: UserWorkspaceAccessRightsDB) -> WorkspaceGet: - return WorkspaceGet( - workspace_id=workspace_db.workspace_id, - name=workspace_db.name, - description=workspace_db.description, - thumbnail=workspace_db.thumbnail, - created_at=workspace_db.created, - modified_at=workspace_db.modified, - trashed_at=workspace_db.trashed, - trashed_by=workspace_db.trashed_by if workspace_db.trashed else None, - my_access_rights=workspace_db.my_access_rights, - access_rights=workspace_db.access_rights, - ) - - async def create_workspace( app: web.Application, *, @@ -48,7 +29,7 @@ async def create_workspace( description: str | None, thumbnail: str | None, product_name: ProductName, -) -> WorkspaceGet: +) -> UserWorkspaceAccessRightsDB: user = await get_user(app, user_id=user_id) created_workspace_db = await db.create_workspace( @@ -59,13 +40,12 @@ async def create_workspace( description=description, thumbnail=thumbnail, ) - workspace_db = await db.get_workspace_for_user( + return await db.get_workspace_for_user( app, user_id=user_id, workspace_id=created_workspace_db.workspace_id, product_name=product_name, ) - return _to_api_model(workspace_db) async def get_workspace( @@ -74,15 +54,14 @@ async def get_workspace( user_id: UserID, workspace_id: WorkspaceID, product_name: ProductName, -) -> WorkspaceGet: - workspace_db = await check_user_workspace_access( +) -> UserWorkspaceAccessRightsDB: + return await check_user_workspace_access( app=app, user_id=user_id, workspace_id=workspace_id, product_name=product_name, permission="read", ) - return _to_api_model(workspace_db) async def list_workspaces( @@ -95,7 +74,7 @@ async def list_workspaces( offset: NonNegativeInt, limit: int, order_by: OrderBy, -) -> WorkspaceGetPage: +) -> tuple[int, list[UserWorkspaceAccessRightsDB]]: total_count, workspaces = await db.list_workspaces_for_user( app, user_id=user_id, @@ -107,10 +86,7 @@ async def list_workspaces( order_by=order_by, ) - return WorkspaceGetPage( - items=[_to_api_model(workspace_db) for workspace_db in workspaces], - total=total_count, - ) + return total_count, workspaces async def update_workspace( @@ -120,7 +96,7 @@ async def update_workspace( user_id: UserID, workspace_id: WorkspaceID, **updates, -) -> WorkspaceGet: +) -> UserWorkspaceAccessRightsDB: await check_user_workspace_access( app=app, @@ -135,13 +111,12 @@ async def update_workspace( product_name=product_name, updates=WorkspaceUpdateDB(**updates), ) - workspace_db = await db.get_workspace_for_user( + return await db.get_workspace_for_user( app, user_id=user_id, workspace_id=workspace_id, product_name=product_name, ) - return _to_api_model(workspace_db) async def delete_workspace( diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py index 42e9d2cf971d..8d302b4a3364 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers__clone_in_workspace_and_folder.py @@ -3,9 +3,8 @@ # pylint: disable=unused-argument # pylint: disable=unused-variable -from collections.abc import Iterator from copy import deepcopy -from typing import Any +from typing import Any, AsyncIterator import pytest import sqlalchemy as sa @@ -30,7 +29,9 @@ @pytest.fixture async def create_workspace_and_folder( client: TestClient, logged_user: UserInfoDict, postgres_db: sa.engine.Engine -) -> Iterator[tuple[WorkspaceID, FolderID]]: +) -> AsyncIterator[tuple[WorkspaceID, FolderID]]: + assert client.app + workspace = await create_workspace( client.app, user_id=logged_user["id"], From 344e4430abf015bb8e83909284d3d039361aa67a Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:28:29 +0100 Subject: [PATCH 50/71] fixes rename --- .../folders/_folders_repository.py | 6 ++++-- .../simcore_service_webserver/folders/_trash_service.py | 5 +++++ .../src/simcore_service_webserver/projects/models.py | 4 ++-- services/web/server/tests/unit/with_dbs/03/test_trash.py | 8 ++++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py index 98b6bd34fabe..590777f74c6b 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py @@ -275,7 +275,7 @@ async def get_for_user_or_workspace( *, folder_id: FolderID, product_name: ProductName, - user_id: UserID | None, + user_id: UserID | None, # owned workspace_id: WorkspaceID | None, ) -> FolderDB: assert not ( @@ -317,6 +317,7 @@ async def update( parent_folder_id: FolderID | None | UnSet = UnSet.VALUE, trashed: datetime | None | UnSet = UnSet.VALUE, trashed_explicitly: bool | UnSet = UnSet.VALUE, + trashed_by: UserID | UnSet = UnSet.VALUE, workspace_id: WorkspaceID | None | UnSet = UnSet.VALUE, user_id: UserID | None | UnSet = UnSet.VALUE, ) -> FolderDB: @@ -328,9 +329,10 @@ async def update( name=name, parent_folder_id=parent_folder_id, trashed=trashed, + trashed_by=trashed_by, # (who trashed) trashed_explicitly=trashed_explicitly, workspace_id=workspace_id, - user_id=user_id, + user_id=user_id, # (who owns) ) query = ( diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py index a7318e228e66..ba1c9f749201 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py +++ b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py @@ -63,6 +63,7 @@ async def _folders_db_update( product_name: ProductName, folder_id: FolderID, trashed_at: datetime | None, + trashed_by: UserID, ): # EXPLICIT un/trash await _folders_repository.update( @@ -72,6 +73,7 @@ async def _folders_db_update( product_name=product_name, trashed=trashed_at, trashed_explicitly=trashed_at is not None, + trashed_by=trashed_by, ) # IMPLICIT un/trash @@ -91,6 +93,7 @@ async def _folders_db_update( product_name=product_name, trashed=trashed_at, trashed_explicitly=False, + trashed_by=trashed_by, ) @@ -119,6 +122,7 @@ async def trash_folder( folder_id=folder_id, product_name=product_name, trashed_at=trashed_at, + trashed_by=user_id, ) # 2. Trash all child projects that I am an owner @@ -164,6 +168,7 @@ async def untrash_folder( folder_id=folder_id, product_name=product_name, trashed_at=None, + trashed_by=user_id, ) # 3.2 UNtrash all child projects that I am an owner diff --git a/services/web/server/src/simcore_service_webserver/projects/models.py b/services/web/server/src/simcore_service_webserver/projects/models.py index ee9321600a8f..b0378159a757 100644 --- a/services/web/server/src/simcore_service_webserver/projects/models.py +++ b/services/web/server/src/simcore_service_webserver/projects/models.py @@ -103,8 +103,8 @@ class ProjectPatchExtended(ProjectPatch): def to_model(self) -> dict[str, Any]: return remap_keys( - self.model_dump(exclude_unset=True, by_alias=False), - rename={"trashed_at": "trash"}, + self.model_dump(exclude_unset=True, by_alias=False, mode="json"), + rename={"trashed_at": "trashed"}, ) diff --git a/services/web/server/tests/unit/with_dbs/03/test_trash.py b/services/web/server/tests/unit/with_dbs/03/test_trash.py index 9080eb74fd82..6c38f65770d5 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_trash.py +++ b/services/web/server/tests/unit/with_dbs/03/test_trash.py @@ -76,20 +76,20 @@ async def test_trash_projects( # noqa: PLR0915 # this test should have no errors stopping services mock_remove_dynamic_services = mocker.patch( - "simcore_service_webserver.projects._trash_api.projects_api.remove_project_dynamic_services", + "simcore_service_webserver.projects._trash_service.projects_api.remove_project_dynamic_services", autospec=True, ) mock_stop_pipeline = mocker.patch( - "simcore_service_webserver.projects._trash_api.director_v2_api.stop_pipeline", + "simcore_service_webserver.projects._trash_service.director_v2_api.stop_pipeline", autospec=True, ) mocker.patch( - "simcore_service_webserver.projects._trash_api.director_v2_api.is_pipeline_running", + "simcore_service_webserver.projects._trash_service.director_v2_api.is_pipeline_running", return_value=is_project_running, autospec=True, ) mocker.patch( - "simcore_service_webserver.projects._trash_api.dynamic_scheduler_api.list_dynamic_services", + "simcore_service_webserver.projects._trash_service.dynamic_scheduler_api.list_dynamic_services", return_value=[mocker.MagicMock()] if is_project_running else [], autospec=True, ) From 80a09f14fd69290d450373ddf7175786476a3dcc Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:29:51 +0100 Subject: [PATCH 51/71] fixes export --- .../web/server/src/simcore_service_webserver/projects/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/models.py b/services/web/server/src/simcore_service_webserver/projects/models.py index b0378159a757..695c9ac030c9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/models.py +++ b/services/web/server/src/simcore_service_webserver/projects/models.py @@ -103,7 +103,7 @@ class ProjectPatchExtended(ProjectPatch): def to_model(self) -> dict[str, Any]: return remap_keys( - self.model_dump(exclude_unset=True, by_alias=False, mode="json"), + self.model_dump(exclude_unset=True, by_alias=False), rename={"trashed_at": "trashed"}, ) From e496bc3dc4b0219ab43d6725fe016c99a0748ed6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:32:15 +0100 Subject: [PATCH 52/71] tests trash pass --- .../src/simcore_service_webserver/projects/_crud_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index 09be85531976..7ffb4d46b0f1 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -362,7 +362,7 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.model_validate(project).data(exclude_unset=True) + data = ProjectGet.from_model(project).data(exclude_unset=True) return web.json_response({"data": data}, dumps=json_dumps) except ProjectInvalidRightsError as exc: From 4867970c7e0d80996afec266d82143215fce82e5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:44:21 +0100 Subject: [PATCH 53/71] rm redundance --- .../folders/_folders_rest.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py b/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py index b77829867a10..80da8bd21e83 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_rest.py @@ -8,7 +8,7 @@ FolderReplaceBodyParams, ) from models_library.rest_ordering import OrderBy -from models_library.rest_pagination import Page +from models_library.rest_pagination import ItemT, Page from models_library.rest_pagination_utils import paginate_data from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( @@ -39,6 +39,13 @@ routes = web.RouteTableDef() +def _create_json_response_from_page(page: Page[ItemT]): + return web.Response( + text=page.model_dump_json(**RESPONSE_MODEL_POLICY), + content_type=MIMETYPE_APPLICATION_JSON, + ) + + @routes.post(f"/{VTAG}/folders", name="create_folder") @login_required @permission_required("folder.create") @@ -93,10 +100,7 @@ async def list_folders(request: web.Request): offset=query_params.offset, ) ) - return web.Response( - text=page.model_dump_json(**RESPONSE_MODEL_POLICY), - content_type=MIMETYPE_APPLICATION_JSON, - ) + return _create_json_response_from_page(page) @routes.get(f"/{VTAG}/folders:search", name="list_folders_full_search") @@ -132,10 +136,7 @@ async def list_folders_full_search(request: web.Request): offset=query_params.offset, ) ) - return web.Response( - text=page.model_dump_json(**RESPONSE_MODEL_POLICY), - content_type=MIMETYPE_APPLICATION_JSON, - ) + return _create_json_response_from_page(page) @routes.get(f"/{VTAG}/folders/{{folder_id}}", name="get_folder") From 3a5d39b2104f2426e2815cb9d293b3081240802c Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:47:04 +0100 Subject: [PATCH 54/71] pylint --- .../simcore_service_webserver/folders/_folders_repository.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py index 590777f74c6b..a2ebbc386b24 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py @@ -307,6 +307,7 @@ async def get_for_user_or_workspace( async def update( + # pylint: disable=too-many-arguments app: web.Application, connection: AsyncConnection | None = None, *, @@ -317,9 +318,9 @@ async def update( parent_folder_id: FolderID | None | UnSet = UnSet.VALUE, trashed: datetime | None | UnSet = UnSet.VALUE, trashed_explicitly: bool | UnSet = UnSet.VALUE, - trashed_by: UserID | UnSet = UnSet.VALUE, + trashed_by: UserID | UnSet = UnSet.VALUE, # who trashed workspace_id: WorkspaceID | None | UnSet = UnSet.VALUE, - user_id: UserID | None | UnSet = UnSet.VALUE, + user_id: UserID | None | UnSet = UnSet.VALUE, # ownership ) -> FolderDB: """ Batch/single patch of folder/s From 08262d4a84d6d289f6a3f98605d0572c91d57ad4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 18:50:43 +0100 Subject: [PATCH 55/71] modifies fixtures to produce projects --- .../src/models_library/api_schemas_webserver/projects.py | 4 ++-- .../src/pytest_simcore/helpers/webserver_projects.py | 6 ++++++ .../tests/unit/with_dbs/02/test_projects_crud_handlers.py | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 63b7fd337fac..fe38a31733ca 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -106,10 +106,10 @@ class ProjectGet(OutputSchema): model_config = ConfigDict(frozen=False) @classmethod - def from_model(cls, project: dict[str, Any]) -> Self: + def from_model(cls, project_data: dict[str, Any]) -> Self: return cls.model_validate( remap_keys( - project, + project_data, rename={"trashed": "trashed_at"}, ) ) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py index 092ab82d6556..77515cffad5f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/webserver_projects.py @@ -12,6 +12,7 @@ from aiohttp import web from aiohttp.test_utils import TestClient +from common_library.dict_tools import remap_keys from models_library.projects_nodes_io import NodeID from models_library.services_resources import ServiceResourcesDictHelpers from simcore_postgres_database.utils_projects_nodes import ProjectNodeCreate @@ -113,6 +114,10 @@ async def create_project( for key in DB_EXCLUSIVE_COLUMNS: project_data.pop(key, None) + new_project: ProjectDict = remap_keys( + new_project, + rename={"trashed": "trashedAt"}, + ) return new_project @@ -163,6 +168,7 @@ async def __aenter__(self) -> ProjectDict: default_project_json=self.tests_data_dir / "fake-project.json", as_template=self.as_template, ) + return self.prj async def __aexit__(self, *args): diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py index dcf954d2b544..5345d7e41b86 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_crud_handlers.py @@ -208,6 +208,7 @@ async def test_list_projects( folder_id = got.pop("folderId") assert got == {k: template_project[k] for k in got} + assert not ProjectState( **project_state ).locked.value, "Templates are not locked" @@ -220,6 +221,7 @@ async def test_list_projects( folder_id = got.pop("folderId") assert got == {k: user_project[k] for k in got} + assert ProjectState(**project_state) assert project_permalink is None assert folder_id is None From 67f973e62e503764203bee4eebd66e482d8b82a5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Mon, 13 Jan 2025 19:01:12 +0100 Subject: [PATCH 56/71] note --- .../workspaces/_trash_services.py | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py index 667ca4841a7e..69644a2d56b5 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py @@ -59,7 +59,10 @@ async def trash_workspace( ) # IMPLICIT trash - child_folders: list[FolderID] = [] # TODO: find children. Check with MD + child_folders: list[FolderID] = ( + [] + # NOTE: follows up with https://github.com/ITISFoundation/osparc-simcore/issues/7034 + ) for folder_id in child_folders: await trash_folder( @@ -70,7 +73,10 @@ async def trash_workspace( force_stop_first=force_stop_first, ) - child_projects: list[ProjectID] = [] # TODO: find children. Check with MD + child_projects: list[ProjectID] = ( + [] + # NOTE: follows up with https://github.com/ITISFoundation/osparc-simcore/issues/7034 + ) for project_id in child_projects: await trash_project( @@ -104,7 +110,10 @@ async def untrash_workspace( updates=WorkspaceUpdateDB(trashed=None, trashed_by=None), ) - child_folders: list[FolderID] = [] # TODO: find children. Check with MD + child_folders: list[FolderID] = ( + [] + # NOTE: follows up with https://github.com/ITISFoundation/osparc-simcore/issues/7034 + ) for folder_id in child_folders: await untrash_folder( @@ -114,7 +123,10 @@ async def untrash_workspace( folder_id=folder_id, ) - child_projects: list[ProjectID] = [] # TODO: find children. Check with MD + child_projects: list[ProjectID] = ( + [] + # NOTE: follows up with https://github.com/ITISFoundation/osparc-simcore/issues/7034 + ) for project_id in child_projects: await untrash_project( From 12633e230e1cef26a26b1e9e3a827154a74b9da5 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:45:57 +0100 Subject: [PATCH 57/71] workspaces --- .../api_schemas_webserver/workspaces.py | 29 +++---- .../src/models_library/workspaces.py | 25 +++--- .../projects/_crud_api_create.py | 20 ++--- .../projects/_tags_api.py | 12 ++- .../projects/projects_api.py | 6 +- .../workspaces/_groups_service.py | 20 +++-- .../workspaces/_trash_services.py | 6 +- .../workspaces/_workspaces_repository.py | 30 +++---- .../workspaces/_workspaces_rest.py | 26 +++--- .../workspaces/_workspaces_service.py | 86 +++++++++++++------ .../workspaces/api.py | 3 +- 11 files changed, 152 insertions(+), 111 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py b/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py index 8552f79308ee..431fdd52da3c 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py @@ -1,14 +1,13 @@ from datetime import datetime from typing import Self -from models_library.basic_types import IDStr -from models_library.groups import GroupID -from models_library.workspaces import WorkspaceID from pydantic import ConfigDict from ..access_rights import AccessRights +from ..basic_types import IDStr +from ..groups import GroupID from ..users import UserID -from ..workspaces import UserWorkspaceAccessRightsDB, WorkspaceID +from ..workspaces import UserWorkspaceWithAccessRights, WorkspaceID from ._base import InputSchema, OutputSchema @@ -25,18 +24,18 @@ class WorkspaceGet(OutputSchema): access_rights: dict[GroupID, AccessRights] @classmethod - def from_model(cls, workspace_db: UserWorkspaceAccessRightsDB) -> Self: + def from_domain(cls, wks: UserWorkspaceWithAccessRights) -> Self: return cls( - workspace_id=workspace_db.workspace_id, - name=workspace_db.name, - description=workspace_db.description, - thumbnail=workspace_db.thumbnail, - created_at=workspace_db.created, - modified_at=workspace_db.modified, - trashed_at=workspace_db.trashed, - trashed_by=workspace_db.trashed_by if workspace_db.trashed else None, - my_access_rights=workspace_db.my_access_rights, - access_rights=workspace_db.access_rights, + workspace_id=wks.workspace_id, + name=wks.name, + description=wks.description, + thumbnail=wks.thumbnail, + created_at=wks.created, + modified_at=wks.modified, + trashed_at=wks.trashed, + trashed_by=wks.trashed_by if wks.trashed else None, + my_access_rights=wks.my_access_rights, + access_rights=wks.access_rights, ) diff --git a/packages/models-library/src/models_library/workspaces.py b/packages/models-library/src/models_library/workspaces.py index f6f33061bfe6..01f66685fa12 100644 --- a/packages/models-library/src/models_library/workspaces.py +++ b/packages/models-library/src/models_library/workspaces.py @@ -31,7 +31,7 @@ class WorkspaceQuery(BaseModel): @field_validator("workspace_id", mode="before") @classmethod - def validate_workspace_id(cls, value, info: ValidationInfo): + def _validate_workspace_id(cls, value, info: ValidationInfo): scope = info.data.get("workspace_scope") if scope == WorkspaceScope.SHARED and value is None: msg = f"workspace_id must be provided when workspace_scope is SHARED. Got {scope=}, {value=}" @@ -43,12 +43,7 @@ def validate_workspace_id(cls, value, info: ValidationInfo): return value -# -# DB -# - - -class WorkspaceDB(BaseModel): +class Workspace(BaseModel): workspace_id: WorkspaceID name: str description: str | None @@ -71,16 +66,16 @@ class WorkspaceDB(BaseModel): model_config = ConfigDict(from_attributes=True) -class UserWorkspaceAccessRightsDB(WorkspaceDB): - my_access_rights: AccessRights - access_rights: dict[GroupID, AccessRights] - - model_config = ConfigDict(from_attributes=True) - - -class WorkspaceUpdateDB(BaseModel): +class WorkspaceUpdates(BaseModel): name: str | None = None description: str | None = None thumbnail: str | None = None trashed: datetime | None = None trashed_by: UserID | None = None + + +class UserWorkspaceWithAccessRights(Workspace): + my_access_rights: AccessRights + access_rights: dict[GroupID, AccessRights] + + model_config = ConfigDict(from_attributes=True) diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 29a1d215c97a..bdd8364c9ee9 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -14,7 +14,7 @@ from models_library.projects_state import ProjectStatus from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder -from models_library.workspaces import UserWorkspaceAccessRightsDB +from models_library.workspaces import UserWorkspaceWithAccessRights from pydantic import TypeAdapter from servicelib.aiohttp.long_running_tasks.server import TaskProgress from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON @@ -34,8 +34,7 @@ get_project_total_size_simcore_s3, ) from ..users.api import get_user_fullname -from ..workspaces import _workspaces_repository as workspaces_db -from ..workspaces.api import check_user_workspace_access +from ..workspaces.api import check_user_workspace_access, get_user_workspace from ..workspaces.errors import WorkspaceAccessForbiddenError from . import _folders_db as project_to_folders_db from . import projects_api @@ -409,17 +408,16 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche # Overwrite project access rights if workspace_id: - workspace_db: UserWorkspaceAccessRightsDB = ( - await workspaces_db.get_workspace_for_user( - app=request.app, - user_id=user_id, - workspace_id=workspace_id, - product_name=product_name, - ) + workspace: UserWorkspaceWithAccessRights = await get_user_workspace( + request.app, + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + permission=None, ) new_project["accessRights"] = { f"{gid}": access.model_dump() - for gid, access in workspace_db.access_rights.items() + for gid, access in workspace.access_rights.items() } # Ensures is like ProjectGet diff --git a/services/web/server/src/simcore_service_webserver/projects/_tags_api.py b/services/web/server/src/simcore_service_webserver/projects/_tags_api.py index 1fa9bdcb5900..93bf232706ba 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_tags_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/_tags_api.py @@ -7,7 +7,7 @@ from aiohttp import web from models_library.projects import ProjectID from models_library.users import UserID -from models_library.workspaces import UserWorkspaceAccessRightsDB +from models_library.workspaces import UserWorkspaceWithAccessRights from ..workspaces import _workspaces_repository as workspaces_db from ._access_rights_api import check_user_project_permission @@ -36,7 +36,7 @@ async def add_tag( ) if project["workspaceId"] is not None: - workspace_db: UserWorkspaceAccessRightsDB = ( + workspace: UserWorkspaceWithAccessRights = ( await workspaces_db.get_workspace_for_user( app=app, user_id=user_id, @@ -45,8 +45,7 @@ async def add_tag( ) ) project["accessRights"] = { - gid: access.model_dump() - for gid, access in workspace_db.access_rights.items() + gid: access.model_dump() for gid, access in workspace.access_rights.items() } return project @@ -71,7 +70,7 @@ async def remove_tag( ) if project["workspaceId"] is not None: - workspace_db: UserWorkspaceAccessRightsDB = ( + workspace: UserWorkspaceWithAccessRights = ( await workspaces_db.get_workspace_for_user( app=app, user_id=user_id, @@ -80,8 +79,7 @@ async def remove_tag( ) ) project["accessRights"] = { - gid: access.model_dump() - for gid, access in workspace_db.access_rights.items() + gid: access.model_dump() for gid, access in workspace.access_rights.items() } return project diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 99c827943f1c..a4b54085c75a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -63,7 +63,7 @@ from models_library.users import UserID from models_library.utils.fastapi_encoders import jsonable_encoder from models_library.wallets import ZERO_CREDITS, WalletID, WalletInfo -from models_library.workspaces import UserWorkspaceAccessRightsDB +from models_library.workspaces import UserWorkspaceWithAccessRights from pydantic import ByteSize, TypeAdapter from servicelib.aiohttp.application_keys import APP_FIRE_AND_FORGET_TASKS_KEY from servicelib.common_headers import ( @@ -208,7 +208,7 @@ async def get_project_for_user( ) if project["workspaceId"] is not None: - workspace_db: UserWorkspaceAccessRightsDB = ( + workspace: UserWorkspaceWithAccessRights = ( await workspaces_db.get_workspace_for_user( app=app, user_id=user_id, @@ -218,7 +218,7 @@ async def get_project_for_user( ) project["accessRights"] = { f"{gid}": access.model_dump() - for gid, access in workspace_db.access_rights.items() + for gid, access in workspace.access_rights.items() } Project.model_validate(project) # NOTE: only validates diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py b/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py index 5ff4449a57a4..37737e735903 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_groups_service.py @@ -5,7 +5,7 @@ from models_library.groups import GroupID from models_library.products import ProductName from models_library.users import UserID -from models_library.workspaces import UserWorkspaceAccessRightsDB, WorkspaceID +from models_library.workspaces import UserWorkspaceWithAccessRights, WorkspaceID from pydantic import BaseModel, ConfigDict from ..users import api as users_api @@ -124,8 +124,13 @@ async def update_workspace_group( delete: bool, product_name: ProductName, ) -> WorkspaceGroupGet: - workspace: UserWorkspaceAccessRightsDB = await workspaces_db.get_workspace_for_user( - app=app, user_id=user_id, workspace_id=workspace_id, product_name=product_name + workspace: UserWorkspaceWithAccessRights = ( + await workspaces_db.get_workspace_for_user( + app=app, + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + ) ) if workspace.my_access_rights.write is False: raise WorkspaceAccessForbiddenError( @@ -165,8 +170,13 @@ async def delete_workspace_group( product_name: ProductName, ) -> None: user: dict = await users_api.get_user(app, user_id=user_id) - workspace: UserWorkspaceAccessRightsDB = await workspaces_db.get_workspace_for_user( - app=app, user_id=user_id, workspace_id=workspace_id, product_name=product_name + workspace: UserWorkspaceWithAccessRights = ( + await workspaces_db.get_workspace_for_user( + app=app, + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + ) ) if user["primary_gid"] != group_id and workspace.my_access_rights.delete is False: raise WorkspaceAccessForbiddenError( diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py index 69644a2d56b5..59f88f7ad908 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_services.py @@ -6,7 +6,7 @@ from models_library.products import ProductName from models_library.projects import ProjectID from models_library.users import UserID -from models_library.workspaces import WorkspaceID, WorkspaceUpdateDB +from models_library.workspaces import WorkspaceID, WorkspaceUpdates from simcore_postgres_database.utils_repos import transaction_context from ..db.plugin import get_asyncpg_engine @@ -55,7 +55,7 @@ async def trash_workspace( connection, product_name=product_name, workspace_id=workspace_id, - updates=WorkspaceUpdateDB(trashed=trashed_at, trashed_by=user_id), + updates=WorkspaceUpdates(trashed=trashed_at, trashed_by=user_id), ) # IMPLICIT trash @@ -107,7 +107,7 @@ async def untrash_workspace( connection, product_name=product_name, workspace_id=workspace_id, - updates=WorkspaceUpdateDB(trashed=None, trashed_by=None), + updates=WorkspaceUpdates(trashed=None, trashed_by=None), ) child_folders: list[FolderID] = ( diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py index b88bdd918ae4..63907713bccc 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py @@ -13,10 +13,10 @@ from models_library.rest_ordering import OrderBy, OrderDirection from models_library.users import UserID from models_library.workspaces import ( - UserWorkspaceAccessRightsDB, - WorkspaceDB, + UserWorkspaceWithAccessRights, + Workspace, WorkspaceID, - WorkspaceUpdateDB, + WorkspaceUpdates, ) from pydantic import NonNegativeInt from simcore_postgres_database.models.workspaces import workspaces @@ -52,8 +52,8 @@ workspaces.c.trashed_by, ) -assert set(WorkspaceDB.model_fields) == {c.name for c in _SELECTION_ARGS} # nosec -assert set(WorkspaceUpdateDB.model_fields).issubset( # nosec +assert set(Workspace.model_fields) == {c.name for c in _SELECTION_ARGS} # nosec +assert set(WorkspaceUpdates.model_fields).issubset( # nosec c.name for c in workspaces.columns ) @@ -67,7 +67,7 @@ async def create_workspace( name: str, description: str | None, thumbnail: str | None, -) -> WorkspaceDB: +) -> Workspace: async with transaction_context(get_asyncpg_engine(app), connection) as conn: result = await conn.stream( workspaces.insert() @@ -83,7 +83,7 @@ async def create_workspace( .returning(*_SELECTION_ARGS) ) row = await result.first() - return WorkspaceDB.model_validate(row) + return Workspace.model_validate(row) _access_rights_subquery = ( @@ -119,7 +119,7 @@ async def list_workspaces_for_user( offset: NonNegativeInt, limit: NonNegativeInt, order_by: OrderBy, -) -> tuple[int, list[UserWorkspaceAccessRightsDB]]: +) -> tuple[int, list[UserWorkspaceWithAccessRights]]: my_access_rights_subquery = create_my_workspace_access_rights_subquery( user_id=user_id ) @@ -163,8 +163,8 @@ async def list_workspaces_for_user( total_count = await conn.scalar(count_query) result = await conn.stream(list_query) - items: list[UserWorkspaceAccessRightsDB] = [ - UserWorkspaceAccessRightsDB.model_validate(row) async for row in result + items: list[UserWorkspaceWithAccessRights] = [ + UserWorkspaceWithAccessRights.model_validate(row) async for row in result ] return cast(int, total_count), items @@ -177,7 +177,7 @@ async def get_workspace_for_user( user_id: UserID, workspace_id: WorkspaceID, product_name: ProductName, -) -> UserWorkspaceAccessRightsDB: +) -> UserWorkspaceWithAccessRights: my_access_rights_subquery = create_my_workspace_access_rights_subquery( user_id=user_id ) @@ -204,7 +204,7 @@ async def get_workspace_for_user( raise WorkspaceAccessForbiddenError( reason=f"User {user_id} does not have access to the workspace {workspace_id}. Or workspace does not exist.", ) - return UserWorkspaceAccessRightsDB.model_validate(row) + return UserWorkspaceWithAccessRights.model_validate(row) async def update_workspace( @@ -213,8 +213,8 @@ async def update_workspace( *, product_name: ProductName, workspace_id: WorkspaceID, - updates: WorkspaceUpdateDB, -) -> WorkspaceDB: + updates: WorkspaceUpdates, +) -> Workspace: # NOTE: at least 'touch' if updated_values is empty _updates = { **updates.model_dump(exclude_unset=True), @@ -234,7 +234,7 @@ async def update_workspace( row = await result.first() if row is None: raise WorkspaceNotFoundError(reason=f"Workspace {workspace_id} not found.") - return WorkspaceDB.model_validate(row) + return Workspace.model_validate(row) async def delete_workspace( diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py index 2fdd08f51ac3..d11ecf454a6d 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py @@ -9,7 +9,7 @@ from models_library.rest_ordering import OrderBy from models_library.rest_pagination import Page from models_library.rest_pagination_utils import paginate_data -from models_library.workspaces import UserWorkspaceAccessRightsDB +from models_library.workspaces import UserWorkspaceWithAccessRights from servicelib.aiohttp import status from servicelib.aiohttp.requests_validation import ( parse_request_body_as, @@ -46,16 +46,18 @@ async def create_workspace(request: web.Request): req_ctx = WorkspacesRequestContext.model_validate(request) body_params = await parse_request_body_as(WorkspaceCreateBodyParams, request) - workspace: UserWorkspaceAccessRightsDB = await _workspaces_service.create_workspace( - request.app, - user_id=req_ctx.user_id, - name=body_params.name, - description=body_params.description, - thumbnail=body_params.thumbnail, - product_name=req_ctx.product_name, + workspace: UserWorkspaceWithAccessRights = ( + await _workspaces_service.create_workspace( + request.app, + user_id=req_ctx.user_id, + name=body_params.name, + description=body_params.description, + thumbnail=body_params.thumbnail, + product_name=req_ctx.product_name, + ) ) - return envelope_json_response(WorkspaceGet.from_model(workspace), web.HTTPCreated) + return envelope_json_response(WorkspaceGet.from_domain(workspace), web.HTTPCreated) @routes.get(f"/{VTAG}/workspaces", name="list_workspaces") @@ -85,7 +87,7 @@ async def list_workspaces(request: web.Request): page = Page[WorkspaceGet].model_validate( paginate_data( - chunk=[WorkspaceGet.from_model(w) for w in workspaces], + chunk=[WorkspaceGet.from_domain(w) for w in workspaces], request_url=request.url, total=total_count, limit=query_params.limit, @@ -113,7 +115,7 @@ async def get_workspace(request: web.Request): product_name=req_ctx.product_name, ) - return envelope_json_response(WorkspaceGet.from_model(workspace)) + return envelope_json_response(WorkspaceGet.from_domain(workspace)) @routes.put( @@ -135,7 +137,7 @@ async def replace_workspace(request: web.Request): product_name=req_ctx.product_name, **body_params.model_dump(), ) - return envelope_json_response(WorkspaceGet.from_model(workspace)) + return envelope_json_response(WorkspaceGet.from_domain(workspace)) @routes.delete( diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py index 3d271b6d746a..721a01dd76ed 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py @@ -7,9 +7,9 @@ from models_library.rest_ordering import OrderBy from models_library.users import UserID from models_library.workspaces import ( - UserWorkspaceAccessRightsDB, + UserWorkspaceWithAccessRights, WorkspaceID, - WorkspaceUpdateDB, + WorkspaceUpdates, ) from pydantic import NonNegativeInt @@ -29,10 +29,9 @@ async def create_workspace( description: str | None, thumbnail: str | None, product_name: ProductName, -) -> UserWorkspaceAccessRightsDB: +) -> UserWorkspaceWithAccessRights: user = await get_user(app, user_id=user_id) - - created_workspace_db = await db.create_workspace( + created = await db.create_workspace( app, product_name=product_name, owner_primary_gid=user["primary_gid"], @@ -43,7 +42,7 @@ async def create_workspace( return await db.get_workspace_for_user( app, user_id=user_id, - workspace_id=created_workspace_db.workspace_id, + workspace_id=created.workspace_id, product_name=product_name, ) @@ -54,8 +53,8 @@ async def get_workspace( user_id: UserID, workspace_id: WorkspaceID, product_name: ProductName, -) -> UserWorkspaceAccessRightsDB: - return await check_user_workspace_access( +) -> UserWorkspaceWithAccessRights: + return await get_user_workspace( app=app, user_id=user_id, workspace_id=workspace_id, @@ -74,7 +73,7 @@ async def list_workspaces( offset: NonNegativeInt, limit: int, order_by: OrderBy, -) -> tuple[int, list[UserWorkspaceAccessRightsDB]]: +) -> tuple[int, list[UserWorkspaceWithAccessRights]]: total_count, workspaces = await db.list_workspaces_for_user( app, user_id=user_id, @@ -96,7 +95,7 @@ async def update_workspace( user_id: UserID, workspace_id: WorkspaceID, **updates, -) -> UserWorkspaceAccessRightsDB: +) -> UserWorkspaceWithAccessRights: await check_user_workspace_access( app=app, @@ -109,7 +108,7 @@ async def update_workspace( app, workspace_id=workspace_id, product_name=product_name, - updates=WorkspaceUpdateDB(**updates), + updates=WorkspaceUpdates(**updates), ) return await db.get_workspace_for_user( app, @@ -134,28 +133,67 @@ async def delete_workspace( permission="delete", ) - await db.delete_workspace(app, workspace_id=workspace_id, product_name=product_name) + await db.delete_workspace( + app, + workspace_id=workspace_id, + product_name=product_name, + ) -async def check_user_workspace_access( +async def get_user_workspace( app: web.Application, *, user_id: UserID, workspace_id: WorkspaceID, product_name: ProductName, - permission: PermissionStr = "read", -) -> UserWorkspaceAccessRightsDB: + permission: PermissionStr | None, +) -> UserWorkspaceWithAccessRights: """ - Raises WorkspaceAccessForbiddenError if no access + + Here checking access is optional. A use case is when the caller has guarantees that + `user_id` has granted access and we do not want to re-check + + Raises: + WorkspaceAccessForbiddenError: if permission not None and user_id does not have access """ - workspace_db: UserWorkspaceAccessRightsDB = await db.get_workspace_for_user( + workspace: UserWorkspaceWithAccessRights = await db.get_workspace_for_user( app=app, user_id=user_id, workspace_id=workspace_id, product_name=product_name ) - if getattr(workspace_db.my_access_rights, permission, False) is False: - raise WorkspaceAccessForbiddenError( - user_id=user_id, - workspace_id=workspace_id, - product_name=product_name, - permission_checked=permission, + + # NOTE: check here is optional + if permission is not None: + has_user_granted_permission = getattr( + workspace.my_access_rights, permission, False ) - return workspace_db + if not has_user_granted_permission: + raise WorkspaceAccessForbiddenError( + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + permission_checked=permission, + ) + return workspace + + +async def check_user_workspace_access( + app: web.Application, + *, + user_id: UserID, + workspace_id: WorkspaceID, + product_name: ProductName, + permission: PermissionStr, +) -> UserWorkspaceWithAccessRights: + """ + As `get_user_workspace` but here check is required + + Raises: + WorkspaceAccessForbiddenError + """ + return await get_user_workspace( + app, + user_id=user_id, + workspace_id=workspace_id, + product_name=product_name, + # NOTE: check here is required + permission=permission, + ) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/api.py b/services/web/server/src/simcore_service_webserver/workspaces/api.py index 7a0e2d338bc5..c0f065d6dd64 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/api.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/api.py @@ -5,6 +5,7 @@ assert check_user_workspace_access # nosec __all__: tuple[str, ...] = ( - "get_workspace", "check_user_workspace_access", + "get_user_workspace", + "get_workspace", ) From fbf52e24fb3bd60699aaf9833221c70863e12995 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:51:50 +0100 Subject: [PATCH 58/71] api --- .../workspaces/_workspaces_service.py | 2 +- .../src/simcore_service_webserver/workspaces/api.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py index 721a01dd76ed..e87dc72d054e 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py @@ -184,7 +184,7 @@ async def check_user_workspace_access( permission: PermissionStr, ) -> UserWorkspaceWithAccessRights: """ - As `get_user_workspace` but here check is required + As `get_user_workspace` but here access check is required Raises: WorkspaceAccessForbiddenError diff --git a/services/web/server/src/simcore_service_webserver/workspaces/api.py b/services/web/server/src/simcore_service_webserver/workspaces/api.py index c0f065d6dd64..117d8d6d43d4 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/api.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/api.py @@ -1,11 +1,15 @@ # mypy: disable-error-code=truthy-function -from ._workspaces_service import check_user_workspace_access, get_workspace -assert get_workspace # nosec -assert check_user_workspace_access # nosec +from ._workspaces_service import ( + check_user_workspace_access, + get_user_workspace, + get_workspace, +) __all__: tuple[str, ...] = ( "check_user_workspace_access", "get_user_workspace", "get_workspace", ) + +# nopycln: file From 1cb104c827a1f464d33c7e280800f9eea594f167 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:55:33 +0100 Subject: [PATCH 59/71] ProjectGet.from_domain --- .../src/models_library/api_schemas_webserver/projects.py | 2 +- .../simcore_service_webserver/projects/_crud_api_create.py | 2 +- .../src/simcore_service_webserver/projects/_crud_api_read.py | 2 +- .../src/simcore_service_webserver/projects/_crud_handlers.py | 4 ++-- services/web/server/tests/conftest.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index fe38a31733ca..c312669f2d3c 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -106,7 +106,7 @@ class ProjectGet(OutputSchema): model_config = ConfigDict(frozen=False) @classmethod - def from_model(cls, project_data: dict[str, Any]) -> Self: + def from_domain(cls, project_data: dict[str, Any]) -> Self: return cls.model_validate( remap_keys( project_data, diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index bdd8364c9ee9..0d2663a375fb 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -421,7 +421,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche } # Ensures is like ProjectGet - data = ProjectGet.from_model(new_project).data(exclude_unset=True) + data = ProjectGet.from_domain(new_project).data(exclude_unset=True) raise web.HTTPCreated( text=json_dumps({"data": data}), diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index fea86ef0a2f7..08799336b3c6 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -45,7 +45,7 @@ async def _append_item( await update_or_pop_permalink_in_project(request, project) # validate - return ProjectListItem.from_model(project).data(exclude_unset=True) + return ProjectListItem.from_domain(project).data(exclude_unset=True) async def list_projects( # pylint: disable=too-many-arguments diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index 7ffb4d46b0f1..a00c4a2ba2db 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -305,7 +305,7 @@ async def get_active_project(request: web.Request) -> web.Response: # updates project's permalink field await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.model_validate(project).data(exclude_unset=True) + data = ProjectGet.from_domain(project).data(exclude_unset=True) return web.json_response({"data": data}, dumps=json_dumps) @@ -362,7 +362,7 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_model(project).data(exclude_unset=True) + data = ProjectGet.from_domain(project).data(exclude_unset=True) return web.json_response({"data": data}, dumps=json_dumps) except ProjectInvalidRightsError as exc: diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index 0f8ab3e09c4e..012a6d60a68d 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -242,7 +242,7 @@ async def _setup( if not as_template: expected_data["name"] = f"{from_study['name']} (Copy)" - expected_data = ProjectGet.from_model(expected_data).model_dump( + expected_data = ProjectGet.from_domain(expected_data).model_dump( mode="json", by_alias=True ) From 28ff2fab0ac4222442d0740ae3b45f7a964038ae Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:58:25 +0100 Subject: [PATCH 60/71] from_domain_model --- .../api_schemas_webserver/groups.py | 16 +++++++------ .../api_schemas_webserver/projects.py | 2 +- .../api_schemas_webserver/users.py | 10 ++++---- .../api_schemas_webserver/workspaces.py | 2 +- packages/models-library/tests/test_users.py | 2 +- .../simcore_webserver_groups_fixtures.py | 2 +- .../groups/_groups_rest.py | 24 +++++++++++-------- .../projects/_crud_api_create.py | 2 +- .../projects/_crud_api_read.py | 2 +- .../projects/_crud_handlers.py | 4 ++-- .../users/_notifications_rest.py | 2 +- .../users/_tokens_rest.py | 6 ++--- .../users/_users_rest.py | 4 ++-- .../workspaces/_workspaces_rest.py | 10 ++++---- services/web/server/tests/conftest.py | 2 +- .../tests/unit/isolated/test_groups_models.py | 4 ++-- 16 files changed, 51 insertions(+), 43 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/groups.py b/packages/models-library/src/models_library/api_schemas_webserver/groups.py index ec9738044b41..01bb310dd278 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/groups.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/groups.py @@ -73,7 +73,7 @@ class GroupGet(OutputSchema): ] = DEFAULT_FACTORY @classmethod - def from_model(cls, group: Group, access_rights: AccessRightsDict) -> Self: + def from_domain_model(cls, group: Group, access_rights: AccessRightsDict) -> Self: # Adapts these domain models into this schema return cls.model_validate( { @@ -230,7 +230,7 @@ class MyGroupsGet(OutputSchema): ) @classmethod - def from_model( + def from_domain_model( cls, groups_by_type: GroupsByTypeTuple, my_product_group: tuple[Group, AccessRightsDict] | None, @@ -239,10 +239,12 @@ def from_model( assert groups_by_type.everyone # nosec return cls( - me=GroupGet.from_model(*groups_by_type.primary), - organizations=[GroupGet.from_model(*gi) for gi in groups_by_type.standard], - all=GroupGet.from_model(*groups_by_type.everyone), - product=GroupGet.from_model(*my_product_group) + me=GroupGet.from_domain_model(*groups_by_type.primary), + organizations=[ + GroupGet.from_domain_model(*gi) for gi in groups_by_type.standard + ], + all=GroupGet.from_domain_model(*groups_by_type.everyone), + product=GroupGet.from_domain_model(*my_product_group) if my_product_group else None, ) @@ -320,7 +322,7 @@ class GroupUserGet(OutputSchemaWithoutCamelCase): ) @classmethod - def from_model(cls, user: GroupMember) -> Self: + def from_domain_model(cls, user: GroupMember) -> Self: return cls.model_validate( { "id": user.id, diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index c312669f2d3c..769350449d29 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -106,7 +106,7 @@ class ProjectGet(OutputSchema): model_config = ConfigDict(frozen=False) @classmethod - def from_domain(cls, project_data: dict[str, Any]) -> Self: + def from_domain_model(cls, project_data: dict[str, Any]) -> Self: return cls.model_validate( remap_keys( project_data, diff --git a/packages/models-library/src/models_library/api_schemas_webserver/users.py b/packages/models-library/src/models_library/api_schemas_webserver/users.py index f5f49bf726cb..bdaf9bbe5f6a 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/users.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/users.py @@ -108,7 +108,7 @@ def _to_upper_string(cls, v): return v @classmethod - def from_model( + def from_domain_model( cls, my_profile: MyProfile, my_groups_by_type: GroupsByTypeTuple, @@ -133,7 +133,7 @@ def from_model( ) return cls( **data, - groups=MyGroupsGet.from_model(my_groups_by_type, my_product_group), + groups=MyGroupsGet.from_domain_model(my_groups_by_type, my_product_group), preferences=my_preferences, ) @@ -221,7 +221,7 @@ class UserGet(OutputSchema): email: EmailStr | None = None @classmethod - def from_model(cls, data): + def from_domain_model(cls, data): return cls.model_validate(data, from_attributes=True) @@ -309,7 +309,7 @@ class MyTokenGet(OutputSchemaWithoutCamelCase): ] = None @classmethod - def from_model(cls, token: UserThirdPartyToken) -> Self: + def from_domain_model(cls, token: UserThirdPartyToken) -> Self: return cls( service=token.service, # type: ignore[arg-type] token_key=token.token_key, # type: ignore[arg-type] @@ -327,5 +327,5 @@ class MyPermissionGet(OutputSchema): allowed: bool @classmethod - def from_model(cls, permission: UserPermission) -> Self: + def from_domain_model(cls, permission: UserPermission) -> Self: return cls(name=permission.name, allowed=permission.allowed) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py b/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py index 431fdd52da3c..17762e9efea7 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/workspaces.py @@ -24,7 +24,7 @@ class WorkspaceGet(OutputSchema): access_rights: dict[GroupID, AccessRights] @classmethod - def from_domain(cls, wks: UserWorkspaceWithAccessRights) -> Self: + def from_domain_model(cls, wks: UserWorkspaceWithAccessRights) -> Self: return cls( workspace_id=wks.workspace_id, name=wks.name, diff --git a/packages/models-library/tests/test_users.py b/packages/models-library/tests/test_users.py index 97496e133a93..4c9d2756934e 100644 --- a/packages/models-library/tests/test_users.py +++ b/packages/models-library/tests/test_users.py @@ -22,6 +22,6 @@ def test_adapter_from_model_to_schema(): ) my_preferences = {"foo": Preference(default_value=3, value=1)} - MyProfileGet.from_model( + MyProfileGet.from_domain_model( my_profile, my_groups_by_type, my_product_group, my_preferences ) diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_groups_fixtures.py b/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_groups_fixtures.py index be032c8f6f4d..cc31177abcec 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_groups_fixtures.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_webserver_groups_fixtures.py @@ -28,7 +28,7 @@ def _groupget_model_dump(group, access_rights) -> dict[str, Any]: - return GroupGet.from_model(group, access_rights).model_dump( + return GroupGet.from_domain_model(group, access_rights).model_dump( mode="json", by_alias=True, exclude_unset=True, diff --git a/services/web/server/src/simcore_service_webserver/groups/_groups_rest.py b/services/web/server/src/simcore_service_webserver/groups/_groups_rest.py index 32b5e507382d..f9d33762b442 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_groups_rest.py +++ b/services/web/server/src/simcore_service_webserver/groups/_groups_rest.py @@ -67,10 +67,14 @@ async def list_groups(request: web.Request): ) my_groups = MyGroupsGet( - me=GroupGet.from_model(*groups_by_type.primary), - organizations=[GroupGet.from_model(*gi) for gi in groups_by_type.standard], - all=GroupGet.from_model(*groups_by_type.everyone), - product=GroupGet.from_model(*my_product_group) if my_product_group else None, + me=GroupGet.from_domain_model(*groups_by_type.primary), + organizations=[ + GroupGet.from_domain_model(*gi) for gi in groups_by_type.standard + ], + all=GroupGet.from_domain_model(*groups_by_type.everyone), + product=GroupGet.from_domain_model(*my_product_group) + if my_product_group + else None, ) return envelope_json_response(my_groups) @@ -94,7 +98,7 @@ async def get_group(request: web.Request): request.app, user_id=req_ctx.user_id, group_id=path_params.gid ) - return envelope_json_response(GroupGet.from_model(group, access_rights)) + return envelope_json_response(GroupGet.from_domain_model(group, access_rights)) @routes.post(f"/{API_VTAG}/groups", name="create_group") @@ -113,7 +117,7 @@ async def create_group(request: web.Request): create=create.to_model(), ) - created_group = GroupGet.from_model(group, access_rights) + created_group = GroupGet.from_domain_model(group, access_rights) return envelope_json_response(created_group, status_cls=web.HTTPCreated) @@ -134,7 +138,7 @@ async def update_group(request: web.Request): update=update.to_model(), ) - updated_group = GroupGet.from_model(group, access_rights) + updated_group = GroupGet.from_domain_model(group, access_rights) return envelope_json_response(updated_group) @@ -173,7 +177,7 @@ async def get_all_group_users(request: web.Request): ) return envelope_json_response( - [GroupUserGet.from_model(user) for user in users_in_group] + [GroupUserGet.from_domain_model(user) for user in users_in_group] ) @@ -216,7 +220,7 @@ async def get_group_user(request: web.Request): request.app, req_ctx.user_id, path_params.gid, path_params.uid ) - return envelope_json_response(GroupUserGet.from_model(user)) + return envelope_json_response(GroupUserGet.from_domain_model(user)) @routes.patch(f"/{API_VTAG}/groups/{{gid}}/users/{{uid}}", name="update_group_user") @@ -236,7 +240,7 @@ async def update_group_user(request: web.Request): access_rights=update.access_rights.model_dump(mode="json"), # type: ignore[arg-type] ) - return envelope_json_response(GroupUserGet.from_model(user)) + return envelope_json_response(GroupUserGet.from_domain_model(user)) @routes.delete(f"/{API_VTAG}/groups/{{gid}}/users/{{uid}}", name="delete_group_user") diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py index 0d2663a375fb..8b656f67cacd 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_create.py @@ -421,7 +421,7 @@ async def create_project( # pylint: disable=too-many-arguments,too-many-branche } # Ensures is like ProjectGet - data = ProjectGet.from_domain(new_project).data(exclude_unset=True) + data = ProjectGet.from_domain_model(new_project).data(exclude_unset=True) raise web.HTTPCreated( text=json_dumps({"data": data}), diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py index 08799336b3c6..3f9248e7237c 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_api_read.py @@ -45,7 +45,7 @@ async def _append_item( await update_or_pop_permalink_in_project(request, project) # validate - return ProjectListItem.from_domain(project).data(exclude_unset=True) + return ProjectListItem.from_domain_model(project).data(exclude_unset=True) async def list_projects( # pylint: disable=too-many-arguments diff --git a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py index a00c4a2ba2db..2ea0d84b8087 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_crud_handlers.py @@ -305,7 +305,7 @@ async def get_active_project(request: web.Request) -> web.Response: # updates project's permalink field await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain(project).data(exclude_unset=True) + data = ProjectGet.from_domain_model(project).data(exclude_unset=True) return web.json_response({"data": data}, dumps=json_dumps) @@ -362,7 +362,7 @@ async def get_project(request: web.Request): # Adds permalink await update_or_pop_permalink_in_project(request, project) - data = ProjectGet.from_domain(project).data(exclude_unset=True) + data = ProjectGet.from_domain_model(project).data(exclude_unset=True) return web.json_response({"data": data}, dumps=json_dumps) except ProjectInvalidRightsError as exc: diff --git a/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py b/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py index fb7e02b08da4..2e243d4da90e 100644 --- a/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py +++ b/services/web/server/src/simcore_service_webserver/users/_notifications_rest.py @@ -130,5 +130,5 @@ async def list_user_permissions(request: web.Request) -> web.Response: request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name ) return envelope_json_response( - [MyPermissionGet.from_model(p) for p in list_permissions] + [MyPermissionGet.from_domain_model(p) for p in list_permissions] ) diff --git a/services/web/server/src/simcore_service_webserver/users/_tokens_rest.py b/services/web/server/src/simcore_service_webserver/users/_tokens_rest.py index 64c971761a7b..0c11c9dcae72 100644 --- a/services/web/server/src/simcore_service_webserver/users/_tokens_rest.py +++ b/services/web/server/src/simcore_service_webserver/users/_tokens_rest.py @@ -46,7 +46,7 @@ async def _wrapper(request: web.Request) -> web.StreamResponse: async def list_tokens(request: web.Request) -> web.Response: req_ctx = UsersRequestContext.model_validate(request) all_tokens = await _tokens_service.list_tokens(request.app, req_ctx.user_id) - return envelope_json_response([MyTokenGet.from_model(t) for t in all_tokens]) + return envelope_json_response([MyTokenGet.from_domain_model(t) for t in all_tokens]) @routes.post(f"/{API_VTAG}/me/tokens", name="create_token") @@ -61,7 +61,7 @@ async def create_token(request: web.Request) -> web.Response: request.app, req_ctx.user_id, token_create.to_model() ) - return envelope_json_response(MyTokenGet.from_model(token), web.HTTPCreated) + return envelope_json_response(MyTokenGet.from_domain_model(token), web.HTTPCreated) class _TokenPathParams(BaseModel): @@ -80,7 +80,7 @@ async def get_token(request: web.Request) -> web.Response: request.app, req_ctx.user_id, req_path_params.service ) - return envelope_json_response(MyTokenGet.from_model(token)) + return envelope_json_response(MyTokenGet.from_domain_model(token)) @routes.delete(f"/{API_VTAG}/me/tokens/{{service}}", name="delete_token") diff --git a/services/web/server/src/simcore_service_webserver/users/_users_rest.py b/services/web/server/src/simcore_service_webserver/users/_users_rest.py index 688b024b40a7..d2ece6885148 100644 --- a/services/web/server/src/simcore_service_webserver/users/_users_rest.py +++ b/services/web/server/src/simcore_service_webserver/users/_users_rest.py @@ -106,7 +106,7 @@ async def get_my_profile(request: web.Request) -> web.Response: request.app, user_id=req_ctx.user_id, product_name=req_ctx.product_name ) - profile = MyProfileGet.from_model( + profile = MyProfileGet.from_domain_model( my_profile, groups_by_type, my_product_group, preferences ) @@ -153,7 +153,7 @@ async def search_users(request: web.Request) -> web.Response: limit=search_params.limit, ) - return envelope_json_response([UserGet.from_model(user) for user in found]) + return envelope_json_response([UserGet.from_domain_model(user) for user in found]) # diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py index d11ecf454a6d..eb40b338ef3c 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_rest.py @@ -57,7 +57,9 @@ async def create_workspace(request: web.Request): ) ) - return envelope_json_response(WorkspaceGet.from_domain(workspace), web.HTTPCreated) + return envelope_json_response( + WorkspaceGet.from_domain_model(workspace), web.HTTPCreated + ) @routes.get(f"/{VTAG}/workspaces", name="list_workspaces") @@ -87,7 +89,7 @@ async def list_workspaces(request: web.Request): page = Page[WorkspaceGet].model_validate( paginate_data( - chunk=[WorkspaceGet.from_domain(w) for w in workspaces], + chunk=[WorkspaceGet.from_domain_model(w) for w in workspaces], request_url=request.url, total=total_count, limit=query_params.limit, @@ -115,7 +117,7 @@ async def get_workspace(request: web.Request): product_name=req_ctx.product_name, ) - return envelope_json_response(WorkspaceGet.from_domain(workspace)) + return envelope_json_response(WorkspaceGet.from_domain_model(workspace)) @routes.put( @@ -137,7 +139,7 @@ async def replace_workspace(request: web.Request): product_name=req_ctx.product_name, **body_params.model_dump(), ) - return envelope_json_response(WorkspaceGet.from_domain(workspace)) + return envelope_json_response(WorkspaceGet.from_domain_model(workspace)) @routes.delete( diff --git a/services/web/server/tests/conftest.py b/services/web/server/tests/conftest.py index 012a6d60a68d..ab862056e038 100644 --- a/services/web/server/tests/conftest.py +++ b/services/web/server/tests/conftest.py @@ -242,7 +242,7 @@ async def _setup( if not as_template: expected_data["name"] = f"{from_study['name']} (Copy)" - expected_data = ProjectGet.from_domain(expected_data).model_dump( + expected_data = ProjectGet.from_domain_model(expected_data).model_dump( mode="json", by_alias=True ) diff --git a/services/web/server/tests/unit/isolated/test_groups_models.py b/services/web/server/tests/unit/isolated/test_groups_models.py index 2e5201422e9e..06729b3519a4 100644 --- a/services/web/server/tests/unit/isolated/test_groups_models.py +++ b/services/web/server/tests/unit/isolated/test_groups_models.py @@ -57,7 +57,7 @@ def test_output_schemas_from_models(faker: Faker): group_type=GroupType.STANDARD, thumbnail=None, ) - output_schema = GroupGet.from_model( + output_schema = GroupGet.from_domain_model( domain_model, access_rights=AccessRightsDict(read=True, write=False, delete=False), ) @@ -73,7 +73,7 @@ def test_output_schemas_from_models(faker: Faker): primary_gid=13, access_rights=AccessRightsDict(read=True, write=False, delete=False), ) - output_schema = GroupUserGet.from_model(user=domain_model) + output_schema = GroupUserGet.from_domain_model(user=domain_model) assert output_schema.user_name == domain_model.name From 8e96ffcbfaf04364dfd3d848bec1633e7203577d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Jan 2025 15:59:29 +0100 Subject: [PATCH 61/71] to_domain_model --- .../src/models_library/api_schemas_webserver/groups.py | 4 ++-- .../src/models_library/api_schemas_webserver/projects.py | 2 +- .../src/models_library/api_schemas_webserver/users.py | 2 +- .../src/simcore_service_webserver/groups/_groups_rest.py | 4 ++-- .../server/src/simcore_service_webserver/projects/models.py | 2 +- .../src/simcore_service_webserver/projects/projects_api.py | 2 +- .../src/simcore_service_webserver/users/_tokens_rest.py | 2 +- services/web/server/tests/unit/isolated/test_groups_models.py | 4 ++-- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/groups.py b/packages/models-library/src/models_library/api_schemas_webserver/groups.py index 01bb310dd278..4755e9c90af9 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/groups.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/groups.py @@ -151,7 +151,7 @@ class GroupCreate(InputSchema): description: str thumbnail: AnyUrl | None = None - def to_model(self) -> StandardGroupCreate: + def to_domain_model(self) -> StandardGroupCreate: data = remap_keys( self.model_dump( mode="json", @@ -169,7 +169,7 @@ class GroupUpdate(InputSchema): description: str | None = None thumbnail: AnyUrl | None = None - def to_model(self) -> StandardGroupUpdate: + def to_domain_model(self) -> StandardGroupUpdate: data = remap_keys( self.model_dump( mode="json", diff --git a/packages/models-library/src/models_library/api_schemas_webserver/projects.py b/packages/models-library/src/models_library/api_schemas_webserver/projects.py index 769350449d29..95b6b50805fa 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/projects.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/projects.py @@ -171,7 +171,7 @@ class ProjectPatch(InputSchema): ] = Field(default=None) quality: dict[str, Any] | None = Field(default=None) - def to_model(self) -> dict[str, Any]: + def to_domain_model(self) -> dict[str, Any]: return self.model_dump(exclude_unset=True, by_alias=False) diff --git a/packages/models-library/src/models_library/api_schemas_webserver/users.py b/packages/models-library/src/models_library/api_schemas_webserver/users.py index bdaf9bbe5f6a..0f6e010a4a85 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/users.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/users.py @@ -293,7 +293,7 @@ class MyTokenCreate(InputSchemaWithoutCamelCase): token_key: IDStr token_secret: IDStr - def to_model(self) -> UserThirdPartyToken: + def to_domain_model(self) -> UserThirdPartyToken: return UserThirdPartyToken( service=self.service, token_key=self.token_key, diff --git a/services/web/server/src/simcore_service_webserver/groups/_groups_rest.py b/services/web/server/src/simcore_service_webserver/groups/_groups_rest.py index f9d33762b442..5456776cfe6d 100644 --- a/services/web/server/src/simcore_service_webserver/groups/_groups_rest.py +++ b/services/web/server/src/simcore_service_webserver/groups/_groups_rest.py @@ -114,7 +114,7 @@ async def create_group(request: web.Request): group, access_rights = await _groups_service.create_standard_group( request.app, user_id=req_ctx.user_id, - create=create.to_model(), + create=create.to_domain_model(), ) created_group = GroupGet.from_domain_model(group, access_rights) @@ -135,7 +135,7 @@ async def update_group(request: web.Request): request.app, user_id=req_ctx.user_id, group_id=path_params.gid, - update=update.to_model(), + update=update.to_domain_model(), ) updated_group = GroupGet.from_domain_model(group, access_rights) diff --git a/services/web/server/src/simcore_service_webserver/projects/models.py b/services/web/server/src/simcore_service_webserver/projects/models.py index 695c9ac030c9..8354bdff549d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/models.py +++ b/services/web/server/src/simcore_service_webserver/projects/models.py @@ -101,7 +101,7 @@ class ProjectPatchExtended(ProjectPatch): model_config = ConfigDict(populate_by_name=True, extra="forbid") - def to_model(self) -> dict[str, Any]: + def to_domain_model(self) -> dict[str, Any]: return remap_keys( self.model_dump(exclude_unset=True, by_alias=False), rename={"trashed_at": "trashed"}, diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index a4b54085c75a..c015bbd51a5d 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -254,7 +254,7 @@ async def patch_project( project_patch: ProjectPatch | ProjectPatchExtended, product_name: ProductName, ): - patch_project_data = project_patch.to_model() + patch_project_data = project_patch.to_domain_model() db: ProjectDBAPI = app[APP_PROJECT_DBAPI] # 1. Get project diff --git a/services/web/server/src/simcore_service_webserver/users/_tokens_rest.py b/services/web/server/src/simcore_service_webserver/users/_tokens_rest.py index 0c11c9dcae72..ef38c61eb295 100644 --- a/services/web/server/src/simcore_service_webserver/users/_tokens_rest.py +++ b/services/web/server/src/simcore_service_webserver/users/_tokens_rest.py @@ -58,7 +58,7 @@ async def create_token(request: web.Request) -> web.Response: token_create = await parse_request_body_as(MyTokenCreate, request) token = await _tokens_service.create_token( - request.app, req_ctx.user_id, token_create.to_model() + request.app, req_ctx.user_id, token_create.to_domain_model() ) return envelope_json_response(MyTokenGet.from_domain_model(token), web.HTTPCreated) diff --git a/services/web/server/tests/unit/isolated/test_groups_models.py b/services/web/server/tests/unit/isolated/test_groups_models.py index 06729b3519a4..a6711f6329a0 100644 --- a/services/web/server/tests/unit/isolated/test_groups_models.py +++ b/services/web/server/tests/unit/isolated/test_groups_models.py @@ -82,13 +82,13 @@ def test_input_schemas_to_models(faker: Faker): input_schema = GroupCreate( label=faker.word(), description=faker.sentence(), thumbnail=faker.url() ) - domain_model = input_schema.to_model() + domain_model = input_schema.to_domain_model() assert isinstance(domain_model, StandardGroupCreate) assert domain_model.name == input_schema.label # input : scheam -> model input_schema = GroupUpdate(label=faker.word()) - domain_model = input_schema.to_model() + domain_model = input_schema.to_domain_model() assert isinstance(domain_model, StandardGroupUpdate) assert domain_model.name == input_schema.label From a42159159221c904b6450823870502082f8020c4 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:00:39 +0100 Subject: [PATCH 62/71] extra missing --- .../server/src/simcore_service_webserver/tags/_rest.py | 10 +++++----- .../src/simcore_service_webserver/tags/_service.py | 6 +++--- .../src/simcore_service_webserver/tags/schemas.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/tags/_rest.py b/services/web/server/src/simcore_service_webserver/tags/_rest.py index ff812486e0bf..7550c8343edd 100644 --- a/services/web/server/src/simcore_service_webserver/tags/_rest.py +++ b/services/web/server/src/simcore_service_webserver/tags/_rest.py @@ -148,7 +148,7 @@ async def list_tag_groups(request: web.Request): caller_user_id=req_ctx.user_id, tag_id=path_params.tag_id, ) - return envelope_json_response([TagGroupGet.from_model(md) for md in got]) + return envelope_json_response([TagGroupGet.from_domain_model(md) for md in got]) @routes.post(f"/{VTAG}/tags/{{tag_id}}/groups/{{group_id}}", name="create_tag_group") @@ -165,11 +165,11 @@ async def create_tag_group(request: web.Request): caller_user_id=req_ctx.user_id, tag_id=path_params.tag_id, group_id=path_params.group_id, - access_rights=body_params.to_model(), + access_rights=body_params.to_domain_model(), ) return envelope_json_response( - TagGroupGet.from_model(got), status_cls=web.HTTPCreated + TagGroupGet.from_domain_model(got), status_cls=web.HTTPCreated ) @@ -187,10 +187,10 @@ async def replace_tag_group(request: web.Request): caller_user_id=req_ctx.user_id, tag_id=path_params.tag_id, group_id=path_params.group_id, - access_rights=body_params.to_model(), + access_rights=body_params.to_domain_model(), ) - return envelope_json_response(TagGroupGet.from_model(got)) + return envelope_json_response(TagGroupGet.from_domain_model(got)) @routes.delete(f"/{VTAG}/tags/{{tag_id}}/groups/{{group_id}}", name="delete_tag_group") diff --git a/services/web/server/src/simcore_service_webserver/tags/_service.py b/services/web/server/src/simcore_service_webserver/tags/_service.py index 36dcbdfcd46e..be73441f224d 100644 --- a/services/web/server/src/simcore_service_webserver/tags/_service.py +++ b/services/web/server/src/simcore_service_webserver/tags/_service.py @@ -35,7 +35,7 @@ async def create_tag( delete=True, **new_tag.model_dump(exclude_unset=True), ) - return TagGet.from_model(tag) + return TagGet.from_domain_model(tag) async def list_tags( @@ -45,7 +45,7 @@ async def list_tags( engine: AsyncEngine = get_async_engine(app) repo = TagsRepo(engine) tags = await repo.list_all(user_id=user_id) - return [TagGet.from_model(t) for t in tags] + return [TagGet.from_domain_model(t) for t in tags] async def update_tag( @@ -59,7 +59,7 @@ async def update_tag( tag_id=tag_id, **tag_updates.model_dump(exclude_unset=True), ) - return TagGet.from_model(tag) + return TagGet.from_domain_model(tag) async def delete_tag(app: web.Application, user_id: UserID, tag_id: IdInt): diff --git a/services/web/server/src/simcore_service_webserver/tags/schemas.py b/services/web/server/src/simcore_service_webserver/tags/schemas.py index 95fdd8e717d1..5f83e3032ecd 100644 --- a/services/web/server/src/simcore_service_webserver/tags/schemas.py +++ b/services/web/server/src/simcore_service_webserver/tags/schemas.py @@ -55,7 +55,7 @@ class TagGet(OutputSchema): access_rights: TagAccessRights = Field(..., alias="accessRights") @classmethod - def from_model(cls, tag: TagDict) -> Self: + def from_domain_model(cls, tag: TagDict) -> Self: # NOTE: cls(access_rights=tag, **tag) would also work because of Config return cls( id=tag["id"], @@ -84,7 +84,7 @@ class TagGroupCreate(InputSchema): write: bool delete: bool - def to_model(self) -> AccessRightsDict: + def to_domain_model(self) -> AccessRightsDict: data = self.model_dump() return AccessRightsDict( read=data["read"], @@ -101,7 +101,7 @@ class TagGroupGet(OutputSchema): delete: bool @classmethod - def from_model(cls, data: TagAccessRightsDict) -> Self: + def from_domain_model(cls, data: TagAccessRightsDict) -> Self: return cls( gid=data["group_id"], read=data["read"], From 2ac76232ff4793cf10c40b5a4c5f2fa8216adbc7 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:26:00 +0100 Subject: [PATCH 63/71] refactor to reduce complexity (sonarcloud) --- .../folders/_folders_repository.py | 95 ++++++++++++------- 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py index a2ebbc386b24..243fba7e858e 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py @@ -95,40 +95,17 @@ async def create( return FolderDB.model_validate(row) -async def list_( # pylint: disable=too-many-arguments,too-many-branches - app: web.Application, - connection: AsyncConnection | None = None, - *, +def _create_private_workspace_query( product_name: ProductName, user_id: UserID, - # hierarchy filters - folder_query: FolderQuery, - workspace_query: WorkspaceQuery, - # attribute filters - filter_trashed: bool | None, - filter_by_text: str | None, - # pagination - offset: NonNegativeInt, - limit: int, - # order - order_by: OrderBy, -) -> tuple[int, list[UserFolderAccessRightsDB]]: - """ - folder_query - Used to filter in which folder we want to list folders. - trashed - If set to true, it returns folders **explicitly** trashed, if false then non-trashed folders. - """ - - workspace_access_rights_subquery = create_my_workspace_access_rights_subquery( - user_id=user_id - ) - - if workspace_query.workspace_scope is not WorkspaceScope.SHARED: - assert workspace_query.workspace_scope in ( # nosec + workspace_scope: WorkspaceScope, +): + if workspace_scope is not WorkspaceScope.SHARED: + assert workspace_scope in ( # nosec WorkspaceScope.PRIVATE, WorkspaceScope.ALL, ) - - private_workspace_query = ( + return ( select( *_SELECTION_ARGS, func.json_build_object( @@ -146,15 +123,25 @@ async def list_( # pylint: disable=too-many-arguments,too-many-branches & (folders_v2.c.user_id == user_id) ) ) - else: - private_workspace_query = None + return None - if workspace_query.workspace_scope is not WorkspaceScope.PRIVATE: - assert workspace_query.workspace_scope in ( # nosec + +def _create_shared_workspace_query( + product_name: ProductName, + user_id: UserID, + workspace_scope: WorkspaceScope, + workspace_id: WorkspaceID | None, +): + if workspace_scope is not WorkspaceScope.PRIVATE: + assert workspace_scope in ( # nosec WorkspaceScope.SHARED, WorkspaceScope.ALL, ) + workspace_access_rights_subquery = create_my_workspace_access_rights_subquery( + user_id=user_id + ) + shared_workspace_query = ( select( *_SELECTION_ARGS, workspace_access_rights_subquery.c.my_access_rights @@ -172,14 +159,52 @@ async def list_( # pylint: disable=too-many-arguments,too-many-branches ) ) - if workspace_query.workspace_scope == WorkspaceScope.SHARED: + if workspace_scope == WorkspaceScope.SHARED: shared_workspace_query = shared_workspace_query.where( - folders_v2.c.workspace_id == workspace_query.workspace_id + folders_v2.c.workspace_id == workspace_id ) else: shared_workspace_query = None + return shared_workspace_query + + +async def list_( # pylint: disable=too-many-arguments,too-many-branches + app: web.Application, + connection: AsyncConnection | None = None, + *, + product_name: ProductName, + user_id: UserID, + # hierarchy filters + folder_query: FolderQuery, + workspace_query: WorkspaceQuery, + # attribute filters + filter_trashed: bool | None, + filter_by_text: str | None, + # pagination + offset: NonNegativeInt, + limit: int, + # order + order_by: OrderBy, +) -> tuple[int, list[UserFolderAccessRightsDB]]: + """ + folder_query - Used to filter in which folder we want to list folders. + trashed - If set to true, it returns folders **explicitly** trashed, if false then non-trashed folders. + """ + + private_workspace_query = _create_private_workspace_query( + workspace_scope=workspace_query.workspace_scope, + product_name=product_name, + user_id=user_id, + ) + shared_workspace_query = _create_shared_workspace_query( + workspace_scope=workspace_query.workspace_scope, + product_name=product_name, + user_id=user_id, + workspace_id=workspace_query.workspace_id, + ) + attributes_filters: list[ColumnElement] = [] if filter_trashed is not None: From 2823cc434452e672dbb4ecee0ee15dc8c0343903 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 14 Jan 2025 17:15:47 +0100 Subject: [PATCH 64/71] fixes tests --- .../web/server/tests/unit/with_dbs/03/test_project_db.py | 8 ++++++-- .../tests/unit/with_dbs/03/version_control/conftest.py | 9 ++++++--- .../test_studies_dispatcher_studies_access.py | 8 +++++--- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/services/web/server/tests/unit/with_dbs/03/test_project_db.py b/services/web/server/tests/unit/with_dbs/03/test_project_db.py index d592bc1d8d90..0d9ef69bcbea 100644 --- a/services/web/server/tests/unit/with_dbs/03/test_project_db.py +++ b/services/web/server/tests/unit/with_dbs/03/test_project_db.py @@ -16,7 +16,7 @@ import pytest import sqlalchemy as sa from aiohttp.test_utils import TestClient -from common_library.dict_tools import copy_from_dict_ex +from common_library.dict_tools import copy_from_dict_ex, remap_keys from faker import Faker from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID, NodeIDStr @@ -710,7 +710,11 @@ async def test_replace_user_project( aiopg_engine: aiopg.sa.engine.Engine, ): PROJECT_DICT_IGNORE_FIELDS = {"lastChangeDate"} - original_project = user_project + original_project = remap_keys( + user_project, + rename={"trashedAt": "trashed"}, + ) + # replace the project with the same should do nothing working_project = await db_api.replace_project( original_project, diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py index 53bdeb18c5b0..d9fc7213a63f 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/conftest.py @@ -10,6 +10,7 @@ import aiohttp import pytest from aiohttp.test_utils import TestClient +from common_library.dict_tools import remap_keys from faker import Faker from models_library.projects import ProjectID from models_library.projects_nodes import Node @@ -198,11 +199,13 @@ async def _go(client: TestClient, project_uuid: UUID) -> None: project["workbench"] = {node_id: jsonable_encoder(node)} db: ProjectDBAPI = ProjectDBAPI.get_from_app_context(client.app) - project.pop("state") + project_db = remap_keys(project, rename={"trashedAt": "trashed"}) + project_db.pop("state") + await db.replace_project( - project, + project_db, logged_user["id"], - project_uuid=project["uuid"], + project_uuid=project_db["uuid"], product_name="osparc", ) diff --git a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py index 582376778fa9..bcbcbb4c6f79 100644 --- a/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/04/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -55,14 +55,16 @@ async def _get_user_projects(client) -> list[ProjectDict]: def _assert_same_projects(got: dict, expected: dict): exclude = { + "accessRights", "creationDate", "lastChangeDate", "prjOwner", + "trashedAt", + "trashedBy", + "trashedExplicitly", + "ui", "uuid", "workbench", - "accessRights", - "ui", - "trashedExplicitly", } for key in expected: if key not in exclude: From 7a22838c52e6e247b323d967025e2158f81003fa Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:39:18 +0100 Subject: [PATCH 65/71] new col --- tests/e2e/tutorials/sleepers_project_template_sql.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/tutorials/sleepers_project_template_sql.csv b/tests/e2e/tutorials/sleepers_project_template_sql.csv index 6dbcd7d2a265..46915eba7792 100644 --- a/tests/e2e/tutorials/sleepers_project_template_sql.csv +++ b/tests/e2e/tutorials/sleepers_project_template_sql.csv @@ -1,2 +1,2 @@ -id,type,uuid,name,description,thumbnail,prj_owner,creation_date,last_change_date,workbench,published,access_rights,dev,ui,classifiers,quality,hidden,workspace_id,trashed_at,trashed_explicitly -10,TEMPLATE,ed6c2f58-dc16-445d-bb97-e989e2611603,Sleepers,5 sleepers interconnected,"",,2019-06-06 14:34:19.631,2019-06-06 14:34:28.647,"{""027e3ff9-3119-45dd-b8a2-2e31661a7385"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 0"", ""inputs"": {""in_2"": 2}, ""inputAccess"": {""in_1"": ""Invisible"", ""in_2"": ""ReadOnly""}, ""inputNodes"": [], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 50, ""y"": 300}}, ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 1"", ""inputs"": {""in_1"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_1""}, ""in_2"": 2}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 300, ""y"": 200}}, ""bf405067-d168-44ba-b6dc-bb3e08542f92"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 2"", ""inputs"": {""in_1"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_2""}}, ""inputNodes"": [""562aaea9-95ff-46f3-8e84-db8f3c9e3a39""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 550, ""y"": 200}}, ""de2578c5-431e-5065-a079-a5a0476e3c10"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 3"", ""inputs"": {""in_2"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_2""}}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 420, ""y"": 400}}, ""de2578c5-431e-559d-aa19-dc9293e10e4c"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 4"", ""inputs"": {""in_1"": {""nodeUuid"": ""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""de2578c5-431e-5065-a079-a5a0476e3c10"", ""output"": ""out_2""}}, ""inputNodes"": [""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""de2578c5-431e-5065-a079-a5a0476e3c10""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 800, ""y"": 300}}}",true,"{""1"": {""read"":true, ""write"":false, ""delete"":false}}", "{}", "{}", "{}", "{}",false,,,false +id,type,uuid,name,description,thumbnail,prj_owner,creation_date,last_change_date,workbench,published,access_rights,dev,ui,classifiers,quality,hidden,workspace_id,trashed,trashed_by,trashed_explicitly +10,TEMPLATE,ed6c2f58-dc16-445d-bb97-e989e2611603,Sleepers,5 sleepers interconnected,"",,2019-06-06 14:34:19.631,2019-06-06 14:34:28.647,"{""027e3ff9-3119-45dd-b8a2-2e31661a7385"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 0"", ""inputs"": {""in_2"": 2}, ""inputAccess"": {""in_1"": ""Invisible"", ""in_2"": ""ReadOnly""}, ""inputNodes"": [], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 50, ""y"": 300}}, ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 1"", ""inputs"": {""in_1"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_1""}, ""in_2"": 2}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 300, ""y"": 200}}, ""bf405067-d168-44ba-b6dc-bb3e08542f92"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 2"", ""inputs"": {""in_1"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_2""}}, ""inputNodes"": [""562aaea9-95ff-46f3-8e84-db8f3c9e3a39""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 550, ""y"": 200}}, ""de2578c5-431e-5065-a079-a5a0476e3c10"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 3"", ""inputs"": {""in_2"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_2""}}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 420, ""y"": 400}}, ""de2578c5-431e-559d-aa19-dc9293e10e4c"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 4"", ""inputs"": {""in_1"": {""nodeUuid"": ""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""de2578c5-431e-5065-a079-a5a0476e3c10"", ""output"": ""out_2""}}, ""inputNodes"": [""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""de2578c5-431e-5065-a079-a5a0476e3c10""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 800, ""y"": 300}}}",true,"{""1"": {""read"":true, ""write"":false, ""delete"":false}}", "{}", "{}", "{}", "{}",false,,,,false From 3f43cf984003658733f60fe347f00e9ec5f6eaf6 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:42:24 +0100 Subject: [PATCH 66/71] test validation --- .../03/version_control/test_version_control_handlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py index ac229a3b410d..b17829bb30be 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py @@ -11,6 +11,7 @@ import aiohttp import pytest from aiohttp.test_utils import TestClient +from models_library.api_schemas_webserver.projects import ProjectGet from models_library.projects import Project, ProjectID from models_library.rest_pagination import Page from models_library.users import UserID @@ -62,7 +63,7 @@ async def test_workflow( # get existing project resp = await client.get(f"/{VX}/projects/{project_uuid}") data, _ = await assert_status(resp, status.HTTP_200_OK) - project = Project.model_validate(data) + project = ProjectGet.model_validate(data) assert project.uuid == UUID(project_uuid) # From 2244e955c5c7f71b77b9dc58d7f88efa20ed4e2e Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Jan 2025 08:59:56 +0100 Subject: [PATCH 67/71] fixes response model in open-project --- .../simcore_service_webserver/projects/_states_handlers.py | 3 ++- .../tests/unit/with_dbs/02/test_projects_states_handlers.py | 4 ++-- .../03/version_control/test_version_control_handlers.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py index 67a21b23eceb..6b6256c00e59 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/_states_handlers.py @@ -8,6 +8,7 @@ import logging from aiohttp import web +from models_library.api_schemas_webserver.projects import ProjectGet from models_library.projects_state import ProjectState from pydantic import BaseModel from servicelib.aiohttp import status @@ -169,7 +170,7 @@ async def open_project(request: web.Request) -> web.Response: ) await projects_api.notify_project_state_update(request.app, project) - return envelope_json_response(project) + return envelope_json_response(ProjectGet.from_domain_model(project)) except DirectorServiceError as exc: # there was an issue while accessing the director-v2/director-v0 diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py index 6fcd177f37b2..f1d1933f6963 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_states_handlers.py @@ -1368,7 +1368,7 @@ async def test_open_shared_project_at_same_time( client: TestClient, client_on_running_server_factory: Callable, logged_user: dict, - shared_project: dict, + shared_project: ProjectDict, socketio_client_factory: Callable, client_session_id_factory: Callable, user_role: UserRole, @@ -1444,7 +1444,7 @@ async def test_open_shared_project_at_same_time( elif data: project_status = ProjectState(**data.pop("state")) data.pop("folderId") - assert data == shared_project + assert data == {k: shared_project[k] for k in data} assert project_status.locked.value assert project_status.locked.owner assert project_status.locked.owner.first_name in [ diff --git a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py index b17829bb30be..a1faf49d35f5 100644 --- a/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py +++ b/services/web/server/tests/unit/with_dbs/03/version_control/test_version_control_handlers.py @@ -12,7 +12,7 @@ import pytest from aiohttp.test_utils import TestClient from models_library.api_schemas_webserver.projects import ProjectGet -from models_library.projects import Project, ProjectID +from models_library.projects import ProjectID from models_library.rest_pagination import Page from models_library.users import UserID from pydantic.main import BaseModel @@ -180,7 +180,7 @@ async def test_workflow( # get working copy resp = await client.get(f"/{VX}/projects/{project_uuid}") data, _ = await assert_status(resp, status.HTTP_200_OK) - project_wc = Project.model_validate(data) + project_wc = ProjectGet.model_validate(data) assert project_wc.uuid == UUID(project_uuid) assert project_wc != project From a7740c749ee62c00db7a1771979376ace1e87e13 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Wed, 15 Jan 2025 10:21:43 +0100 Subject: [PATCH 68/71] fixes csv --- tests/e2e/tutorials/sleepers_project_template_sql.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/e2e/tutorials/sleepers_project_template_sql.csv b/tests/e2e/tutorials/sleepers_project_template_sql.csv index 46915eba7792..17f406b0aa03 100644 --- a/tests/e2e/tutorials/sleepers_project_template_sql.csv +++ b/tests/e2e/tutorials/sleepers_project_template_sql.csv @@ -1,2 +1,2 @@ -id,type,uuid,name,description,thumbnail,prj_owner,creation_date,last_change_date,workbench,published,access_rights,dev,ui,classifiers,quality,hidden,workspace_id,trashed,trashed_by,trashed_explicitly -10,TEMPLATE,ed6c2f58-dc16-445d-bb97-e989e2611603,Sleepers,5 sleepers interconnected,"",,2019-06-06 14:34:19.631,2019-06-06 14:34:28.647,"{""027e3ff9-3119-45dd-b8a2-2e31661a7385"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 0"", ""inputs"": {""in_2"": 2}, ""inputAccess"": {""in_1"": ""Invisible"", ""in_2"": ""ReadOnly""}, ""inputNodes"": [], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 50, ""y"": 300}}, ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 1"", ""inputs"": {""in_1"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_1""}, ""in_2"": 2}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 300, ""y"": 200}}, ""bf405067-d168-44ba-b6dc-bb3e08542f92"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 2"", ""inputs"": {""in_1"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_2""}}, ""inputNodes"": [""562aaea9-95ff-46f3-8e84-db8f3c9e3a39""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 550, ""y"": 200}}, ""de2578c5-431e-5065-a079-a5a0476e3c10"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 3"", ""inputs"": {""in_2"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_2""}}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 420, ""y"": 400}}, ""de2578c5-431e-559d-aa19-dc9293e10e4c"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 4"", ""inputs"": {""in_1"": {""nodeUuid"": ""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""de2578c5-431e-5065-a079-a5a0476e3c10"", ""output"": ""out_2""}}, ""inputNodes"": [""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""de2578c5-431e-5065-a079-a5a0476e3c10""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 800, ""y"": 300}}}",true,"{""1"": {""read"":true, ""write"":false, ""delete"":false}}", "{}", "{}", "{}", "{}",false,,,,false +id,type,uuid,name,description,thumbnail,prj_owner,creation_date,last_change_date,workbench,published,access_rights,dev,classifiers,ui,quality,hidden,workspace_id,trashed,trashed_explicitly,trashed_by +10,TEMPLATE,ed6c2f58-dc16-445d-bb97-e989e2611603,Sleepers,5 sleepers interconnected,"",,2019-06-06 14:34:19.631,2019-06-06 14:34:28.647,"{""027e3ff9-3119-45dd-b8a2-2e31661a7385"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 0"", ""inputs"": {""in_2"": 2}, ""inputAccess"": {""in_1"": ""Invisible"", ""in_2"": ""ReadOnly""}, ""inputNodes"": [], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 50, ""y"": 300}}, ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 1"", ""inputs"": {""in_1"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_1""}, ""in_2"": 2}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 300, ""y"": 200}}, ""bf405067-d168-44ba-b6dc-bb3e08542f92"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 2"", ""inputs"": {""in_1"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""562aaea9-95ff-46f3-8e84-db8f3c9e3a39"", ""output"": ""out_2""}}, ""inputNodes"": [""562aaea9-95ff-46f3-8e84-db8f3c9e3a39""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 550, ""y"": 200}}, ""de2578c5-431e-5065-a079-a5a0476e3c10"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 3"", ""inputs"": {""in_2"": {""nodeUuid"": ""027e3ff9-3119-45dd-b8a2-2e31661a7385"", ""output"": ""out_2""}}, ""inputNodes"": [""027e3ff9-3119-45dd-b8a2-2e31661a7385""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 420, ""y"": 400}}, ""de2578c5-431e-559d-aa19-dc9293e10e4c"": {""key"": ""simcore/services/comp/itis/sleeper"", ""version"": ""1.0.0"", ""label"": ""sleeper 4"", ""inputs"": {""in_1"": {""nodeUuid"": ""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""output"": ""out_1""}, ""in_2"": {""nodeUuid"": ""de2578c5-431e-5065-a079-a5a0476e3c10"", ""output"": ""out_2""}}, ""inputNodes"": [""bf405067-d168-44ba-b6dc-bb3e08542f92"", ""de2578c5-431e-5065-a079-a5a0476e3c10""], ""outputs"": {}, ""progress"": 0, ""thumbnail"": """", ""position"": {""x"": 800, ""y"": 300}}}",t,"{""1"": {""read"": true, ""write"": false, ""delete"": false}}",{},{},{},{},f,,,f, From 2bec88de21e3a93de7d9fb461dc101af04e27dbe Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Jan 2025 15:57:09 +0100 Subject: [PATCH 69/71] adds logging --- .../users/_users_repository.py | 4 ++-- .../unit/with_dbs/docker-compose-devel.yml | 19 ++++++++++++++----- .../tests/unit/with_dbs/docker-compose.yml | 12 ++++++++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/services/web/server/src/simcore_service_webserver/users/_users_repository.py b/services/web/server/src/simcore_service_webserver/users/_users_repository.py index 5fcc88af4a1e..16730437394b 100644 --- a/services/web/server/src/simcore_service_webserver/users/_users_repository.py +++ b/services/web/server/src/simcore_service_webserver/users/_users_repository.py @@ -196,13 +196,13 @@ async def get_user_fullname(app: web.Application, *, user_id: UserID) -> FullNam user_id = _parse_as_user(user_id) async with pass_or_acquire_connection(engine=get_asyncpg_engine(app)) as conn: - result = await conn.stream( + result = await conn.execute( sa.select( users.c.first_name, users.c.last_name, ).where(users.c.id == user_id) ) - user = await result.first() + user = result.first() if not user: raise UserNotFoundError(user_id=user_id) diff --git a/services/web/server/tests/unit/with_dbs/docker-compose-devel.yml b/services/web/server/tests/unit/with_dbs/docker-compose-devel.yml index 198e2d6e462e..2a4402c85a28 100644 --- a/services/web/server/tests/unit/with_dbs/docker-compose-devel.yml +++ b/services/web/server/tests/unit/with_dbs/docker-compose-devel.yml @@ -20,11 +20,20 @@ services: # - net.ipv4.tcp_keepalive_intvl=600 # - net.ipv4.tcp_keepalive_probes=9 # - net.ipv4.tcp_keepalive_time=600 - command: | - postgres - -c tcp_keepalives_idle=600 - -c tcp_keepalives_interval=600 - -c tcp_keepalives_count=5 + command: + - "postgres" + - "-c" + - "tcp_keepalives_idle=600" + - "-c" + - "tcp_keepalives_interval=600" + - "-c" + - "tcp_keepalives_count=5" + - "-c" + - "log_statement=all" + - "-c" + - "log_min_duration_statement=500" + - "-c" + - "log_lock_waits=on" adminer: image: adminer:4.8.1 init: true diff --git a/services/web/server/tests/unit/with_dbs/docker-compose.yml b/services/web/server/tests/unit/with_dbs/docker-compose.yml index a5b504af55f6..6fde4baab748 100644 --- a/services/web/server/tests/unit/with_dbs/docker-compose.yml +++ b/services/web/server/tests/unit/with_dbs/docker-compose.yml @@ -19,8 +19,16 @@ services: # - net.ipv4.tcp_keepalive_intvl=600 # - net.ipv4.tcp_keepalive_probes=9 # - net.ipv4.tcp_keepalive_time=600 - command: postgres -c tcp_keepalives_idle=600 -c tcp_keepalives_interval=600 -c tcp_keepalives_count=5 - + command: + - "postgres" + - "-c" + - "tcp_keepalives_idle=600" + - "-c" + - "tcp_keepalives_interval=600" + - "-c" + - "tcp_keepalives_count=5" + - "-c" + - "log_lock_waits=on" redis: image: "redis:6.2.6@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d" init: true From 7b12d1edee946b5ae329b54898ba6ef1582a4c4d Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Jan 2025 16:10:39 +0100 Subject: [PATCH 70/71] fixes flaky by terminating open transactions in teardown --- services/web/server/tests/unit/with_dbs/conftest.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/unit/with_dbs/conftest.py b/services/web/server/tests/unit/with_dbs/conftest.py index 12d34d5be78c..6de000d96868 100644 --- a/services/web/server/tests/unit/with_dbs/conftest.py +++ b/services/web/server/tests/unit/with_dbs/conftest.py @@ -527,7 +527,17 @@ def postgres_db( with engine.begin() as conn: conn.execute(sa.DDL("DROP TABLE IF EXISTS alembic_version")) - orm.metadata.drop_all(engine) + conn.execute( + # NOTE: terminates all open transactions before droping all tables + # This solves https://github.com/ITISFoundation/osparc-simcore/issues/7008 + sa.DDL( + "SELECT pg_terminate_backend(pid) " + "FROM pg_stat_activity " + "WHERE state = 'idle in transaction';" + ) + ) + orm.metadata.drop_all(bind=conn) + engine.dispose() From 6a6fb1a9fe694eeef68908a176c4fb6c9cbdcb85 Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Thu, 16 Jan 2025 17:58:26 +0100 Subject: [PATCH 71/71] adds termination as well for integration tests --- .../src/pytest_simcore/helpers/postgres_tools.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/postgres_tools.py b/packages/pytest-simcore/src/pytest_simcore/helpers/postgres_tools.py index 6587f9052fa5..5803adee9f84 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/postgres_tools.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/postgres_tools.py @@ -49,11 +49,19 @@ def migrated_pg_tables_context( simcore_postgres_database.cli.downgrade.callback("base") simcore_postgres_database.cli.clean.callback() # just cleans discover cache - # FIXME: migration downgrade fails to remove User types - # SEE https://github.com/ITISFoundation/osparc-simcore/issues/1776 - # Added drop_all as tmp fix postgres_engine = sa.create_engine(dsn) - metadata.drop_all(bind=postgres_engine) + with postgres_engine.begin() as conn: + conn.execute( + # NOTE: terminates all open transactions before droping all tables + # This solves https://github.com/ITISFoundation/osparc-simcore/issues/7008 + sa.DDL( + "SELECT pg_terminate_backend(pid) " + "FROM pg_stat_activity " + "WHERE state = 'idle in transaction';" + ) + ) + # SEE https://github.com/ITISFoundation/osparc-simcore/issues/1776 + metadata.drop_all(bind=postgres_engine) def is_postgres_responsive(url) -> bool: