✨ Add ENV to prevent copying if project too large (⚠️ devops) (ITISFoundation#3286)

Author: sanderegg

.env-devel

@@ -87,6 +87,7 @@ TRACING_THRIFT_COMPACT_ENDPOINT=http://jaeger:5775
 TRAEFIK_SIMCORE_ZONE=internal_simcore_stack
 
 # NOTE: WEBSERVER_SESSION_SECRET_KEY = $(python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key())")
+PROJECTS_MAX_COPY_SIZE_BYTES=30Gib
 WEBSERVER_DEV_FEATURES_ENABLED=0
 WEBSERVER_HOST=webserver
 WEBSERVER_LOGIN_REGISTRATION_CONFIRMATION_REQUIRED=0

.env-wb-garbage-collector

@@ -15,6 +15,7 @@ WEBSERVER_EXPORTER=null
 WEBSERVER_FRONTEND=null
 #WEBSERVER_GARBAGE_COLLECTOR explicit in
 WEBSERVER_LOGIN=null
+WEBSERVER_PROJECTS=null
 #WEBSERVER_REDIS from .env
 #WEBSERVER_REST needed for the healthcheck
 #WEBSERVER_RESOURCE_MANAGER from .env
@@ -28,7 +29,6 @@ WEBSERVER_CLUSTERS=0
 WEBSERVER_GROUPS=0
 WEBSERVER_META_MODELING=0
 WEBSERVER_PRODUCTS=0
-WEBSERVER_PROJECTS=0
 WEBSERVER_PUBLICATIONS=0
 WEBSERVER_SOCKETIO=0
 WEBSERVER_STUDIES_DISPATCHER=null

services/web/client/source/class/osparc/data/PollTask.js

@@ -108,7 +108,7 @@ qx.Class.define("osparc.data.PollTask", {
           .then(res => res.json())
           .then(result => {
             if ("error" in result && result["error"]) {
-              throw new Error(result["error"]);
+              throw new Error(result["error"]["message"]);
             }
             if ("data" in result && result["data"]) {
               const resultData = result["data"];

services/web/server/src/simcore_service_webserver/application_settings.py

@@ -1,6 +1,6 @@
 import logging
 from functools import cached_property
-from typing import Any, Dict, List, Optional
+from typing import Any, Optional
 
 from aiohttp import web
 from models_library.basic_types import (
@@ -33,6 +33,7 @@
 from .exporter.settings import ExporterSettings
 from .garbage_collector_settings import GarbageCollectorSettings
 from .login.settings import LoginSettings
+from .projects.projects_settings import ProjectsSettings
 from .resource_manager.settings import ResourceManagerSettings
 from .rest_settings import RestSettings
 from .scicrunch.settings import SciCrunchSettings
@@ -169,12 +170,15 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings):
         auto_default_from_env=True, description="tracing plugin"
     )
 
+    WEBSERVER_PROJECTS: Optional[ProjectsSettings] = Field(
+        auto_default_from_env=True, description="projects plugin"
+    )
+
     # These plugins only require (for the moment) an entry to toggle between enabled/disabled
     WEBSERVER_CLUSTERS: bool = True
     WEBSERVER_GROUPS: bool = True
     WEBSERVER_META_MODELING: bool = True
     WEBSERVER_PRODUCTS: bool = True
-    WEBSERVER_PROJECTS: bool = True
     WEBSERVER_PUBLICATIONS: bool = True
     WEBSERVER_REMOTE_DEBUG: bool = True
     WEBSERVER_SOCKETIO: bool = True
@@ -249,7 +253,7 @@ def get_healthcheck_timeout_in_seconds(cls, v):
     def is_enabled(self, field_name: str) -> bool:
         return bool(getattr(self, field_name, None))
 
-    def _get_disabled_public_plugins(self) -> List[str]:
+    def _get_disabled_public_plugins(self) -> list[str]:
         plugins_disabled = []
         # NOTE: this list is limited for security reasons. An unbounded list
         # might reveal critical info on the settings of a deploy to the client.
@@ -267,17 +271,15 @@ def _get_disabled_public_plugins(self) -> List[str]:
                 plugins_disabled.append(field_name)
         return plugins_disabled
 
-    def public_dict(self) -> Dict[str, Any]:
+    def public_dict(self) -> dict[str, Any]:
         """Data publicaly available"""
 
         data = {"invitation_required": False}
         if self.WEBSERVER_LOGIN:
             data[
                 "invitation_required"
             ] = self.WEBSERVER_LOGIN.LOGIN_REGISTRATION_INVITATION_REQUIRED
-            data[
-                "login_2fa_required"
-            ] = self.WEBSERVER_LOGIN.LOGIN_2FA_REQUIRED
+            data["login_2fa_required"] = self.WEBSERVER_LOGIN.LOGIN_2FA_REQUIRED
 
         data.update(
             self.dict(
@@ -294,7 +296,7 @@ def public_dict(self) -> Dict[str, Any]:
         )
         return data
 
-    def to_client_statics(self) -> Dict[str, Any]:
+    def to_client_statics(self) -> dict[str, Any]:
         data = self.dict(
             include={
                 "APP_NAME",

services/web/server/src/simcore_service_webserver/projects/plugin.py

@@ -53,7 +53,7 @@ def _create_routes(tag, specs, *handlers_module):
     logger=logger,
 )
 def setup_projects(app: web.Application) -> bool:
-    assert app[APP_SETTINGS_KEY].WEBSERVER_PROJECTS is True  # nosec
+    assert app[APP_SETTINGS_KEY].WEBSERVER_PROJECTS  # nosec
 
     # API routes
     specs = app[APP_OPENAPI_SPECS_KEY]

services/web/server/src/simcore_service_webserver/projects/projects_handlers_crud.py

@@ -37,13 +37,14 @@
 from .. import catalog, director_v2_api
 from .._constants import RQ_PRODUCT_KEY
 from .._meta import api_version_prefix as VTAG
+from ..application_settings import get_settings
 from ..login.decorators import RQT_USERID_KEY, login_required
 from ..long_running_tasks import start_task_with_context
 from ..resource_manager.websocket_manager import PROJECT_ID_KEY, managed_resource
 from ..rest_constants import RESPONSE_MODEL_POLICY
 from ..security_api import check_permission
 from ..security_decorators import permission_required
-from ..storage_api import copy_data_folders_from_project
+from ..storage_api import copy_data_folders_from_project, get_project_total_size
 from ..users_api import get_user_name
 from . import projects_api
 from .project_models import ProjectDict, ProjectTypeAPI
@@ -169,6 +170,18 @@ async def _init_project_from_request(
         include_templates=True,
     )
 
+    if max_bytes := get_settings(app).WEBSERVER_PROJECTS.PROJECTS_MAX_COPY_SIZE_BYTES:
+        # get project total data size
+        project_data_size = await get_project_total_size(
+            app, user_id, ProjectID(query_params.from_study)
+        )
+        if project_data_size >= max_bytes:
+            raise web.HTTPUnprocessableEntity(
+                reason=f"Source project data size is {project_data_size.human_readable()}."
+                f"This is larger than the maximum {max_bytes.human_readable()} allowed for copying."
+                "TIP: Please reduce the study size or contact application support."
+            )
+
     # clone template as user project
     new_project, nodes_map = clone_project_document(
         source_project,

services/web/server/src/simcore_service_webserver/projects/projects_settings.py

@@ -0,0 +1,10 @@
+from pydantic import ByteSize, Field, parse_obj_as
+from settings_library.base import BaseCustomSettings
+
+
+class ProjectsSettings(BaseCustomSettings):
+    PROJECTS_MAX_COPY_SIZE_BYTES: ByteSize = Field(
+        parse_obj_as(ByteSize, "30Gib"),
+        description="defines the maximum authorized project data size"
+        " when copying a project (disable with 0)",
+    )

services/web/server/src/simcore_service_webserver/storage_api.py

@@ -4,9 +4,14 @@
 import asyncio
 import logging
 from pprint import pformat
-from typing import Any, Dict, Tuple
+from typing import Any
 
 from aiohttp import ClientError, ClientSession, ClientTimeout, web
+from models_library.api_schemas_storage import FileLocationArray, FileMetaDataGet
+from models_library.generics import Envelope
+from models_library.projects import ProjectID
+from models_library.users import UserID
+from pydantic import ByteSize, parse_obj_as
 from pydantic.types import PositiveInt
 from servicelib.aiohttp.client_session import get_client_session
 from servicelib.aiohttp.rest_responses import unwrap_envelope
@@ -19,7 +24,7 @@
 TOTAL_TIMEOUT_TO_COPY_DATA_SECS = 60 * 60
 
 
-def _get_storage_client(app: web.Application) -> Tuple[ClientSession, URL]:
+def _get_storage_client(app: web.Application) -> tuple[ClientSession, URL]:
     settings: StorageSettings = get_plugin_settings(app)
     # storage service API endpoint
     endpoint = URL(settings.base_url)
@@ -28,16 +33,59 @@ def _get_storage_client(app: web.Application) -> Tuple[ClientSession, URL]:
     return session, endpoint
 
 
+async def get_storage_locations(
+    app: web.Application, user_id: UserID
+) -> FileLocationArray:
+    log.debug("getting %s accessible locations...", f"{user_id=}")
+    session, api_endpoint = _get_storage_client(app)
+    locations_url = (api_endpoint / "locations").with_query(user_id=user_id)
+    async with session.get(f"{locations_url}") as response:
+        response.raise_for_status()
+        locations_enveloped = Envelope[FileLocationArray].parse_obj(
+            await response.json()
+        )
+        assert locations_enveloped.data  # nosec
+        log.info("%s can access %s", f"{user_id=}", f"{locations_enveloped.data=}")
+        return locations_enveloped.data
+
+
+async def get_project_total_size(
+    app: web.Application, user_id: UserID, project_uuid: ProjectID
+) -> ByteSize:
+    log.debug("getting %s total size for %s", f"{project_uuid=}", f"{user_id=}")
+    user_accessible_locations = await get_storage_locations(app, user_id)
+    session, api_endpoint = _get_storage_client(app)
+
+    project_size_bytes = 0
+    for location in user_accessible_locations:
+        files_metadata_url = (
+            api_endpoint / "locations" / f"{location.id}" / "files" / "metadata"
+        ).with_query(user_id=user_id, uuid_filter=f"{project_uuid}")
+        async with session.get(f"{files_metadata_url}") as response:
+            response.raise_for_status()
+            list_of_files_enveloped = Envelope[list[FileMetaDataGet]].parse_obj(
+                await response.json()
+            )
+            assert list_of_files_enveloped.data  # nosec
+        for file_metadata in list_of_files_enveloped.data:
+            project_size_bytes += file_metadata.file_size
+    project_size = parse_obj_as(ByteSize, project_size_bytes)
+    log.info(
+        "%s total size is %s", f"{project_uuid}", f"{project_size.human_readable()}"
+    )
+    return project_size
+
+
 async def copy_data_folders_from_project(
     app: web.Application,
-    source_project: Dict,
-    destination_project: Dict,
-    nodes_map: Dict,
+    source_project: dict,
+    destination_project: dict,
+    nodes_map: dict,
     user_id: int,
 ):
     # TODO: optimize if project has actualy data or not before doing the call
     client, api_endpoint = _get_storage_client(app)
-    log.debug("Coying %d nodes", len(nodes_map))
+    log.debug("Copying %d nodes", len(nodes_map))
 
     # /simcore-s3/folders:
     url = (api_endpoint / "simcore-s3/folders").with_query(user_id=user_id)
@@ -113,7 +161,7 @@ async def is_healthy(app: web.Application) -> bool:
         return False
 
 
-async def get_app_status(app: web.Application) -> Dict[str, Any]:
+async def get_app_status(app: web.Application) -> dict[str, Any]:
     client, api_endpoint = _get_storage_client(app)
 
     data = {}

services/web/server/tests/conftest.py

@@ -18,6 +18,7 @@
 from models_library.projects_networks import PROJECT_NETWORK_PREFIX
 from models_library.projects_state import ProjectState
 from pytest_simcore.helpers.utils_assert import assert_status
+from pytest_simcore.helpers.utils_dict import ConfigDict
 from pytest_simcore.helpers.utils_login import LoggedUser, UserInfoDict
 from servicelib.aiohttp.long_running_tasks.server import TaskStatus
 from servicelib.json_serialization import json_dumps
@@ -129,7 +130,9 @@ async def logged_user(client, user_role: UserRole) -> AsyncIterator[UserInfoDict
 
 
 @pytest.fixture
-def monkeypatch_setenv_from_app_config(monkeypatch: MonkeyPatch) -> Callable:
+def monkeypatch_setenv_from_app_config(
+    monkeypatch: MonkeyPatch,
+) -> Callable[[ConfigDict], dict[str, str]]:
     # TODO: Change signature to be analogous to
     # packages/pytest-simcore/src/pytest_simcore/helpers/utils_envs.py
     # That solution is more flexible e.g. for context manager with monkeypatch
@@ -270,9 +273,12 @@ async def _creator(
                 )
 
         # get result GET /{task_id}/result
-        print(f"--> getting project creation result...")
+        print("--> getting project creation result...")
         result = await client.get(f"{result_url}")
         data, error = await assert_status(result, expected_creation_response)
+        if error:
+            assert not data
+            return {}
         assert data
         assert not error
         print(f"<-- result: {data}")

services/web/server/tests/data/default_app_config-unit.yaml

@@ -56,7 +56,8 @@ meta_modeling:
 products:
   enabled: true
 projects:
-  enabled: false
+  enabled: true
+  projects_max_copy_size_bytes: 5368709120
 publications:
   enabled: true
 remote_debug: