fixed timeout and error handling with access

Author: Andrei Neagu

packages/models-library/src/models_library/api_schemas_storage/data_export_async_jobs.py

@@ -15,5 +15,5 @@ class InvalidFileIdentifierError(StorageRpcBaseError):
 
 class AccessRightError(StorageRpcBaseError):
     msg_template: str = (
-        "User {user_id} does not have access to file {file_id} with location {location_id}"
+        "User {user_id} does not have access to one or multiple files in {paths_to_download} with location {location_id}"
     )

services/storage/src/simcore_service_storage/api/_worker_tasks/_simcore_s3.py

@@ -12,7 +12,7 @@
 from pydantic import TypeAdapter
 from servicelib.logging_utils import log_context
 from servicelib.progress_bar import ProgressBarData
-from simcore_service_storage.exceptions.errors import FileAccessRightError
+from simcore_service_storage.exceptions.errors import ProjectAccessRightError
 
 from ...dsm import get_dsm_provider
 from ...modules.celery.models import TaskID, TaskId
@@ -80,20 +80,10 @@ async def export_data(
         assert isinstance(dsm, SimcoreS3DataManager)  # nosec
 
         paths_to_export = [
-            TypeAdapter(StorageFileID).validate_python(path_to_export)
+            TypeAdapter(S3ObjectKey).validate_python(path_to_export)
             for path_to_export in paths_to_export
         ]
 
-        try:
-            for path_to_export in paths_to_export:
-                await dsm.can_read_file(user_id=user_id, file_id=path_to_export)
-        except FileAccessRightError as err:
-            raise AccessRightError(
-                user_id=user_id,
-                file_id=path_to_export,
-                location_id=SimcoreS3DataManager.get_location_id(),
-            ) from err
-
         async def _progress_cb(report: ProgressReport) -> None:
             assert task.name  # nosec
             get_celery_worker(task.app).set_task_progress(task.name, task_id, report)
@@ -104,6 +94,13 @@ async def _progress_cb(report: ProgressReport) -> None:
             description=f"'{task_id}' export data",
             progress_report_cb=_progress_cb,
         ) as progress_bar:
-            return await dsm.create_s3_export(
-                user_id, paths_to_export, progress_bar=progress_bar
-            )
+            try:
+                return await dsm.create_s3_export(
+                    user_id, paths_to_export, progress_bar=progress_bar
+                )
+            except ProjectAccessRightError as err:
+                raise AccessRightError(
+                    user_id=user_id,
+                    paths_to_download=paths_to_export,
+                    location_id=SimcoreS3DataManager.get_location_id(),
+                ) from err

services/storage/src/simcore_service_storage/modules/celery/_task.py

@@ -17,7 +17,7 @@
 
 _logger = logging.getLogger(__name__)
 
-_DEFAULT_TASK_TIMEOUT: Final[timedelta] | None = None
+_DEFAULT_TASK_TIMEOUT: Final[timedelta | None] = None
 _DEFAULT_MAX_RETRIES: Final[NonNegativeInt] = 3
 _DEFAULT_WAIT_BEFORE_RETRY: Final[timedelta] = timedelta(seconds=5)
 

services/storage/src/simcore_service_storage/simcore_s3_dsm.py

@@ -1250,6 +1250,20 @@ async def create_s3_export(
     ) -> StorageFileID:
         source_object_keys: set[StorageFileID] = set()
 
+        # check access rights
+        for object_key in object_keys:
+            project_id = None
+            with contextlib.suppress(ValueError):
+                project_id = ProjectID(Path(object_key).parts[0])
+
+            accessible_projects_ids = await get_accessible_project_ids(
+                get_db_engine(self.app), user_id=user_id, project_id=project_id
+            )
+            if project_id is None or project_id not in accessible_projects_ids:
+                raise ProjectAccessRightError(
+                    access_right="read", project_id=project_id
+                )
+
         for object_key in object_keys:
             async for meta_data_files in get_s3_client(self.app).list_objects_paginated(
                 self.simcore_bucket_name, object_key