handle file not found in datcore

Author: sanderegg

services/storage/src/simcore_service_storage/datcore_dsm.py

@@ -3,6 +3,7 @@
 from pathlib import Path
 
 import arrow
+from aws_library.s3._errors import S3KeyNotFoundError
 from fastapi import FastAPI
 from models_library.api_schemas_storage.storage_schemas import (
     DatCoreCollectionName,
@@ -31,6 +32,7 @@
 from .modules.datcore_adapter import datcore_adapter
 from .modules.datcore_adapter.datcore_adapter_exceptions import (
     DatcoreAdapterMultipleFilesError,
+    DatcoreAdapterResponseError,
 )
 from .modules.db.tokens import get_api_token_and_secret
 
@@ -191,6 +193,15 @@ async def compute_path_total_size(self, user_id: UserID, *, path: Path) -> ByteS
         """returns the total size of an arbitrary path"""
         api_token, api_secret = await self._get_datcore_tokens(user_id)
         api_token, api_secret = _check_api_credentials(api_token, api_secret)
+        try:
+            paths = await self.list_paths(
+                user_id, file_filter=path, cursor=None, limit=1
+            )
+            if len(paths) == 0:
+                return ByteSize(0)
+        except ValidationError:
+            # invalid path
+            return ByteSize(0)
         raise NotImplementedError
 
     async def list_files(
@@ -268,9 +279,13 @@ async def create_file_download_link(
     ) -> AnyUrl:
         api_token, api_secret = await self._get_datcore_tokens(user_id)
         api_token, api_secret = _check_api_credentials(api_token, api_secret)
-        return await datcore_adapter.get_file_download_presigned_link(
-            self.app, api_token, api_secret, file_id
-        )
+        try:
+            return await datcore_adapter.get_file_download_presigned_link(
+                self.app, api_token, api_secret, file_id
+            )
+        except DatcoreAdapterResponseError as exc:
+            if exc.status == status.HTTP_404_NOT_FOUND:
+                raise S3KeyNotFoundError(key=file_id, bucket=self.buck)
 
     async def delete_file(self, user_id: UserID, file_id: StorageFileID) -> None:
         api_token, api_secret = await self._get_datcore_tokens(user_id)

services/storage/src/simcore_service_storage/exceptions/handlers.py

@@ -9,6 +9,7 @@
 )
 
 from ..modules.datcore_adapter.datcore_adapter_exceptions import (
+    DatcoreAdapterFileNotFoundError,
     DatcoreAdapterTimeoutError,
 )
 from .errors import (
@@ -42,6 +43,7 @@ def set_exception_handlers(app: FastAPI) -> None:
         FileMetaDataNotFoundError,
         S3KeyNotFoundError,
         ProjectNotFoundError,
+        DatcoreAdapterFileNotFoundError,
     ):
         app.add_exception_handler(
             exc_not_found,

services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter.py

@@ -2,7 +2,7 @@
 from typing import Any, TypeAlias, cast
 
 import httpx
-from fastapi import FastAPI
+from fastapi import FastAPI, status
 from fastapi_pagination import Page
 from models_library.api_schemas_datcore_adapter.datasets import (
     DatasetMetaData as DatCoreDatasetMetaData,
@@ -33,7 +33,11 @@
     TotalNumber,
 )
 from .datcore_adapter_client_utils import request, retrieve_all_pages
-from .datcore_adapter_exceptions import DatcoreAdapterError
+from .datcore_adapter_exceptions import (
+    DatcoreAdapterError,
+    DatcoreAdapterFileNotFoundError,
+    DatcoreAdapterResponseError,
+)
 from .utils import (
     create_path_meta_data_from_datcore_fmd,
     create_path_meta_data_from_datcore_package,
@@ -316,11 +320,16 @@ async def list_datasets(
 async def get_file_download_presigned_link(
     app: FastAPI, api_key: str, api_secret: str, file_id: str
 ) -> AnyUrl:
-    file_download_data = cast(
-        dict[str, Any],
-        await request(app, api_key, api_secret, "GET", f"/files/{file_id}"),
-    )
-    return TypeAdapter(AnyUrl).validate_python(file_download_data["link"])
+    try:
+        file_download_data = cast(
+            dict[str, Any],
+            await request(app, api_key, api_secret, "GET", f"/files/{file_id}"),
+        )
+        return TypeAdapter(AnyUrl).validate_python(file_download_data["link"])
+    except DatcoreAdapterResponseError as exc:
+        if exc.status == status.HTTP_404_NOT_FOUND:
+            raise DatcoreAdapterFileNotFoundError(file_id=file_id) from exc
+        raise
 
 
 async def get_package_files(

services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_exceptions.py

@@ -44,3 +44,11 @@ def __init__(self, status: int, reason: str) -> None:
         super().__init__(
             msg=f"forwarded call failed with status {status}, reason {reason}"
         )
+
+
+class DatcoreAdapterFileNotFoundError(DatcoreAdapterError):
+    """special error to check the assumption that /packages/{package_id}/files returns only one file"""
+
+    def __init__(self, file_id: str) -> None:
+        self.file_id = file_id
+        super().__init__(msg=f"file {file_id} not found!")

services/storage/tests/unit/test_handlers_paths.py

@@ -740,7 +740,6 @@ async def test_path_compute_size_inexistent_path(
     faker: Faker,
     fake_datcore_tokens: tuple[str, str],
 ):
-
     await _assert_compute_path_total_size(
         initialized_app,
         client,