diff --git a/.env-devel b/.env-devel index e4f191a0c1ec..e90466c32f0f 100644 --- a/.env-devel +++ b/.env-devel @@ -250,8 +250,6 @@ SMTP_PROTOCOL=UNENCRYPTED SMTP_USERNAME=it_doesnt_matter # STORAGE ---- -BF_API_KEY=none -BF_API_SECRET=none STORAGE_ENDPOINT=storage:8080 STORAGE_HOST=storage STORAGE_LOGLEVEL=INFO diff --git a/packages/models-library/src/models_library/api_schemas_datcore_adapter/datasets.py b/packages/models-library/src/models_library/api_schemas_datcore_adapter/datasets.py index 8011f3f9ea40..16d67cb8dddc 100644 --- a/packages/models-library/src/models_library/api_schemas_datcore_adapter/datasets.py +++ b/packages/models-library/src/models_library/api_schemas_datcore_adapter/datasets.py @@ -1,13 +1,17 @@ from datetime import datetime from enum import Enum, unique from pathlib import Path +from typing import Annotated -from pydantic import BaseModel, ByteSize +from pydantic import BaseModel, ByteSize, Field class DatasetMetaData(BaseModel): id: str display_name: str + size: Annotated[ + ByteSize | None, Field(description="Size of the dataset in bytes if available") + ] @unique diff --git a/packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py b/packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py index 05fad38ee423..000db167a10d 100644 --- a/packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py +++ b/packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py @@ -462,3 +462,31 @@ def _update_json_schema_extra(schema: JsonDict) -> None: model_config = ConfigDict( extra="forbid", json_schema_extra=_update_json_schema_extra ) + + +class PathTotalSizeCreate(BaseModel): + path: Path + size: ByteSize + + @staticmethod + def _update_json_schema_extra(schema: JsonDict) -> None: + schema.update( + { + "examples": [ + # a folder + { + "path": "f8da77a9-24b9-4eab-aee7-1f0608da1e3e", + "size": 15728640, + }, + # 1 file + { + "path": f"f8da77a9-24b9-4eab-aee7-1f0608da1e3e/2f94f80f-633e-4dfa-a983-226b7babe3d7/outputs/output5/{FileMetaDataGet.model_json_schema()['examples'][0]['file_name']}", + "size": 1024, + }, + ] + } + ) + + model_config = ConfigDict( + extra="forbid", json_schema_extra=_update_json_schema_extra + ) diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils.py b/packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils.py index f889cf31a776..39c8e2d91d77 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/storage_utils.py @@ -1,5 +1,4 @@ import logging -import os from dataclasses import dataclass from pathlib import Path from typing import Any, TypedDict @@ -14,14 +13,6 @@ log = logging.getLogger(__name__) -def has_datcore_tokens() -> bool: - api_key = os.environ.get("BF_API_KEY") - api_secret = os.environ.get("BF_API_SECRET") - if not api_key or not api_secret: - return False - return not (api_key == "none" or api_secret == "none") # noqa: S105 - - async def get_updated_project( sqlalchemy_async_engine: AsyncEngine, project_id: str ) -> dict[str, Any]: diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_datcore_adapter.py b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_datcore_adapter.py index 1b054eecff00..892d63060e0f 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_storage_datcore_adapter.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_storage_datcore_adapter.py @@ -1,7 +1,16 @@ +# pylint:disable=unused-variable +# pylint:disable=unused-argument +# pylint:disable=redefined-outer-name + import re +from collections.abc import Iterator +import httpx import pytest import respx +from faker import Faker +from fastapi_pagination import Page, Params +from pytest_simcore.helpers.host import get_localhost_ip from servicelib.aiohttp import status from simcore_service_storage.modules.datcore_adapter.datcore_adapter_settings import ( DatcoreAdapterSettings, @@ -9,7 +18,7 @@ @pytest.fixture -def datcore_adapter_service_mock() -> respx.MockRouter: +def datcore_adapter_service_mock(faker: Faker) -> Iterator[respx.MockRouter]: dat_core_settings = DatcoreAdapterSettings.create_from_envs() datcore_adapter_base_url = dat_core_settings.endpoint # mock base endpoint @@ -18,15 +27,40 @@ def datcore_adapter_service_mock() -> respx.MockRouter: assert_all_called=False, assert_all_mocked=True, ) as respx_mocker: + # NOTE: passthrough the locahost and the local ip + respx_mocker.route(host="127.0.0.1").pass_through() + respx_mocker.route(host=get_localhost_ip()).pass_through() + + respx_mocker.get("/user/profile", name="get_user_profile").respond( + status.HTTP_200_OK, json=faker.pydict(allowed_types=(str,)) + ) respx_mocker.get( - datcore_adapter_base_url, - name="healthcheck", - ).respond(status.HTTP_200_OK) - list_datasets_re = re.compile(rf"^{datcore_adapter_base_url}/datasets") + re.compile(r"/datasets/(?P[^/]+)/files_legacy") + ).respond(status.HTTP_200_OK, json=[]) + list_datasets_re = re.compile(r"/datasets") respx_mocker.get(list_datasets_re, name="list_datasets").respond( - status.HTTP_200_OK + status.HTTP_200_OK, + json=Page.create(items=[], params=Params(size=10), total=0).model_dump( + mode="json" + ), ) - respx_mocker.get(datcore_adapter_base_url, name="base_endpoint").respond( - status.HTTP_200_OK, json={} + + def _create_download_link(request, file_id): + return httpx.Response( + status.HTTP_404_NOT_FOUND, + json={"error": f"{file_id} not found!"}, + ) + + respx_mocker.get( + re.compile(r"/files/(?P[^/]+)"), name="get_file_dowload_link" + ).mock(side_effect=_create_download_link) + + respx_mocker.get( + "/", + name="healthcheck", + ).respond(status.HTTP_200_OK, json={"message": "ok"}) + respx_mocker.get("", name="base_endpoint").respond( + status.HTTP_200_OK, json={"message": "root entrypoint"} ) - return respx_mocker + + yield respx_mocker diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/rest/datasets.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/rest/datasets.py index bdfb37cf7859..2090a22938c6 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/api/rest/datasets.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/api/rest/datasets.py @@ -56,6 +56,31 @@ async def list_datasets( return create_page(datasets, total=total, params=params) # type: ignore[return-value] +@router.get( + "/datasets/{dataset_id}", + status_code=status.HTTP_200_OK, + response_model=DatasetMetaData, +) +@cancel_on_disconnect +async def get_dataset( + request: Request, + x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")], + x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")], + pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)], + params: Annotated[Params, Depends()], + dataset_id: str, +) -> DatasetMetaData: + assert request # nosec + raw_params: RawParams = resolve_params(params).to_raw_params() + assert raw_params.limit is not None # nosec + assert raw_params.offset is not None # nosec + return await pennsieve_client.get_dataset( + api_key=x_datcore_api_key, + api_secret=x_datcore_api_secret, + dataset_id=dataset_id, + ) + + @router.get( "/datasets/{dataset_id}/files", summary="list top level files/folders in a dataset", diff --git a/services/datcore-adapter/src/simcore_service_datcore_adapter/modules/pennsieve.py b/services/datcore-adapter/src/simcore_service_datcore_adapter/modules/pennsieve.py index 781d86f4916c..d1189f6c76c6 100644 --- a/services/datcore-adapter/src/simcore_service_datcore_adapter/modules/pennsieve.py +++ b/services/datcore-adapter/src/simcore_service_datcore_adapter/modules/pennsieve.py @@ -14,6 +14,7 @@ DataType, FileMetaData, ) +from pydantic import ByteSize from servicelib.logging_utils import log_context from servicelib.utils import logged_gather from starlette import status @@ -81,9 +82,9 @@ class PennsieveAuthorizationHeaders(TypedDict): Authorization: str -_TTL_CACHE_AUTHORIZATION_HEADERS_SECONDS: Final[ - int -] = 3530 # NOTE: observed while developing this code, pennsieve authorizes 3600 seconds, so we cache a bit less +_TTL_CACHE_AUTHORIZATION_HEADERS_SECONDS: Final[int] = ( + 3530 # NOTE: observed while developing this code, pennsieve authorizes 3600 seconds, so we cache a bit less +) ExpirationTimeSecs = int @@ -346,12 +347,25 @@ async def list_datasets( DatasetMetaData( id=d["content"]["id"], display_name=d["content"]["name"], + size=ByteSize(d["storage"]) if d["storage"] > 0 else None, ) for d in dataset_page["datasets"] ], dataset_page["totalCount"], ) + async def get_dataset( + self, api_key: str, api_secret: str, dataset_id: str + ) -> DatasetMetaData: + dataset_pck = await self._get_dataset(api_key, api_secret, dataset_id) + return DatasetMetaData( + id=dataset_pck["content"]["id"], + display_name=dataset_pck["content"]["name"], + size=( + ByteSize(dataset_pck["storage"]) if dataset_pck["storage"] > 0 else None + ), + ) + async def list_packages_in_dataset( self, api_key: str, diff --git a/services/datcore-adapter/tests/unit/conftest.py b/services/datcore-adapter/tests/unit/conftest.py index be4e44d726f7..6090efe85ae2 100644 --- a/services/datcore-adapter/tests/unit/conftest.py +++ b/services/datcore-adapter/tests/unit/conftest.py @@ -231,7 +231,13 @@ def pennsieve_random_fake_datasets( ) -> dict[str, Any]: return { "datasets": [ - {"content": {"id": create_pennsieve_fake_dataset_id(), "name": fake.text()}} + { + "content": { + "id": create_pennsieve_fake_dataset_id(), + "name": fake.text(), + }, + "storage": fake.pyint(), + } for _ in range(10) ], "totalCount": 20, @@ -308,7 +314,11 @@ async def pennsieve_subsystem_mock( ).respond( status.HTTP_200_OK, json={ - "content": {"name": "Some dataset name that is awesome"}, + "content": { + "name": "Some dataset name that is awesome", + "id": pennsieve_dataset_id, + }, + "storage": fake.pyint(), "children": pennsieve_mock_dataset_packages["packages"], }, ) diff --git a/services/datcore-adapter/tests/unit/test_route_datasets.py b/services/datcore-adapter/tests/unit/test_route_datasets.py index 1bfd55269fcf..913cb578a966 100644 --- a/services/datcore-adapter/tests/unit/test_route_datasets.py +++ b/services/datcore-adapter/tests/unit/test_route_datasets.py @@ -14,6 +14,23 @@ from starlette import status +async def test_get_dataset_entrypoint( + async_client: httpx.AsyncClient, + pennsieve_dataset_id: str, + pennsieve_subsystem_mock: respx.MockRouter | None, + pennsieve_api_headers: dict[str, str], +): + response = await async_client.get( + f"v0/datasets/{pennsieve_dataset_id}", + headers=pennsieve_api_headers, + ) + + assert response.status_code == status.HTTP_200_OK + data = response.json() + assert data + TypeAdapter(DatasetMetaData).validate_python(data) + + async def test_list_datasets_entrypoint( async_client: httpx.AsyncClient, pennsieve_subsystem_mock: respx.MockRouter | None, diff --git a/services/docker-compose.yml b/services/docker-compose.yml index a7cd2c549701..03bb9a865934 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -1153,8 +1153,6 @@ services: init: true hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}" environment: - BF_API_KEY: ${BF_API_KEY} - BF_API_SECRET: ${BF_API_SECRET} DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} diff --git a/services/storage/openapi.json b/services/storage/openapi.json index 73ddcf302d28..2c5676b08bc5 100644 --- a/services/storage/openapi.json +++ b/services/storage/openapi.json @@ -1078,6 +1078,69 @@ } } }, + "/v0/locations/{location_id}/paths/{path}:size": { + "post": { + "tags": [ + "files" + ], + "summary": "Compute Path Size", + "operationId": "compute_path_size_v0_locations__location_id__paths__path__size_post", + "parameters": [ + { + "name": "path", + "in": "path", + "required": true, + "schema": { + "type": "string", + "format": "path", + "title": "Path" + } + }, + { + "name": "location_id", + "in": "path", + "required": true, + "schema": { + "type": "integer", + "title": "Location Id" + } + }, + { + "name": "user_id", + "in": "query", + "required": true, + "schema": { + "type": "integer", + "exclusiveMinimum": true, + "title": "User Id", + "minimum": 0 + } + } + ], + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Envelope_PathTotalSizeCreate_" + } + } + } + }, + "422": { + "description": "Validation Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + } + } + } + } + }, "/v0/simcore-s3:access": { "post": { "tags": [ @@ -1748,6 +1811,31 @@ "type": "object", "title": "Envelope[HealthCheck]" }, + "Envelope_PathTotalSizeCreate_": { + "properties": { + "data": { + "anyOf": [ + { + "$ref": "#/components/schemas/PathTotalSizeCreate" + }, + { + "type": "null" + } + ] + }, + "error": { + "anyOf": [ + {}, + { + "type": "null" + } + ], + "title": "Error" + } + }, + "type": "object", + "title": "Envelope[PathTotalSizeCreate]" + }, "Envelope_S3Settings_": { "properties": { "data": { @@ -2465,6 +2553,27 @@ ], "title": "PathMetaDataGet" }, + "PathTotalSizeCreate": { + "properties": { + "path": { + "type": "string", + "format": "path", + "title": "Path" + }, + "size": { + "type": "integer", + "minimum": 0, + "title": "Size" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "path", + "size" + ], + "title": "PathTotalSizeCreate" + }, "S3Settings": { "properties": { "S3_ACCESS_KEY": { diff --git a/services/storage/src/simcore_service_storage/api/rest/_paths.py b/services/storage/src/simcore_service_storage/api/rest/_paths.py index 7cbd959259ee..bdb5a171f0cc 100644 --- a/services/storage/src/simcore_service_storage/api/rest/_paths.py +++ b/services/storage/src/simcore_service_storage/api/rest/_paths.py @@ -4,7 +4,11 @@ from fastapi import APIRouter, Depends from fastapi_pagination import create_page -from models_library.api_schemas_storage.storage_schemas import PathMetaDataGet +from models_library.api_schemas_storage.storage_schemas import ( + PathMetaDataGet, + PathTotalSizeCreate, +) +from models_library.generics import Envelope from models_library.users import UserID from servicelib.fastapi.rest_pagination import ( CustomizedPathsCursorPage, @@ -46,3 +50,19 @@ async def list_paths( params=page_params, next_=next_cursor, ) + + +@router.post( + "/locations/{location_id}/paths/{path:path}:size", + response_model=Envelope[PathTotalSizeCreate], +) +async def compute_path_size( + dsm: Annotated[BaseDataManager, Depends(get_data_manager)], + user_id: UserID, + path: Path, +): + return Envelope[PathTotalSizeCreate]( + data=PathTotalSizeCreate( + path=path, size=await dsm.compute_path_size(user_id, path=path) + ) + ) diff --git a/services/storage/src/simcore_service_storage/core/settings.py b/services/storage/src/simcore_service_storage/core/settings.py index 05711540b9a1..f2a5c62d16ea 100644 --- a/services/storage/src/simcore_service_storage/core/settings.py +++ b/services/storage/src/simcore_service_storage/core/settings.py @@ -36,13 +36,6 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): STORAGE_MONITORING_ENABLED: bool = False STORAGE_PROFILING: bool = False - BF_API_KEY: str | None = Field( - None, description="Pennsieve API key ONLY for testing purposes" - ) - BF_API_SECRET: str | None = Field( - None, description="Pennsieve API secret ONLY for testing purposes" - ) - STORAGE_POSTGRES: PostgresSettings | None = Field( json_schema_extra={"auto_default_from_env": True} ) diff --git a/services/storage/src/simcore_service_storage/datcore_dsm.py b/services/storage/src/simcore_service_storage/datcore_dsm.py index b1def97a6350..7d61d3f60627 100644 --- a/services/storage/src/simcore_service_storage/datcore_dsm.py +++ b/services/storage/src/simcore_service_storage/datcore_dsm.py @@ -5,6 +5,7 @@ import arrow from fastapi import FastAPI from models_library.api_schemas_storage.storage_schemas import ( + UNDEFINED_SIZE_TYPE, DatCoreCollectionName, DatCoreDatasetName, DatCorePackageName, @@ -187,6 +188,63 @@ async def list_paths( 1, ) + async def compute_path_size(self, user_id: UserID, *, path: Path) -> ByteSize: + """returns the total size of an arbitrary path""" + api_token, api_secret = await self._get_datcore_tokens(user_id) + api_token, api_secret = _check_api_credentials(api_token, api_secret) + + # if this is a dataset we might have the size directly + with contextlib.suppress(ValidationError): + dataset_id = TypeAdapter(DatCoreDatasetName).validate_python(f"{path}") + _, dataset_size = await datcore_adapter.get_dataset( + self.app, + api_key=api_token, + api_secret=api_secret, + dataset_id=dataset_id, + ) + if dataset_size is not None: + return dataset_size + + # generic computation (slow and unoptimized - could be improved if necessary by using datcore data better) + try: + accumulated_size = ByteSize(0) + paths_to_process = [path] + + while paths_to_process: + current_path = paths_to_process.pop() + paths, cursor, _ = await self.list_paths( + user_id, file_filter=current_path, cursor=None, limit=50 + ) + + while paths: + for p in paths: + if p.file_meta_data is not None: + # this is a file + assert ( + p.file_meta_data.file_size is not UNDEFINED_SIZE_TYPE + ) # nosec + assert isinstance( + p.file_meta_data.file_size, ByteSize + ) # nosec + accumulated_size = ByteSize( + accumulated_size + p.file_meta_data.file_size + ) + continue + paths_to_process.append(p.path) + + if cursor: + paths, cursor, _ = await self.list_paths( + user_id, file_filter=current_path, cursor=cursor, limit=50 + ) + else: + break + + return accumulated_size + + except ValidationError: + # invalid path + return ByteSize(0) + async def list_files( self, user_id: UserID, diff --git a/services/storage/src/simcore_service_storage/dsm_factory.py b/services/storage/src/simcore_service_storage/dsm_factory.py index 28cb4abd98a1..749bbf9a5e61 100644 --- a/services/storage/src/simcore_service_storage/dsm_factory.py +++ b/services/storage/src/simcore_service_storage/dsm_factory.py @@ -80,6 +80,10 @@ async def list_paths( ) -> tuple[list[PathMetaData], GenericCursor | None, TotalNumber | None]: """returns a page of the file meta data a user has access to""" + @abstractmethod + async def compute_path_size(self, user_id: UserID, *, path: Path) -> ByteSize: + """returns the total size of an arbitrary path""" + @abstractmethod async def get_file(self, user_id: UserID, file_id: StorageFileID) -> FileMetaData: """returns the file meta data of file_id if user_id has the rights to""" diff --git a/services/storage/src/simcore_service_storage/exceptions/handlers.py b/services/storage/src/simcore_service_storage/exceptions/handlers.py index 3c1d0bfa0061..78a93d8f46c8 100644 --- a/services/storage/src/simcore_service_storage/exceptions/handlers.py +++ b/services/storage/src/simcore_service_storage/exceptions/handlers.py @@ -9,6 +9,7 @@ ) from ..modules.datcore_adapter.datcore_adapter_exceptions import ( + DatcoreAdapterFileNotFoundError, DatcoreAdapterTimeoutError, ) from .errors import ( @@ -42,6 +43,7 @@ def set_exception_handlers(app: FastAPI) -> None: FileMetaDataNotFoundError, S3KeyNotFoundError, ProjectNotFoundError, + DatcoreAdapterFileNotFoundError, ): app.add_exception_handler( exc_not_found, diff --git a/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter.py b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter.py index 20a8300a8cde..0f5ec1dae124 100644 --- a/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter.py +++ b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter.py @@ -2,7 +2,7 @@ from typing import Any, TypeAlias, cast import httpx -from fastapi import FastAPI +from fastapi import FastAPI, status from fastapi_pagination import Page from models_library.api_schemas_datcore_adapter.datasets import ( DatasetMetaData as DatCoreDatasetMetaData, @@ -10,14 +10,16 @@ from models_library.api_schemas_datcore_adapter.datasets import ( FileMetaData as DatCoreFileMetaData, ) -from models_library.api_schemas_datcore_adapter.datasets import PackageMetaData +from models_library.api_schemas_datcore_adapter.datasets import ( + PackageMetaData, +) from models_library.api_schemas_storage.storage_schemas import ( DatCoreCollectionName, DatCoreDatasetName, DatCorePackageName, ) from models_library.users import UserID -from pydantic import AnyUrl, BaseModel, NonNegativeInt, TypeAdapter +from pydantic import AnyUrl, BaseModel, ByteSize, NonNegativeInt, TypeAdapter from servicelib.fastapi.client_session import get_client_session from servicelib.utils import logged_gather @@ -31,7 +33,11 @@ TotalNumber, ) from .datcore_adapter_client_utils import request, retrieve_all_pages -from .datcore_adapter_exceptions import DatcoreAdapterError +from .datcore_adapter_exceptions import ( + DatcoreAdapterError, + DatcoreAdapterFileNotFoundError, + DatcoreAdapterResponseError, +) from .utils import ( create_path_meta_data_from_datcore_fmd, create_path_meta_data_from_datcore_package, @@ -311,14 +317,44 @@ async def list_datasets( ) +async def get_dataset( + app: FastAPI, + *, + api_key: str, + api_secret: str, + dataset_id: DatCoreDatasetName, +) -> tuple[DatasetMetaData, ByteSize | None]: + response = await request( + app, + api_key, + api_secret, + "GET", + f"/datasets/{dataset_id}", + ) + assert isinstance(response, dict) # nosec + datcore_dataset = DatCoreDatasetMetaData(**response) + + return ( + DatasetMetaData( + dataset_id=datcore_dataset.id, display_name=datcore_dataset.display_name + ), + datcore_dataset.size, + ) + + async def get_file_download_presigned_link( app: FastAPI, api_key: str, api_secret: str, file_id: str ) -> AnyUrl: - file_download_data = cast( - dict[str, Any], - await request(app, api_key, api_secret, "GET", f"/files/{file_id}"), - ) - return TypeAdapter(AnyUrl).validate_python(file_download_data["link"]) + try: + file_download_data = cast( + dict[str, Any], + await request(app, api_key, api_secret, "GET", f"/files/{file_id}"), + ) + return TypeAdapter(AnyUrl).validate_python(file_download_data["link"]) + except DatcoreAdapterResponseError as exc: + if exc.status == status.HTTP_404_NOT_FOUND: + raise DatcoreAdapterFileNotFoundError(file_id=file_id) from exc + raise async def get_package_files( diff --git a/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_exceptions.py b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_exceptions.py index 2e2ccc8548ff..f4643380ab01 100644 --- a/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_exceptions.py +++ b/services/storage/src/simcore_service_storage/modules/datcore_adapter/datcore_adapter_exceptions.py @@ -44,3 +44,11 @@ def __init__(self, status: int, reason: str) -> None: super().__init__( msg=f"forwarded call failed with status {status}, reason {reason}" ) + + +class DatcoreAdapterFileNotFoundError(DatcoreAdapterError): + """special error to check the assumption that /packages/{package_id}/files returns only one file""" + + def __init__(self, file_id: str) -> None: + self.file_id = file_id + super().__init__(msg=f"file {file_id} not found!") diff --git a/services/storage/src/simcore_service_storage/modules/db/tokens.py b/services/storage/src/simcore_service_storage/modules/db/tokens.py index eaeb7feb8cba..c703de879cf6 100644 --- a/services/storage/src/simcore_service_storage/modules/db/tokens.py +++ b/services/storage/src/simcore_service_storage/modules/db/tokens.py @@ -7,7 +7,6 @@ from simcore_postgres_database.storage_models import tokens from sqlalchemy.ext.asyncio import AsyncEngine -from ...core.settings import get_application_settings from . import get_db_engine _logger = logging.getLogger(__name__) @@ -21,23 +20,17 @@ async def _get_tokens_from_db(engine: AsyncEngine, user_id: UserID) -> dict[str, ).where(tokens.c.user_id == user_id) ) row = result.one_or_none() - return dict(row) if row else {} + return row._asdict() if row else {} async def get_api_token_and_secret( app: FastAPI, user_id: UserID ) -> tuple[str | None, str | None]: - # from the client side together with the userid? engine = get_db_engine(app) - app_settings = get_application_settings(app) - # defaults from config if any, othewise None - api_token = app_settings.BF_API_KEY - api_secret = app_settings.BF_API_SECRET - data = await _get_tokens_from_db(engine, user_id) data = data.get("token_data", {}) - api_token = data.get("token_key", api_token) - api_secret = data.get("token_secret", api_secret) + api_token = data.get("token_key") + api_secret = data.get("token_secret") return api_token, api_secret diff --git a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py index 8bb08788ac31..f63e9e022799 100644 --- a/services/storage/src/simcore_service_storage/simcore_s3_dsm.py +++ b/services/storage/src/simcore_service_storage/simcore_s3_dsm.py @@ -34,7 +34,7 @@ StorageFileID, ) from models_library.users import UserID -from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter +from pydantic import AnyUrl, ByteSize, NonNegativeInt, TypeAdapter, ValidationError from servicelib.aiohttp.long_running_tasks.server import TaskProgress from servicelib.fastapi.client_session import get_client_session from servicelib.logging_utils import log_context @@ -81,6 +81,7 @@ from .utils.simcore_s3_dsm_utils import ( compute_file_id_prefix, expand_directory, + get_accessible_project_ids, get_directory_file_id, list_child_paths_from_repository, list_child_paths_from_s3, @@ -191,17 +192,9 @@ async def list_paths( project_id = ProjectID(file_filter.parts[0]) if file_filter else None async with self.engine.connect() as conn: - if project_id: - project_access_rights = await get_project_access_rights( - conn=conn, user_id=user_id, project_id=project_id - ) - if not project_access_rights.read: - raise ProjectAccessRightError( - access_right="read", project_id=project_id - ) - accessible_projects_ids = [project_id] - else: - accessible_projects_ids = await get_readable_project_ids(conn, user_id) + accessible_projects_ids = await get_accessible_project_ids( + conn, user_id=user_id, project_id=project_id + ) # check if the file_filter is a directory or inside one dir_fmd = None @@ -251,6 +244,69 @@ async def list_paths( return paths_metadata, next_cursor, total + async def compute_path_size(self, user_id: UserID, *, path: Path) -> ByteSize: + """returns the total size of an arbitrary path""" + # check access rights first + project_id = None + with contextlib.suppress(ValueError): + # NOTE: we currently do not support anything else than project_id/node_id/file_path here, sorry chap + project_id = ProjectID(path.parts[0]) + async with self.engine.connect() as conn: + accessible_projects_ids = await get_accessible_project_ids( + conn, user_id=user_id, project_id=project_id + ) + + # use-cases: + # 1. path is not a valid StorageFileID (e.g. a project or project/node) --> all entries are in the DB (files and folder) + # 2. path is valid StorageFileID and not in the DB --> entries are only in S3 + # 3. path is valid StorageFileID and in the DB --> return directly from the DB + + use_db_data = True + with contextlib.suppress(ValidationError): + file_id: StorageFileID = TypeAdapter(StorageFileID).validate_python( + f"{path}" + ) + # path is a valid StorageFileID + async with self.engine.connect() as conn: + if ( + dir_fmd := await file_meta_data.try_get_directory(conn, path) + ) and dir_fmd.file_id != file_id: + # this is pure S3 aka use-case 2 + use_db_data = False + + if not use_db_data: + assert file_id # nosec + s3_metadata = await get_s3_client(self.app).get_directory_metadata( + bucket=self.simcore_bucket_name, prefix=file_id + ) + assert s3_metadata.size # nosec + return s3_metadata.size + + # all other use-cases are in the DB + async with self.engine.connect() as conn: + fmds = await file_meta_data.list_filter_with_partial_file_id( + conn, + user_or_project_filter=UserOrProjectFilter( + user_id=user_id, project_ids=accessible_projects_ids + ), + file_id_prefix=f"{path}", + partial_file_id=None, + sha256_checksum=None, + is_directory=None, + ) + + # ensure file sizes are uptodate + updated_fmds = [] + for metadata in fmds: + if is_file_entry_valid(metadata): + updated_fmds.append(convert_db_to_model(metadata)) + continue + updated_fmds.append( + convert_db_to_model(await self._update_database_from_storage(metadata)) + ) + + return ByteSize(sum(fmd.file_size for fmd in updated_fmds)) + async def list_files( self, user_id: UserID, @@ -711,9 +767,9 @@ async def deep_copy_project_simcore_s3( task_progress, f"Collecting files of '{src_project['name']}'..." ) async with self.engine.connect() as conn: - src_project_files: list[ - FileMetaDataAtDB - ] = await file_meta_data.list_fmds(conn, project_ids=[src_project_uuid]) + src_project_files: list[FileMetaDataAtDB] = ( + await file_meta_data.list_fmds(conn, project_ids=[src_project_uuid]) + ) with log_context( _logger, @@ -827,19 +883,19 @@ async def search_owned_files( offset: int | None = None, ) -> list[FileMetaData]: async with self.engine.connect() as conn: - file_metadatas: list[ - FileMetaDataAtDB - ] = await file_meta_data.list_filter_with_partial_file_id( - conn, - user_or_project_filter=UserOrProjectFilter( - user_id=user_id, project_ids=[] - ), - file_id_prefix=file_id_prefix, - partial_file_id=None, - is_directory=False, - sha256_checksum=sha256_checksum, - limit=limit, - offset=offset, + file_metadatas: list[FileMetaDataAtDB] = ( + await file_meta_data.list_filter_with_partial_file_id( + conn, + user_or_project_filter=UserOrProjectFilter( + user_id=user_id, project_ids=[] + ), + file_id_prefix=file_id_prefix, + partial_file_id=None, + is_directory=False, + sha256_checksum=sha256_checksum, + limit=limit, + offset=offset, + ) ) resolved_fmds = [] for fmd in file_metadatas: diff --git a/services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py b/services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py index 2243caea7dea..253a7a241d88 100644 --- a/services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py +++ b/services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py @@ -10,13 +10,18 @@ SimcoreS3FileID, StorageFileID, ) +from models_library.users import UserID from pydantic import ByteSize, NonNegativeInt, TypeAdapter from servicelib.utils import ensure_ends_with from sqlalchemy.ext.asyncio import AsyncConnection -from ..exceptions.errors import FileMetaDataNotFoundError +from ..exceptions.errors import FileMetaDataNotFoundError, ProjectAccessRightError from ..models import FileMetaData, FileMetaDataAtDB, GenericCursor, PathMetaData from ..modules.db import file_meta_data +from ..modules.db.access_layer import ( + get_project_access_rights, + get_readable_project_ids, +) from .utils import convert_db_to_model @@ -207,3 +212,16 @@ async def list_child_paths_from_repository( ) return paths_metadata, next_cursor, total + + +async def get_accessible_project_ids( + conn: AsyncConnection, *, user_id: UserID, project_id: ProjectID | None +) -> list[ProjectID]: + if project_id: + project_access_rights = await get_project_access_rights( + conn=conn, user_id=user_id, project_id=project_id + ) + if not project_access_rights.read: + raise ProjectAccessRightError(access_right="read", project_id=project_id) + return [project_id] + return await get_readable_project_ids(conn, user_id) diff --git a/services/storage/tests/conftest.py b/services/storage/tests/conftest.py index fd0181a0997b..b766c7655211 100644 --- a/services/storage/tests/conftest.py +++ b/services/storage/tests/conftest.py @@ -57,9 +57,11 @@ from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.aiohttp import status from servicelib.utils import limited_gather +from simcore_postgres_database.models.tokens import tokens from simcore_postgres_database.storage_models import file_meta_data, projects, users from simcore_service_storage.core.application import create_app from simcore_service_storage.core.settings import ApplicationSettings +from simcore_service_storage.datcore_dsm import DatCoreDataManager from simcore_service_storage.dsm import get_dsm_provider from simcore_service_storage.models import FileMetaData, FileMetaDataAtDB, S3BucketName from simcore_service_storage.modules.long_running_tasks import ( @@ -80,6 +82,7 @@ "pytest_simcore.aws_s3_service", "pytest_simcore.aws_server", "pytest_simcore.cli_runner", + "pytest_simcore.disk_usage_monitoring", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", "pytest_simcore.environment_configs", @@ -242,11 +245,11 @@ def simcore_file_id( @pytest.fixture( params=[ SimcoreS3DataManager.get_location_id(), - # DatCoreDataManager.get_location_id(), + DatCoreDataManager.get_location_id(), ], ids=[ SimcoreS3DataManager.get_location_name(), - # DatCoreDataManager.get_location_name(), + DatCoreDataManager.get_location_name(), ], ) def location_id(request: pytest.FixtureRequest) -> LocationID: @@ -855,7 +858,10 @@ async def with_random_project_with_files( ], ], project_params: ProjectWithFilesParams, -) -> tuple[dict[str, Any], dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],]: +) -> tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], +]: return await random_project_with_files(project_params) @@ -894,3 +900,34 @@ async def output_file( result = await conn.execute( file_meta_data.delete().where(file_meta_data.c.file_id == row.file_id) ) + + +@pytest.fixture +async def fake_datcore_tokens( + user_id: UserID, sqlalchemy_async_engine: AsyncEngine, faker: Faker +) -> AsyncIterator[tuple[str, str]]: + token_key = cast(str, faker.uuid4()) + token_secret = cast(str, faker.uuid4()) + created_token_ids = [] + async with sqlalchemy_async_engine.begin() as conn: + result = await conn.execute( + tokens.insert() + .values( + user_id=user_id, + token_service="pytest", # noqa: S106 + token_data={ + "service": "pytest", + "token_secret": token_secret, + "token_key": token_key, + }, + ) + .returning(tokens.c.token_id) + ) + row = result.one() + created_token_ids.append(row.token_id) + yield token_key, token_secret + + async with sqlalchemy_async_engine.begin() as conn: + await conn.execute( + tokens.delete().where(tokens.c.token_id.in_(created_token_ids)) + ) diff --git a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py index bdb11cf8f1df..f66443d96819 100644 --- a/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py +++ b/services/storage/tests/unit/test__legacy_storage_sdk_compatibility.py @@ -12,16 +12,16 @@ """ import logging +from collections.abc import AsyncIterator from pathlib import Path from threading import Thread -from typing import AsyncIterator import aiohttp import httpx import pytest import uvicorn from faker import Faker -from models_library.projects_nodes_io import LocationID, SimcoreS3FileID +from models_library.projects_nodes_io import SimcoreS3FileID from models_library.users import UserID from pytest_simcore.helpers.logging_tools import log_context from servicelib.utils import unused_port @@ -112,21 +112,22 @@ def file_id(simcore_file_id: SimcoreS3FileID) -> str: @pytest.fixture -def location_id() -> LocationID: - return SimcoreS3DataManager.get_location_id() - - -@pytest.fixture -def location_name() -> str: +def simcore_location_name() -> str: return SimcoreS3DataManager.get_location_name() +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_storage_client_used_in_simcore_sdk_0_3_2( real_storage_server: URL, str_user_id: str, file_id: str, location_id: int, - location_name: str, + simcore_location_name: str, tmp_path: Path, faker: Faker, ): @@ -211,7 +212,7 @@ async def test_storage_client_used_in_simcore_sdk_0_3_2( resp_model = await api.get_storage_locations(user_id=str_user_id) print(f"{resp_model=}") for location in resp_model.data: - assert location["name"] == location_name + assert location["name"] == simcore_location_name assert location["id"] == location_id # _get_download_link diff --git a/services/storage/tests/unit/test_db_data_export.py b/services/storage/tests/unit/test_db_data_export.py index 1c9d3b01edbc..717231787499 100644 --- a/services/storage/tests/unit/test_db_data_export.py +++ b/services/storage/tests/unit/test_db_data_export.py @@ -31,6 +31,7 @@ from simcore_service_storage.api.rpc._async_jobs import AsyncJobNameData from simcore_service_storage.api.rpc._data_export import AccessRightError from simcore_service_storage.core.settings import ApplicationSettings +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager pytest_plugins = [ "pytest_simcore.rabbit_service", @@ -86,6 +87,12 @@ class UserWithFile(NamedTuple): file: Path +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "project_params,_type", [ @@ -117,10 +124,9 @@ async def test_start_data_export_success( user_id: UserID, _type: Literal["file", "folder"], ): - _, list_of_files = with_random_project_with_files workspace_files = [ - p for p in list(list_of_files.values())[0].keys() if "/workspace/" in p + p for p in next(iter(list_of_files.values())) if "/workspace/" in p ] assert len(workspace_files) > 0 file_or_folder_id: SimcoreS3FileID @@ -152,7 +158,6 @@ async def test_start_data_export_success( async def test_start_data_export_fail( rpc_client: RabbitMQRPCClient, user_id: UserID, faker: Faker ): - with pytest.raises(AccessRightError): _ = await async_jobs.submit_job( rpc_client, diff --git a/services/storage/tests/unit/test_dsm_dsmcleaner.py b/services/storage/tests/unit/test_dsm_dsmcleaner.py index 6144571e3a37..98eef5c62999 100644 --- a/services/storage/tests/unit/test_dsm_dsmcleaner.py +++ b/services/storage/tests/unit/test_dsm_dsmcleaner.py @@ -213,6 +213,12 @@ async def test_clean_expired_uploads_deletes_expired_pending_uploads( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -389,9 +395,9 @@ async def test_clean_expired_uploads_does_not_clean_multipart_upload_on_creation assert len(started_multipart_uploads_upload_id) == len(file_ids_to_upload) # ensure we have now an upload id - all_ongoing_uploads: list[ - tuple[UploadID, SimcoreS3FileID] - ] = await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) + all_ongoing_uploads: list[tuple[UploadID, SimcoreS3FileID]] = ( + await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) + ) assert len(all_ongoing_uploads) == len(file_ids_to_upload) for ongoing_upload_id, ongoing_file_id in all_ongoing_uploads: diff --git a/services/storage/tests/unit/test_handlers_datasets.py b/services/storage/tests/unit/test_handlers_datasets.py index 9bdbc743772b..5808a63f1f1b 100644 --- a/services/storage/tests/unit/test_handlers_datasets.py +++ b/services/storage/tests/unit/test_handlers_datasets.py @@ -4,7 +4,6 @@ # pylint:disable=too-many-arguments # pylint:disable=no-name-in-module - from collections.abc import Awaitable, Callable from pathlib import Path @@ -17,7 +16,7 @@ FileMetaDataGet, ) from models_library.projects import ProjectID -from models_library.projects_nodes_io import SimcoreS3FileID +from models_library.projects_nodes_io import LocationID, SimcoreS3FileID from models_library.users import UserID from pydantic import ByteSize from pytest_mock import MockerFixture @@ -28,6 +27,7 @@ parametrized_file_size, ) from servicelib.aiohttp import status +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] @@ -38,7 +38,8 @@ async def test_list_dataset_files_metadata_with_no_files_returns_empty_array( client: AsyncClient, user_id: UserID, project_id: ProjectID, - location_id: int, + location_id: LocationID, + fake_datcore_tokens: tuple[str, str], ): url = url_from_operation_id( client, @@ -54,6 +55,12 @@ async def test_list_dataset_files_metadata_with_no_files_returns_empty_array( assert not error +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [parametrized_file_size("100Mib")], @@ -65,7 +72,7 @@ async def test_list_dataset_files_metadata( client: AsyncClient, user_id: UserID, project_id: ProjectID, - location_id: int, + location_id: LocationID, file_size: ByteSize, faker: Faker, ): @@ -94,11 +101,17 @@ async def test_list_dataset_files_metadata( assert fmd.file_size == file.stat().st_size +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_list_datasets_metadata( initialized_app: FastAPI, client: AsyncClient, user_id: UserID, - location_id: int, + location_id: LocationID, project_id: ProjectID, ): url = url_from_operation_id( @@ -119,13 +132,19 @@ async def test_list_datasets_metadata( assert dataset.dataset_id == project_id +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_ensure_expand_dirs_defaults_true( mocker: MockerFixture, initialized_app: FastAPI, client: AsyncClient, user_id: UserID, project_id: ProjectID, - location_id: int, + location_id: LocationID, ): mocked_object = mocker.patch( "simcore_service_storage.simcore_s3_dsm.SimcoreS3DataManager.list_files_in_dataset", diff --git a/services/storage/tests/unit/test_handlers_datcore.py b/services/storage/tests/unit/test_handlers_datcore.py new file mode 100644 index 000000000000..a71626100317 --- /dev/null +++ b/services/storage/tests/unit/test_handlers_datcore.py @@ -0,0 +1,48 @@ +import httpx +import pytest +from fastapi import FastAPI, status +from models_library.projects_nodes_io import LocationID +from models_library.users import UserID +from pytest_simcore.helpers.fastapi import url_from_operation_id +from pytest_simcore.helpers.httpx_assert_checks import assert_status +from simcore_service_storage.datcore_dsm import DatCoreDataManager + +pytest_simcore_core_services_selection = ["postgres"] +pytest_simcore_ops_services_selection = ["adminer"] + + +@pytest.mark.parametrize( + "entrypoint", + [ + "list_datasets_metadata", + # "list_dataset_files_metadata", needs dataset_id + "list_files_metadata", + # "get_file_metadata", needs file_id + # "download_file", needs file_id + "list_paths", + ], +) +@pytest.mark.parametrize( + "location_id", + [DatCoreDataManager.get_location_id()], + ids=[DatCoreDataManager.get_location_name()], + indirect=True, +) +async def test_entrypoint_without_api_tokens_return_401( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + entrypoint: str, + user_id: UserID, +): + url = url_from_operation_id( + client, initialized_app, entrypoint, location_id=f"{location_id}" + ).with_query( + user_id=user_id, + ) + response = await client.get(f"{url}") + assert_status( + response, + status.HTTP_401_UNAUTHORIZED, + None, + ) diff --git a/services/storage/tests/unit/test_handlers_files.py b/services/storage/tests/unit/test_handlers_files.py index 1e76da434c19..3304e226dd9c 100644 --- a/services/storage/tests/unit/test_handlers_files.py +++ b/services/storage/tests/unit/test_handlers_files.py @@ -58,6 +58,7 @@ from simcore_service_storage.modules.long_running_tasks import ( get_completed_upload_tasks, ) +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager from sqlalchemy.ext.asyncio import AsyncEngine from tenacity.asyncio import AsyncRetrying from tenacity.retry import retry_if_exception_type @@ -87,9 +88,9 @@ async def assert_multipart_uploads_in_progress( expected_upload_ids: list[str] | None, ): """if None is passed, then it checks that no uploads are in progress""" - list_uploads: list[ - tuple[UploadID, S3ObjectKey] - ] = await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) + list_uploads: list[tuple[UploadID, S3ObjectKey]] = ( + await storage_s3_client.list_ongoing_multipart_uploads(bucket=storage_s3_bucket) + ) if expected_upload_ids is None: assert ( not list_uploads @@ -109,6 +110,12 @@ class SingleLinkParam: expected_chunk_size: ByteSize +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "single_link_param", [ @@ -238,6 +245,12 @@ async def _link_creator(file_id: SimcoreS3FileID, **query_kwargs) -> PresignedLi await asyncio.gather(*clean_tasks) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "single_link_param", [ @@ -319,6 +332,12 @@ class MultiPartParam: expected_chunk_size: ByteSize +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "test_param", [ @@ -417,6 +436,12 @@ async def test_create_upload_file_presigned_with_file_size_returns_multipart_lin ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "link_type, file_size", [ @@ -481,6 +506,12 @@ async def test_delete_unuploaded_file_correctly_cleans_up_db_and_s3( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "link_type, file_size", [ @@ -567,6 +598,12 @@ def complex_file_name(faker: Faker) -> str: return f"subfolder_1/sub_folder 2/some file name with spaces and special characters -_ü!öäàé+|}} {{3245_{faker.file_name()}" +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -586,6 +623,12 @@ async def test_upload_real_file( await upload_file(file_size, complex_file_name) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -688,6 +731,12 @@ async def test_upload_real_file_with_emulated_storage_restart_after_completion_w assert s3_metadata.e_tag == completion_etag +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_upload_of_single_presigned_link_lazily_update_database_on_get( sqlalchemy_async_engine: AsyncEngine, storage_s3_client: SimcoreS3API, @@ -731,6 +780,12 @@ async def test_upload_of_single_presigned_link_lazily_update_database_on_get( assert received_fmd.entity_tag == upload_e_tag +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_upload_real_file_with_s3_client( sqlalchemy_async_engine: AsyncEngine, storage_s3_client: SimcoreS3API, @@ -830,6 +885,12 @@ async def test_upload_real_file_with_s3_client( assert s3_metadata.e_tag == completion_etag +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -930,12 +991,13 @@ async def _assert_file_downloaded( async def test_download_file_no_file_was_uploaded( initialized_app: FastAPI, client: httpx.AsyncClient, - location_id: int, + location_id: LocationID, project_id: ProjectID, node_id: NodeID, user_id: UserID, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, + fake_datcore_tokens: tuple[str, str], ): missing_file = TypeAdapter(SimcoreS3FileID).validate_python( f"{project_id}/{node_id}/missing.file" @@ -961,12 +1023,18 @@ async def test_download_file_no_file_was_uploaded( assert missing_file in error["errors"][0] +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_download_file_1_to_1_with_file_meta_data( initialized_app: FastAPI, client: httpx.AsyncClient, file_size: ByteSize, upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], - location_id: int, + location_id: LocationID, user_id: UserID, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, @@ -1001,11 +1069,17 @@ async def test_download_file_1_to_1_with_file_meta_data( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_download_file_from_inside_a_directory( initialized_app: FastAPI, client: httpx.AsyncClient, file_size: ByteSize, - location_id: int, + location_id: LocationID, user_id: UserID, project_id: ProjectID, node_id: NodeID, @@ -1063,10 +1137,16 @@ async def test_download_file_from_inside_a_directory( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_download_file_the_file_is_missing_from_the_directory( initialized_app: FastAPI, client: httpx.AsyncClient, - location_id: int, + location_id: LocationID, user_id: UserID, project_id: ProjectID, node_id: NodeID, @@ -1096,10 +1176,16 @@ async def test_download_file_the_file_is_missing_from_the_directory( assert missing_s3_file_id in error["errors"][0] +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_download_file_access_rights( initialized_app: FastAPI, client: httpx.AsyncClient, - location_id: int, + location_id: LocationID, user_id: UserID, storage_s3_client: SimcoreS3API, storage_s3_bucket: S3BucketName, @@ -1129,6 +1215,12 @@ async def test_download_file_access_rights( assert "Insufficient access rights" in error["errors"][0] +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "file_size", [ @@ -1144,7 +1236,7 @@ async def test_delete_file( client: httpx.AsyncClient, file_size: ByteSize, upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], - location_id: int, + location_id: LocationID, user_id: UserID, faker: Faker, ): @@ -1177,6 +1269,12 @@ async def test_delete_file( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_copy_as_soft_link( initialized_app: FastAPI, client: httpx.AsyncClient, @@ -1275,6 +1373,12 @@ async def _list_files_and_directories( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize("link_type", LinkType) @pytest.mark.parametrize( "file_size", @@ -1316,12 +1420,18 @@ async def test_is_directory_link_forces_link_type_and_size( assert files_and_directories[0].file_size == 0 +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_ensure_expand_dirs_defaults_true( initialized_app: FastAPI, mocker: MockerFixture, client: httpx.AsyncClient, user_id: UserID, - location_id: int, + location_id: LocationID, ): mocked_object = mocker.patch( "simcore_service_storage.simcore_s3_dsm.SimcoreS3DataManager.list_files", @@ -1342,6 +1452,12 @@ async def test_ensure_expand_dirs_defaults_true( assert call_args_list.kwargs["expand_dirs"] is True +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_upload_file_is_directory_and_remove_content( initialized_app: FastAPI, create_empty_directory: Callable[ @@ -1451,6 +1567,12 @@ async def test_upload_file_is_directory_and_remove_content( assert len(files_and_directories) == 0 +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize("files_count", [1002]) async def test_listing_more_than_1000_objects_in_bucket( create_directory_with_files: Callable[ @@ -1483,6 +1605,12 @@ async def test_listing_more_than_1000_objects_in_bucket( assert len(list_of_files) == 1000 +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize("uuid_filter", [True, False]) @pytest.mark.parametrize( "project_params", diff --git a/services/storage/tests/unit/test_handlers_files_metadata.py b/services/storage/tests/unit/test_handlers_files_metadata.py index 4c70256a5748..dd8bd4a2728d 100644 --- a/services/storage/tests/unit/test_handlers_files_metadata.py +++ b/services/storage/tests/unit/test_handlers_files_metadata.py @@ -2,7 +2,6 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument - from collections.abc import Awaitable, Callable from copy import deepcopy from pathlib import Path @@ -18,11 +17,13 @@ SimcoreS3FileID, ) from models_library.projects import ProjectID +from models_library.projects_nodes_io import LocationID from models_library.users import UserID from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.fastapi import url_from_operation_id from pytest_simcore.helpers.httpx_assert_checks import assert_status from servicelib.aiohttp import status +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager from yarl import URL pytest_simcore_core_services_selection = ["postgres"] @@ -37,10 +38,15 @@ async def __call__( read: bool, write: bool, delete: bool, - ) -> None: - ... + ) -> None: ... +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_list_files_metadata( upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], create_project_access_rights: CreateProjectAccessRightsCallable, @@ -48,7 +54,7 @@ async def test_list_files_metadata( client: httpx.AsyncClient, user_id: UserID, other_user_id: UserID, - location_id: int, + location_id: LocationID, project_id: ProjectID, faker: Faker, ): @@ -142,7 +148,7 @@ async def test_get_file_metadata_is_legacy_services_compatible( initialized_app: FastAPI, client: httpx.AsyncClient, user_id: UserID, - location_id: int, + location_id: LocationID, simcore_file_id: SimcoreS3FileID, ): url = ( @@ -162,12 +168,18 @@ async def test_get_file_metadata_is_legacy_services_compatible( assert response.status_code == status.HTTP_404_NOT_FOUND +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_get_file_metadata( upload_file: Callable[[ByteSize, str], Awaitable[tuple[Path, SimcoreS3FileID]]], initialized_app: FastAPI, client: httpx.AsyncClient, user_id: UserID, - location_id: int, + location_id: LocationID, project_id: ProjectID, simcore_file_id: SimcoreS3FileID, faker: Faker, diff --git a/services/storage/tests/unit/test_handlers_locations.py b/services/storage/tests/unit/test_handlers_locations.py index 04eed7b85ebf..4aae75e69de7 100644 --- a/services/storage/tests/unit/test_handlers_locations.py +++ b/services/storage/tests/unit/test_handlers_locations.py @@ -7,27 +7,53 @@ import httpx from fastapi import FastAPI, status +from models_library.api_schemas_storage.storage_schemas import FileLocation from models_library.users import UserID from pytest_simcore.helpers.fastapi import url_from_operation_id -from pytest_simcore.helpers.storage_utils import has_datcore_tokens +from pytest_simcore.helpers.httpx_assert_checks import assert_status +from simcore_service_storage.datcore_dsm import DatCoreDataManager +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager pytest_simcore_core_services_selection = ["postgres"] pytest_simcore_ops_services_selection = ["adminer"] async def test_locations( - initialized_app: FastAPI, client: httpx.AsyncClient, user_id: UserID + initialized_app: FastAPI, + client: httpx.AsyncClient, + user_id: UserID, + fake_datcore_tokens: tuple[str, str], ): url = url_from_operation_id( client, initialized_app, "list_storage_locations" ).with_query(user_id=user_id) - resp = await client.get(f"{url}") - - payload = resp.json() - assert resp.status_code == status.HTTP_200_OK, str(payload) - - data, error = tuple(payload.get(k) for k in ("data", "error")) - - _locs = 2 if has_datcore_tokens() else 1 - assert len(data) == _locs - assert not error + response = await client.get(f"{url}") + data, _ = assert_status(response, status.HTTP_200_OK, list[FileLocation]) + assert data + assert len(data) == 2 + assert data[0] == FileLocation( + id=SimcoreS3DataManager.get_location_id(), + name=SimcoreS3DataManager.get_location_name(), + ) + assert data[1] == FileLocation( + id=DatCoreDataManager.get_location_id(), + name=DatCoreDataManager.get_location_name(), + ) + + +async def test_locations_without_tokens( + initialized_app: FastAPI, + client: httpx.AsyncClient, + user_id: UserID, +): + url = url_from_operation_id( + client, initialized_app, "list_storage_locations" + ).with_query(user_id=user_id) + response = await client.get(f"{url}") + data, _ = assert_status(response, status.HTTP_200_OK, list[FileLocation]) + assert data + assert len(data) == 1 + assert data[0] == FileLocation( + id=SimcoreS3DataManager.get_location_id(), + name=SimcoreS3DataManager.get_location_name(), + ) diff --git a/services/storage/tests/unit/test_handlers_paths.py b/services/storage/tests/unit/test_handlers_paths.py index cf53b25542d2..31cb5c850617 100644 --- a/services/storage/tests/unit/test_handlers_paths.py +++ b/services/storage/tests/unit/test_handlers_paths.py @@ -16,9 +16,13 @@ import httpx import pytest import sqlalchemy as sa +from faker import Faker from fastapi import FastAPI, status from fastapi_pagination.cursor import CursorPage -from models_library.api_schemas_storage.storage_schemas import PathMetaDataGet +from models_library.api_schemas_storage.storage_schemas import ( + PathMetaDataGet, + PathTotalSizeCreate, +) from models_library.api_schemas_webserver.storage import MAX_NUMBER_OF_PATHS_PER_PAGE from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID from models_library.users import UserID @@ -27,6 +31,7 @@ from pytest_simcore.helpers.httpx_assert_checks import assert_status from pytest_simcore.helpers.storage_utils import FileIDDict, ProjectWithFilesParams from simcore_postgres_database.models.projects import projects +from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager from sqlalchemy.ext.asyncio import AsyncEngine pytest_simcore_core_services_selection = ["postgres"] @@ -116,6 +121,7 @@ async def test_list_paths_root_folder_of_empty_returns_nothing( client: httpx.AsyncClient, location_id: LocationID, user_id: UserID, + fake_datcore_tokens: tuple[str, str], ): await _assert_list_paths( initialized_app, @@ -127,6 +133,12 @@ async def test_list_paths_root_folder_of_empty_returns_nothing( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "project_params", [ @@ -204,6 +216,12 @@ async def test_list_paths_pagination( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "project_params", [ @@ -246,6 +264,12 @@ async def test_list_paths_pagination_large_page( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "project_params, num_projects", [ @@ -408,6 +432,12 @@ async def test_list_paths( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "project_params", [ @@ -538,3 +568,183 @@ async def test_list_paths_with_display_name_containing_slashes( assert page_of_paths.items[0].display_path == Path( expected_display_path ), "display path parts should be url encoded" + + +async def _assert_compute_path_size( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + *, + path: Path, + expected_total_size: int, +) -> ByteSize: + url = url_from_operation_id( + client, + initialized_app, + "compute_path_size", + location_id=f"{location_id}", + path=f"{path}", + ).with_query(user_id=user_id) + response = await client.post(f"{url}") + + received, _ = assert_status( + response, + status.HTTP_200_OK, + PathTotalSizeCreate, + ) + assert received + assert received.path == path + assert received.size == expected_total_size + return received.size + + +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=5, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("1b"),), + workspace_files_count=10, + ) + ], + ids=str, +) +async def test_path_compute_size( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], + project_params: ProjectWithFilesParams, +): + assert ( + len(project_params.allowed_file_sizes) == 1 + ), "test preconditions are not filled! allowed file sizes should have only 1 option for this test" + project, list_of_files = with_random_project_with_files + + total_num_files = sum( + len(files_in_node) for files_in_node in list_of_files.values() + ) + + # get size of a full project + expected_total_size = project_params.allowed_file_sizes[0] * total_num_files + path = Path(project["uuid"]) + await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of one of the nodes + selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + path = Path(project["uuid"]) / f"{selected_node_id}" + selected_node_s3_keys = [ + Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of the outputs of one of the nodes + path = Path(project["uuid"]) / f"{selected_node_id}" / "outputs" + selected_node_s3_keys = [ + Path(s3_object_id) + for s3_object_id in list_of_files[selected_node_id] + if s3_object_id.startswith(f"{path}") + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of workspace in one of the nodes (this is semi-cached in the DB) + path = Path(project["uuid"]) / f"{selected_node_id}" / "workspace" + selected_node_s3_keys = [ + Path(s3_object_id) + for s3_object_id in list_of_files[selected_node_id] + if s3_object_id.startswith(f"{path}") + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + workspace_total_size = await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=path, + expected_total_size=expected_total_size, + ) + + # get size of folders inside the workspace + folders_inside_workspace = [ + p[0] + for p in _filter_and_group_paths_one_level_deeper(selected_node_s3_keys, path) + if p[1] is False + ] + accumulated_subfolder_size = 0 + for workspace_subfolder in folders_inside_workspace: + selected_node_s3_keys = [ + Path(s3_object_id) + for s3_object_id in list_of_files[selected_node_id] + if s3_object_id.startswith(f"{workspace_subfolder}") + ] + expected_total_size = project_params.allowed_file_sizes[0] * len( + selected_node_s3_keys + ) + accumulated_subfolder_size += await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=workspace_subfolder, + expected_total_size=expected_total_size, + ) + + assert workspace_total_size == accumulated_subfolder_size + + +async def test_path_compute_size_inexistent_path( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + faker: Faker, + fake_datcore_tokens: tuple[str, str], +): + await _assert_compute_path_size( + initialized_app, + client, + location_id, + user_id, + path=Path(faker.file_path(absolute=False)), + expected_total_size=0, + ) diff --git a/services/storage/tests/unit/test_handlers_simcore_s3.py b/services/storage/tests/unit/test_handlers_simcore_s3.py index 8449e5ee7c76..7a5fbb9b71f7 100644 --- a/services/storage/tests/unit/test_handlers_simcore_s3.py +++ b/services/storage/tests/unit/test_handlers_simcore_s3.py @@ -209,6 +209,12 @@ def short_dsm_cleaner_interval(monkeypatch: pytest.MonkeyPatch) -> int: return 1 +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "project_params", [ @@ -285,6 +291,12 @@ async def test_copy_folders_from_valid_project_with_one_large_file( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "project_params", [ @@ -461,6 +473,12 @@ async def test_connect_to_external( print(data) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "project_params", [ @@ -498,6 +516,12 @@ async def test_create_and_delete_folders_from_project( ) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize( "project_params", [ @@ -581,6 +605,12 @@ async def search_files_query_params( return q +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize("expected_number_of_user_files", [0, 1, 3]) @pytest.mark.parametrize("query_params_choice", ["default", "limited", "with_offset"]) async def test_search_files_request( @@ -612,6 +642,12 @@ async def test_search_files_request( assert [_.file_uuid for _ in found] == expected +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) @pytest.mark.parametrize("search_startswith", [True, False]) @pytest.mark.parametrize("search_sha256_checksum", [True, False]) @pytest.mark.parametrize("kind", ["owned", "read", None]) diff --git a/services/storage/tests/unit/test_simcore_s3_dsm.py b/services/storage/tests/unit/test_simcore_s3_dsm.py index 50b664199d37..d355df627eb8 100644 --- a/services/storage/tests/unit/test_simcore_s3_dsm.py +++ b/services/storage/tests/unit/test_simcore_s3_dsm.py @@ -29,12 +29,17 @@ def file_size() -> ByteSize: @pytest.fixture def mock_copy_transfer_cb() -> Callable[..., None]: - def copy_transfer_cb(total_bytes_copied: int, *, file_name: str) -> None: - ... + def copy_transfer_cb(total_bytes_copied: int, *, file_name: str) -> None: ... return copy_transfer_cb +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test__copy_path_s3_s3( simcore_s3_dsm: SimcoreS3DataManager, create_directory_with_files: Callable[ @@ -103,6 +108,12 @@ async def _count_files(s3_file_id: SimcoreS3FileID, expected_count: int) -> None await _copy_s3_path(simcore_file_id) +@pytest.mark.parametrize( + "location_id", + [SimcoreS3DataManager.get_location_id()], + ids=[SimcoreS3DataManager.get_location_name()], + indirect=True, +) async def test_upload_and_search( simcore_s3_dsm: SimcoreS3DataManager, upload_file: Callable[..., Awaitable[tuple[Path, SimcoreS3FileID]]],