storage code change

Author: sanderegg

services/storage/src/simcore_service_storage/api/rest/_files.py

@@ -81,6 +81,7 @@ async def get_file_metadata(
     user_agent: Annotated[str | None, Header()],
     request: Request,
 ):
+    # NOTE: Used by legacy dynamic services -> MUST BE BACKWARDS COMPATIBLE
     dsm = get_dsm_provider(request.app).get(location_id)
     try:
         data = await dsm.get_file(
@@ -133,6 +134,7 @@ async def download_file(
     query_params: Annotated[FileDownloadQueryParams, Depends()],
     request: Request,
 ) -> Envelope[FileDownloadResponse]:
+    # NOTE: Used by legacy dynamic services -> MUST BE BACKWARDS COMPATIBLE
     dsm = get_dsm_provider(request.app).get(location_id)
     link = await dsm.create_file_download_link(
         query_params.user_id, file_id, query_params.link_type
@@ -178,6 +180,7 @@ async def upload_file(
     Use-case v2.2: if query.file_size > 0 and query.link_type=presigned or None, returns 1 or more presigned links depending on the file size (limited to a single 5TB file)
     Use-case v2.3: if query.link_type=s3 and query.file_size>=0, returns a single s3 direct link (limited to a single 5TB file)
     """
+    # NOTE: Used by legacy dynamic services with single presigned link -> MUST BE BACKWARDS COMPATIBLE
     dsm = get_dsm_provider(request.app).get(location_id)
     links: UploadLinks = await dsm.create_file_upload_links(
         user_id=query_params.user_id,

services/storage/src/simcore_service_storage/api/rest/_locations.py

@@ -5,7 +5,6 @@
 from models_library.api_schemas_storage.storage_schemas import FileLocation
 from models_library.generics import Envelope
 
-# Exclusive for simcore-s3 storage -----------------------
 from ...dsm import get_dsm_provider
 from ...models import StorageQueryParamsBase
 
@@ -16,7 +15,6 @@
 )
 
 
-# HANDLERS ---------------------------------------------------
 @router.get(
     "/locations",
     status_code=status.HTTP_200_OK,

services/storage/src/simcore_service_storage/api/rest/_paths.py

@@ -0,0 +1,45 @@
+import logging
+from pathlib import Path
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from fastapi_pagination import create_page
+from fastapi_pagination.cursor import CursorPage, CursorParams
+from models_library.api_schemas_storage.storage_schemas import PathMetaDataGet
+from models_library.users import UserID
+
+from ...dsm_factory import BaseDataManager
+from .dependencies.dsm_prodiver import get_data_manager
+
+_logger = logging.getLogger(__name__)
+
+router = APIRouter(
+    tags=[
+        "files",
+    ],
+)
+
+
+@router.get(
+    "/locations/{location_id}/paths",
+    response_model=CursorPage[PathMetaDataGet],
+)
+async def list_paths(
+    page_params: Annotated[CursorParams, Depends()],
+    dsm: Annotated[BaseDataManager, Depends(get_data_manager)],
+    user_id: UserID,
+    file_filter: Path | None = None,
+):
+    """Returns one level of files (paginated)"""
+    items, next_cursor, total_number = await dsm.list_paths(
+        user_id=user_id,
+        file_filter=file_filter,
+        limit=page_params.size,
+        cursor=page_params.to_raw_params().cursor,
+    )
+    return create_page(
+        [_.to_api_model() for _ in items],
+        total=total_number,
+        params=page_params,
+        next_=next_cursor,
+    )

services/storage/src/simcore_service_storage/api/rest/dependencies/application.py

@@ -0,0 +1,18 @@
+# mypy: disable-error-code=truthy-function
+from fastapi import Request
+from servicelib.fastapi.dependencies import get_app, get_reverse_url_mapper
+
+from ....core.settings import ApplicationSettings, get_application_settings
+
+
+def get_settings(request: Request) -> ApplicationSettings:
+    return get_application_settings(request.app)
+
+
+assert get_reverse_url_mapper  # nosec
+assert get_app  # nosec
+
+__all__: tuple[str, ...] = (
+    "get_app",
+    "get_reverse_url_mapper",
+)

services/storage/src/simcore_service_storage/api/rest/dependencies/dsm_prodiver.py

@@ -0,0 +1,22 @@
+from typing import Annotated
+
+from fastapi import Depends, FastAPI
+from models_library.projects_nodes_io import LocationID
+from servicelib.fastapi.dependencies import get_app
+from simcore_service_storage.dsm import get_dsm_provider
+from simcore_service_storage.dsm_factory import BaseDataManager, DataManagerProvider
+
+
+def get_data_manager_provider(
+    app: Annotated[FastAPI, Depends(get_app)],
+) -> DataManagerProvider:
+    return get_dsm_provider(app)
+
+
+async def get_data_manager(
+    location_id: LocationID,
+    data_manager_provider: Annotated[
+        DataManagerProvider, Depends(get_data_manager_provider)
+    ],
+) -> BaseDataManager:
+    return data_manager_provider.get(location_id)

services/storage/src/simcore_service_storage/api/rest/routes.py

@@ -1,23 +1,15 @@
 from fastapi import APIRouter, FastAPI
 
-from . import _datasets, _files, _health, _locations, _simcore_s3
+from . import _datasets, _files, _health, _locations, _paths, _simcore_s3
 
 v0_router = APIRouter()
 
-# health
 health_router = _health.router
 v0_router.include_router(_health.router)
-
-# locations
 v0_router.include_router(_locations.router)
-
-# datasets
 v0_router.include_router(_datasets.router)
-
-# files
 v0_router.include_router(_files.router)
-
-# simcore-s3
+v0_router.include_router(_paths.router)
 v0_router.include_router(_simcore_s3.router)
 
 

services/storage/src/simcore_service_storage/core/application.py

@@ -8,6 +8,7 @@
 from common_library.basic_types import BootModeEnum
 from fastapi import FastAPI
 from fastapi.middleware.gzip import GZipMiddleware
+from fastapi_pagination import add_pagination
 from servicelib.fastapi import timing_middleware
 from servicelib.fastapi.client_session import setup_client_session
 from servicelib.fastapi.openapi import override_fastapi_openapi_method
@@ -64,13 +65,14 @@ def create_app(settings: ApplicationSettings) -> FastAPI:
         debug=settings.SC_BOOT_MODE
         in [BootModeEnum.DEBUG, BootModeEnum.DEVELOPMENT, BootModeEnum.LOCAL],
         title=APP_NAME,
-        description="Service to auto-scale swarm",
+        description="Service that manages osparc storage backend",
         version=API_VERSION,
         openapi_url=f"/api/{API_VTAG}/openapi.json",
         docs_url="/dev/doc",
         redoc_url=None,  # default disabled
     )
     override_fastapi_openapi_method(app)
+    add_pagination(app)
 
     # STATE
     app.state.settings = settings