webserver api

Author: sanderegg

services/web/server/requirements/_test.in

@@ -16,6 +16,7 @@ coverage
 docker
 Faker
 fastapi[standard]
+fastapi-pagination
 flaky
 hypothesis
 jsonref

services/web/server/requirements/_test.txt

@@ -14,10 +14,16 @@ alembic==1.8.1
     # via
     #   -c requirements/_base.txt
     #   -r requirements/_test.in
+annotated-types==0.7.0
+    # via
+    #   -c requirements/_base.txt
+    #   pydantic
 anyio==4.3.0
     # via
     #   -c requirements/_base.txt
     #   httpx
+    #   starlette
+    #   watchfiles
 async-timeout==4.0.3
     # via
     #   -c requirements/_base.txt
@@ -77,6 +83,8 @@ fastapi==0.115.6
     # via -r requirements/_test.in
 fastapi-cli==0.0.5
     # via fastapi
+fastapi-pagination==0.12.34
+    # via -r requirements/_test.in
 flaky==3.8.1
     # via -r requirements/_test.in
 frozenlist==1.4.1
@@ -92,14 +100,18 @@ h11==0.14.0
     # via
     #   -c requirements/_base.txt
     #   httpcore
+    #   uvicorn
 httpcore==1.0.7
     # via
     #   -c requirements/_base.txt
     #   httpx
+httptools==0.6.4
+    # via uvicorn
 httpx==0.28.1
     # via
     #   -c requirements/../../../../requirements/constraints.txt
     #   -c requirements/_base.txt
+    #   fastapi
     #   respx
 hypothesis==6.91.0
     # via -r requirements/_test.in
@@ -109,6 +121,7 @@ idna==3.3
     # via
     #   -c requirements/_base.txt
     #   anyio
+    #   email-validator
     #   httpx
     #   requests
     #   yarl
@@ -175,6 +188,7 @@ pydantic==2.10.2
     #   -c requirements/../../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   fastapi
+    #   fastapi-pagination
 pydantic-core==2.27.1
     # via
     #   -c requirements/_base.txt
@@ -253,6 +267,10 @@ requests==2.32.2
     #   docker
 respx==0.22.0
     # via -r requirements/_test.in
+rich==13.4.2
+    # via
+    #   -c requirements/_base.txt
+    #   typer
 setuptools==69.1.1
     # via
     #   -c requirements/_base.txt
@@ -310,6 +328,7 @@ typing-extensions==4.12.2
     #   -c requirements/_base.txt
     #   asyncpg-stubs
     #   fastapi
+    #   fastapi-pagination
     #   mypy
     #   pydantic
     #   pydantic-core

services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml

@@ -6024,6 +6024,54 @@ paths:
                   $ref: '#/components/schemas/FileLocation'
                 type: array
                 title: Response List Storage Locations
+  /v0/storage/locations/{location_id}/paths:
+    get:
+      tags:
+      - storage
+      summary: List Storage Paths
+      description: Lists the files/directories in WorkingDirectory
+      operationId: list_storage_paths
+      parameters:
+      - name: location_id
+        in: path
+        required: true
+        schema:
+          type: integer
+          title: Location Id
+      - name: size
+        in: query
+        required: false
+        schema:
+          type: integer
+          minimum: 1
+          exclusiveMaximum: true
+          default: 20
+          title: Size
+          maximum: 50
+      - name: cursor
+        in: query
+        required: false
+        schema:
+          anyOf:
+          - type: string
+          - type: 'null'
+          title: Cursor
+      - name: fileFilter
+        in: query
+        required: false
+        schema:
+          anyOf:
+          - type: string
+            format: path
+          - type: 'null'
+          title: Filefilter
+      responses:
+        '200':
+          description: Successful Response
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/CursorPage_FileMetaDataGet_'
   /v0/storage/locations/{location_id}/datasets:
     get:
       tags:
@@ -8362,6 +8410,47 @@ components:
       required:
       - priceDollars
       title: CreateWalletPayment
+    CursorPage_FileMetaDataGet_:
+      properties:
+        items:
+          items:
+            $ref: '#/components/schemas/FileMetaDataGet'
+          type: array
+          title: Items
+        total:
+          anyOf:
+          - type: integer
+          - type: 'null'
+          title: Total
+          description: Total items
+        current_page:
+          anyOf:
+          - type: string
+          - type: 'null'
+          title: Current Page
+          description: Cursor to refetch the current page
+        current_page_backwards:
+          anyOf:
+          - type: string
+          - type: 'null'
+          title: Current Page Backwards
+          description: Cursor to refetch the current page starting from the last item
+        previous_page:
+          anyOf:
+          - type: string
+          - type: 'null'
+          title: Previous Page
+          description: Cursor for the previous page
+        next_page:
+          anyOf:
+          - type: string
+          - type: 'null'
+          title: Next Page
+          description: Cursor for the next page
+      type: object
+      required:
+      - items
+      title: CursorPage[FileMetaDataGet]
     DatCoreFileLink:
       properties:
         store:

services/web/server/src/simcore_service_webserver/storage/_rest.py

@@ -25,6 +25,7 @@
     StorageAsyncJobStatus,
 )
 from models_library.projects_nodes_io import LocationID
+from models_library.utils.change_case import camel_to_snake
 from models_library.utils.fastapi_encoders import jsonable_encoder
 from pydantic import AnyUrl, BaseModel, ByteSize, TypeAdapter
 from servicelib.aiohttp import status
@@ -45,12 +46,12 @@
 )
 from servicelib.request_keys import RQT_USERID_KEY
 from servicelib.rest_responses import unwrap_envelope
-from simcore_service_webserver.rabbitmq import get_rabbitmq_rpc_client
 from yarl import URL
 
 from .._meta import API_VTAG
 from ..login.decorators import login_required
 from ..models import RequestContext
+from ..rabbitmq import get_rabbitmq_rpc_client
 from ..security.decorators import permission_required
 from ._exception_handlers import handle_data_export_exceptions
 from .schemas import StorageFileIDStr
@@ -91,7 +92,7 @@ def _to_storage_url(request: web.Request) -> URL:
 
     return (
         url.joinpath(fastapi_encoded_suffix, encoded=True)
-        .with_query(request.query)
+        .with_query({camel_to_snake(k): v for k, v in request.query.items()})
         .update_query(user_id=userid)
     )
 
@@ -123,18 +124,29 @@ class _ResponseTuple(NamedTuple):
 
 
 async def _forward_request_to_storage(
-    request: web.Request, method: str, body: dict[str, Any] | None = None, **kwargs
+    request: web.Request,
+    method: str,
+    body: dict[str, Any] | None = None,
+    **kwargs,
 ) -> _ResponseTuple:
     url = _to_storage_url(request)
     session = get_client_session(request.app)
 
     async with session.request(
         method.upper(), url, ssl=False, json=body, **kwargs
     ) as resp:
-        if resp.status >= status.HTTP_400_BAD_REQUEST:
-            raise web.HTTPException(reason=await resp.text())
-        payload = await resp.json()
-        return _ResponseTuple(payload=payload, status_code=resp.status)
+        match resp.status:
+            case status.HTTP_422_UNPROCESSABLE_ENTITY:
+                raise web.HTTPUnprocessableEntity(
+                    reason=await resp.text(), content_type=resp.content_type
+                )
+            case status.HTTP_404_NOT_FOUND:
+                raise web.HTTPNotFound(reason=await resp.text())
+            case _ if resp.status >= status.HTTP_400_BAD_REQUEST:
+                raise web.HTTPError(reason=await resp.text())
+            case _:
+                payload = await resp.json()
+                return _ResponseTuple(payload=payload, status_code=resp.status)
 
 
 # ---------------------------------------------------------------------
@@ -152,6 +164,16 @@ async def list_storage_locations(request: web.Request) -> web.Response:
     return create_data_response(payload, status=resp_status)
 
 
+@routes.get(
+    f"{_storage_locations_prefix}/{{location_id}}/paths", name="list_storage_paths"
+)
+@login_required
+@permission_required("storage.files.*")
+async def list_paths(request: web.Request) -> web.Response:
+    payload, resp_status = await _forward_request_to_storage(request, "GET", body=None)
+    return create_data_response(payload, status=resp_status)
+
+
 @routes.get(
     _storage_locations_prefix + "/{location_id}/datasets", name="list_datasets_metadata"
 )
@@ -423,7 +445,6 @@ class _PathParams(BaseModel):
 @permission_required("storage.files.*")
 @handle_data_export_exceptions
 async def get_async_jobs(request: web.Request) -> web.Response:
-
     _req_ctx = RequestContext.model_validate(request)
 
     rabbitmq_rpc_client = get_rabbitmq_rpc_client(request.app)
@@ -449,7 +470,6 @@ async def get_async_jobs(request: web.Request) -> web.Response:
 @permission_required("storage.files.*")
 @handle_data_export_exceptions
 async def get_async_job_status(request: web.Request) -> web.Response:
-
     _req_ctx = RequestContext.model_validate(request)
     rabbitmq_rpc_client = get_rabbitmq_rpc_client(request.app)
 
@@ -503,7 +523,6 @@ async def abort_async_job(request: web.Request) -> web.Response:
 @permission_required("storage.files.*")
 @handle_data_export_exceptions
 async def get_async_job_result(request: web.Request) -> web.Response:
-
     _req_ctx = RequestContext.model_validate(request)
 
     rabbitmq_rpc_client = get_rabbitmq_rpc_client(request.app)