diff --git a/api/specs/web-server/_storage.py b/api/specs/web-server/_storage.py index b32dcd2f53d6..fac4c4fde874 100644 --- a/api/specs/web-server/_storage.py +++ b/api/specs/web-server/_storage.py @@ -8,7 +8,6 @@ from uuid import UUID from fastapi import APIRouter, Depends, Query, status -from fastapi_pagination.cursor import CursorPage from models_library.api_schemas_storage.storage_schemas import ( FileLocation, FileMetaDataGet, @@ -32,6 +31,7 @@ from models_library.projects_nodes_io import LocationID from models_library.users import UserID from pydantic import AnyUrl, ByteSize +from servicelib.fastapi.rest_pagination import CustomizedPathsCursorPage from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.storage.schemas import DatasetMetaData, FileMetaData @@ -59,7 +59,7 @@ async def list_storage_locations(): @router.get( "/storage/locations/{location_id}/paths", - response_model=CursorPage[PathMetaDataGet], + response_model=CustomizedPathsCursorPage[PathMetaDataGet], ) async def list_storage_paths( _path: Annotated[StorageLocationPathParams, Depends()], diff --git a/packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py b/packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py index 1e02d504f27e..05fad38ee423 100644 --- a/packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py +++ b/packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py @@ -9,7 +9,7 @@ from datetime import datetime from enum import Enum from pathlib import Path -from typing import Annotated, Any, Literal, Self, TypeAlias +from typing import Annotated, Any, Final, Literal, Self, TypeAlias from uuid import UUID from pydantic import ( @@ -404,6 +404,10 @@ class SoftCopyBody(BaseModel): link_id: SimcoreS3FileID +DEFAULT_NUMBER_OF_PATHS_PER_PAGE: Final[int] = 50 +MAX_NUMBER_OF_PATHS_PER_PAGE: Final[int] = 1000 + + class PathMetaDataGet(BaseModel): path: Annotated[Path, Field(description="the path to the current path")] display_path: Annotated[ diff --git a/packages/models-library/src/models_library/api_schemas_webserver/storage.py b/packages/models-library/src/models_library/api_schemas_webserver/storage.py index 9c793ad2ee7d..57ccf1f88b03 100644 --- a/packages/models-library/src/models_library/api_schemas_webserver/storage.py +++ b/packages/models-library/src/models_library/api_schemas_webserver/storage.py @@ -1,8 +1,12 @@ from datetime import datetime from pathlib import Path -from typing import Any +from typing import Annotated, Any -from pydantic import BaseModel +from models_library.api_schemas_storage.storage_schemas import ( + DEFAULT_NUMBER_OF_PATHS_PER_PAGE, + MAX_NUMBER_OF_PATHS_PER_PAGE, +) +from pydantic import BaseModel, Field from ..api_schemas_rpc_async_jobs.async_jobs import ( AsyncJobGet, @@ -13,7 +17,9 @@ from ..api_schemas_storage.data_export_async_jobs import DataExportTaskStartInput from ..progress_bar import ProgressReport from ..projects_nodes_io import LocationID, StorageFileID -from ..rest_pagination import CursorQueryParameters +from ..rest_pagination import ( + CursorQueryParameters, +) from ._base import InputSchema, OutputSchema @@ -24,6 +30,15 @@ class StorageLocationPathParams(BaseModel): class ListPathsQueryParams(InputSchema, CursorQueryParameters): file_filter: Path | None = None + size: Annotated[ + int, + Field( + description="maximum number of items to return (pagination)", + ge=1, + lt=MAX_NUMBER_OF_PATHS_PER_PAGE, + ), + ] = DEFAULT_NUMBER_OF_PATHS_PER_PAGE + class DataExportPost(InputSchema): paths: list[StorageFileID] diff --git a/packages/service-library/src/servicelib/fastapi/rest_pagination.py b/packages/service-library/src/servicelib/fastapi/rest_pagination.py new file mode 100644 index 000000000000..0a199152acea --- /dev/null +++ b/packages/service-library/src/servicelib/fastapi/rest_pagination.py @@ -0,0 +1,28 @@ +from typing import TypeAlias, TypeVar + +from fastapi import Query +from fastapi_pagination.cursor import CursorPage # type: ignore[import-not-found] +from fastapi_pagination.customization import ( # type: ignore[import-not-found] + CustomizedPage, + UseParamsFields, +) +from models_library.api_schemas_storage.storage_schemas import ( + DEFAULT_NUMBER_OF_PATHS_PER_PAGE, + MAX_NUMBER_OF_PATHS_PER_PAGE, +) + +_T = TypeVar("_T") + +CustomizedPathsCursorPage = CustomizedPage[ + CursorPage[_T], + # Customizes the maximum value to fit frontend needs + UseParamsFields( + size=Query( + DEFAULT_NUMBER_OF_PATHS_PER_PAGE, + ge=1, + le=MAX_NUMBER_OF_PATHS_PER_PAGE, + description="Page size", + ) + ), +] +CustomizedPathsCursorPageParams: TypeAlias = CustomizedPathsCursorPage.__params_type__ # type: ignore diff --git a/services/api-server/openapi.json b/services/api-server/openapi.json index f5729e0fa332..b7c0befbb470 100644 --- a/services/api-server/openapi.json +++ b/services/api-server/openapi.json @@ -805,9 +805,9 @@ "required": false, "schema": { "type": "integer", - "maximum": 100, + "maximum": 50, "minimum": 1, - "default": 50, + "default": 20, "title": "Limit" } }, @@ -3352,9 +3352,9 @@ "required": false, "schema": { "type": "integer", - "maximum": 100, + "maximum": 50, "minimum": 1, - "default": 50, + "default": 20, "title": "Limit" } }, @@ -4164,9 +4164,9 @@ "required": false, "schema": { "type": "integer", - "maximum": 100, + "maximum": 50, "minimum": 1, - "default": 50, + "default": 20, "title": "Limit" } }, @@ -5351,9 +5351,9 @@ "required": false, "schema": { "type": "integer", - "maximum": 100, + "maximum": 50, "minimum": 1, - "default": 50, + "default": 20, "title": "Limit" } }, @@ -5648,9 +5648,9 @@ "required": false, "schema": { "type": "integer", - "maximum": 100, + "maximum": 50, "minimum": 1, - "default": 50, + "default": 20, "title": "Limit" } }, diff --git a/services/api-server/src/simcore_service_api_server/models/pagination.py b/services/api-server/src/simcore_service_api_server/models/pagination.py index c8f883f05e41..935cd044f203 100644 --- a/services/api-server/src/simcore_service_api_server/models/pagination.py +++ b/services/api-server/src/simcore_service_api_server/models/pagination.py @@ -1,4 +1,4 @@ -""" Overrides models in fastapi_pagination +"""Overrides models in fastapi_pagination Usage: from fastapi_pagination.api import create_page @@ -11,7 +11,6 @@ from fastapi import Query from fastapi_pagination.customization import CustomizedPage, UseName, UseParamsFields -from fastapi_pagination.limit_offset import LimitOffsetParams as _LimitOffsetParams from fastapi_pagination.links import LimitOffsetPage as _LimitOffsetPage from models_library.rest_pagination import ( DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, @@ -43,7 +42,7 @@ UseName(name="Page"), ] -PaginationParams: TypeAlias = _LimitOffsetParams +PaginationParams: TypeAlias = Page.__params_type__ # type: ignore class OnePage(BaseModel, Generic[T]): diff --git a/services/storage/openapi.json b/services/storage/openapi.json index f826e6b05261..73ddcf302d28 100644 --- a/services/storage/openapi.json +++ b/services/storage/openapi.json @@ -1047,8 +1047,8 @@ "required": false, "schema": { "type": "integer", - "maximum": 100, - "minimum": 0, + "maximum": 1000, + "minimum": 1, "default": 50, "title": "Size" } @@ -1060,7 +1060,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CursorPage_PathMetaDataGet_" + "$ref": "#/components/schemas/CursorPage___T_Customized_PathMetaDataGet_" } } } @@ -1419,7 +1419,7 @@ ], "title": "AppStatusCheck" }, - "CursorPage_PathMetaDataGet_": { + "CursorPage___T_Customized_PathMetaDataGet_": { "properties": { "items": { "items": { @@ -1493,7 +1493,7 @@ "required": [ "items" ], - "title": "CursorPage[PathMetaDataGet]" + "title": "CursorPage[~_T]Customized[PathMetaDataGet]" }, "DatasetMetaDataGet": { "properties": { diff --git a/services/storage/src/simcore_service_storage/api/rest/_paths.py b/services/storage/src/simcore_service_storage/api/rest/_paths.py index 3d1d3b643402..7cbd959259ee 100644 --- a/services/storage/src/simcore_service_storage/api/rest/_paths.py +++ b/services/storage/src/simcore_service_storage/api/rest/_paths.py @@ -4,9 +4,12 @@ from fastapi import APIRouter, Depends from fastapi_pagination import create_page -from fastapi_pagination.cursor import CursorPage, CursorParams from models_library.api_schemas_storage.storage_schemas import PathMetaDataGet from models_library.users import UserID +from servicelib.fastapi.rest_pagination import ( + CustomizedPathsCursorPage, + CustomizedPathsCursorPageParams, +) from ...dsm_factory import BaseDataManager from .dependencies.dsm_prodiver import get_data_manager @@ -22,10 +25,10 @@ @router.get( "/locations/{location_id}/paths", - response_model=CursorPage[PathMetaDataGet], + response_model=CustomizedPathsCursorPage[PathMetaDataGet], ) async def list_paths( - page_params: Annotated[CursorParams, Depends()], + page_params: Annotated[CustomizedPathsCursorPageParams, Depends()], dsm: Annotated[BaseDataManager, Depends(get_data_manager)], user_id: UserID, file_filter: Path | None = None, diff --git a/services/storage/tests/unit/test_handlers_paths.py b/services/storage/tests/unit/test_handlers_paths.py index 636d8a24f8b3..cf53b25542d2 100644 --- a/services/storage/tests/unit/test_handlers_paths.py +++ b/services/storage/tests/unit/test_handlers_paths.py @@ -19,6 +19,7 @@ from fastapi import FastAPI, status from fastapi_pagination.cursor import CursorPage from models_library.api_schemas_storage.storage_schemas import PathMetaDataGet +from models_library.api_schemas_webserver.storage import MAX_NUMBER_OF_PATHS_PER_PAGE from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID from models_library.users import UserID from pydantic import ByteSize, TypeAdapter @@ -203,6 +204,48 @@ async def test_list_paths_pagination( ) +@pytest.mark.parametrize( + "project_params", + [ + ProjectWithFilesParams( + num_nodes=1, + allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("0b"),), + workspace_files_count=MAX_NUMBER_OF_PATHS_PER_PAGE, + ) + ], + ids=str, +) +async def test_list_paths_pagination_large_page( + initialized_app: FastAPI, + client: httpx.AsyncClient, + location_id: LocationID, + user_id: UserID, + with_random_project_with_files: tuple[ + dict[str, Any], + dict[NodeID, dict[SimcoreS3FileID, FileIDDict]], + ], +): + project, list_of_files = with_random_project_with_files + selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311 + selected_node_s3_keys = [ + Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id] + ] + workspace_file_filter = Path(project["uuid"]) / f"{selected_node_id}" / "workspace" + expected_paths = _filter_and_group_paths_one_level_deeper( + selected_node_s3_keys, workspace_file_filter + ) + await _assert_list_paths( + initialized_app, + client, + location_id, + user_id, + file_filter=workspace_file_filter, + expected_paths=expected_paths, + check_total=False, + limit=MAX_NUMBER_OF_PATHS_PER_PAGE, + ) + + @pytest.mark.parametrize( "project_params, num_projects", [ diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 252a7267012e..1cd96ab7b28b 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -6009,9 +6009,9 @@ paths: type: integer minimum: 1 exclusiveMaximum: true - default: 20 + default: 50 title: Size - maximum: 50 + maximum: 1000 - name: cursor in: query required: false @@ -6035,7 +6035,7 @@ paths: content: application/json: schema: - $ref: '#/components/schemas/CursorPage_PathMetaDataGet_' + $ref: '#/components/schemas/CursorPage___T_Customized_PathMetaDataGet_' /v0/storage/locations/{location_id}/datasets: get: tags: @@ -8404,7 +8404,7 @@ components: - usdPerCredit - minPaymentAmountUsd title: CreditPriceGet - CursorPage_PathMetaDataGet_: + CursorPage___T_Customized_PathMetaDataGet_: properties: items: items: @@ -8444,7 +8444,7 @@ components: type: object required: - items - title: CursorPage[PathMetaDataGet] + title: CursorPage[~_T]Customized[PathMetaDataGet] DatCoreFileLink: properties: store: