Skip to content

Commit a059026

Browse files
sandereggsanderegg
authored
🎨Storage: Increased maximum number of returned paths to 1000 (ITISFoundation#7305)
1 parent 996febd commit a059026

File tree

10 files changed

+124
-32
lines changed

10 files changed

+124
-32
lines changed

‎api/specs/web-server/_storage.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
from uuid import UUID
99

1010
from fastapi import APIRouter, Depends, Query, status
11-
from fastapi_pagination.cursor import CursorPage
1211
from models_library.api_schemas_storage.storage_schemas import (
1312
FileLocation,
1413
FileMetaDataGet,
@@ -32,6 +31,7 @@
3231
from models_library.projects_nodes_io import LocationID
3332
from models_library.users import UserID
3433
from pydantic import AnyUrl, ByteSize
34+
from servicelib.fastapi.rest_pagination import CustomizedPathsCursorPage
3535
from simcore_service_webserver._meta import API_VTAG
3636
from simcore_service_webserver.storage.schemas import DatasetMetaData, FileMetaData
3737

@@ -59,7 +59,7 @@ async def list_storage_locations():
5959

6060
@router.get(
6161
"/storage/locations/{location_id}/paths",
62-
response_model=CursorPage[PathMetaDataGet],
62+
response_model=CustomizedPathsCursorPage[PathMetaDataGet],
6363
)
6464
async def list_storage_paths(
6565
_path: Annotated[StorageLocationPathParams, Depends()],

‎packages/models-library/src/models_library/api_schemas_storage/storage_schemas.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
from datetime import datetime
1010
from enum import Enum
1111
from pathlib import Path
12-
from typing import Annotated, Any, Literal, Self, TypeAlias
12+
from typing import Annotated, Any, Final, Literal, Self, TypeAlias
1313
from uuid import UUID
1414

1515
from pydantic import (
@@ -404,6 +404,10 @@ class SoftCopyBody(BaseModel):
404404
link_id: SimcoreS3FileID
405405

406406

407+
DEFAULT_NUMBER_OF_PATHS_PER_PAGE: Final[int] = 50
408+
MAX_NUMBER_OF_PATHS_PER_PAGE: Final[int] = 1000
409+
410+
407411
class PathMetaDataGet(BaseModel):
408412
path: Annotated[Path, Field(description="the path to the current path")]
409413
display_path: Annotated[

‎packages/models-library/src/models_library/api_schemas_webserver/storage.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
from datetime import datetime
22
from pathlib import Path
3-
from typing import Any
3+
from typing import Annotated, Any
44

5-
from pydantic import BaseModel
5+
from models_library.api_schemas_storage.storage_schemas import (
6+
DEFAULT_NUMBER_OF_PATHS_PER_PAGE,
7+
MAX_NUMBER_OF_PATHS_PER_PAGE,
8+
)
9+
from pydantic import BaseModel, Field
610

711
from ..api_schemas_rpc_async_jobs.async_jobs import (
812
AsyncJobGet,
@@ -13,7 +17,9 @@
1317
from ..api_schemas_storage.data_export_async_jobs import DataExportTaskStartInput
1418
from ..progress_bar import ProgressReport
1519
from ..projects_nodes_io import LocationID, StorageFileID
16-
from ..rest_pagination import CursorQueryParameters
20+
from ..rest_pagination import (
21+
CursorQueryParameters,
22+
)
1723
from ._base import InputSchema, OutputSchema
1824

1925

@@ -24,6 +30,15 @@ class StorageLocationPathParams(BaseModel):
2430
class ListPathsQueryParams(InputSchema, CursorQueryParameters):
2531
file_filter: Path | None = None
2632

33+
size: Annotated[
34+
int,
35+
Field(
36+
description="maximum number of items to return (pagination)",
37+
ge=1,
38+
lt=MAX_NUMBER_OF_PATHS_PER_PAGE,
39+
),
40+
] = DEFAULT_NUMBER_OF_PATHS_PER_PAGE
41+
2742

2843
class DataExportPost(InputSchema):
2944
paths: list[StorageFileID]

‎packages/service-library/src/servicelib/fastapi/rest_pagination.py‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
from typing import TypeAlias, TypeVar
2+
3+
from fastapi import Query
4+
from fastapi_pagination.cursor import CursorPage # type: ignore[import-not-found]
5+
from fastapi_pagination.customization import ( # type: ignore[import-not-found]
6+
CustomizedPage,
7+
UseParamsFields,
8+
)
9+
from models_library.api_schemas_storage.storage_schemas import (
10+
DEFAULT_NUMBER_OF_PATHS_PER_PAGE,
11+
MAX_NUMBER_OF_PATHS_PER_PAGE,
12+
)
13+
14+
_T = TypeVar("_T")
15+
16+
CustomizedPathsCursorPage = CustomizedPage[
17+
CursorPage[_T],
18+
# Customizes the maximum value to fit frontend needs
19+
UseParamsFields(
20+
size=Query(
21+
DEFAULT_NUMBER_OF_PATHS_PER_PAGE,
22+
ge=1,
23+
le=MAX_NUMBER_OF_PATHS_PER_PAGE,
24+
description="Page size",
25+
)
26+
),
27+
]
28+
CustomizedPathsCursorPageParams: TypeAlias = CustomizedPathsCursorPage.__params_type__ # type: ignore

‎services/api-server/openapi.json‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -805,9 +805,9 @@
805805
"required": false,
806806
"schema": {
807807
"type": "integer",
808-
"maximum": 100,
808+
"maximum": 50,
809809
"minimum": 1,
810-
"default": 50,
810+
"default": 20,
811811
"title": "Limit"
812812
}
813813
},
@@ -3352,9 +3352,9 @@
33523352
"required": false,
33533353
"schema": {
33543354
"type": "integer",
3355-
"maximum": 100,
3355+
"maximum": 50,
33563356
"minimum": 1,
3357-
"default": 50,
3357+
"default": 20,
33583358
"title": "Limit"
33593359
}
33603360
},
@@ -4164,9 +4164,9 @@
41644164
"required": false,
41654165
"schema": {
41664166
"type": "integer",
4167-
"maximum": 100,
4167+
"maximum": 50,
41684168
"minimum": 1,
4169-
"default": 50,
4169+
"default": 20,
41704170
"title": "Limit"
41714171
}
41724172
},
@@ -5351,9 +5351,9 @@
53515351
"required": false,
53525352
"schema": {
53535353
"type": "integer",
5354-
"maximum": 100,
5354+
"maximum": 50,
53555355
"minimum": 1,
5356-
"default": 50,
5356+
"default": 20,
53575357
"title": "Limit"
53585358
}
53595359
},
@@ -5648,9 +5648,9 @@
56485648
"required": false,
56495649
"schema": {
56505650
"type": "integer",
5651-
"maximum": 100,
5651+
"maximum": 50,
56525652
"minimum": 1,
5653-
"default": 50,
5653+
"default": 20,
56545654
"title": "Limit"
56555655
}
56565656
},

‎services/api-server/src/simcore_service_api_server/models/pagination.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
""" Overrides models in fastapi_pagination
1+
"""Overrides models in fastapi_pagination
22
33
Usage:
44
from fastapi_pagination.api import create_page
@@ -11,7 +11,6 @@
1111

1212
from fastapi import Query
1313
from fastapi_pagination.customization import CustomizedPage, UseName, UseParamsFields
14-
from fastapi_pagination.limit_offset import LimitOffsetParams as _LimitOffsetParams
1514
from fastapi_pagination.links import LimitOffsetPage as _LimitOffsetPage
1615
from models_library.rest_pagination import (
1716
DEFAULT_NUMBER_OF_ITEMS_PER_PAGE,
@@ -43,7 +42,7 @@
4342
UseName(name="Page"),
4443
]
4544

46-
PaginationParams: TypeAlias = _LimitOffsetParams
45+
PaginationParams: TypeAlias = Page.__params_type__ # type: ignore
4746

4847

4948
class OnePage(BaseModel, Generic[T]):

‎services/storage/openapi.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1047,8 +1047,8 @@
10471047
"required": false,
10481048
"schema": {
10491049
"type": "integer",
1050-
"maximum": 100,
1051-
"minimum": 0,
1050+
"maximum": 1000,
1051+
"minimum": 1,
10521052
"default": 50,
10531053
"title": "Size"
10541054
}
@@ -1060,7 +1060,7 @@
10601060
"content": {
10611061
"application/json": {
10621062
"schema": {
1063-
"$ref": "#/components/schemas/CursorPage_PathMetaDataGet_"
1063+
"$ref": "#/components/schemas/CursorPage___T_Customized_PathMetaDataGet_"
10641064
}
10651065
}
10661066
}
@@ -1419,7 +1419,7 @@
14191419
],
14201420
"title": "AppStatusCheck"
14211421
},
1422-
"CursorPage_PathMetaDataGet_": {
1422+
"CursorPage___T_Customized_PathMetaDataGet_": {
14231423
"properties": {
14241424
"items": {
14251425
"items": {
@@ -1493,7 +1493,7 @@
14931493
"required": [
14941494
"items"
14951495
],
1496-
"title": "CursorPage[PathMetaDataGet]"
1496+
"title": "CursorPage[~_T]Customized[PathMetaDataGet]"
14971497
},
14981498
"DatasetMetaDataGet": {
14991499
"properties": {

‎services/storage/src/simcore_service_storage/api/rest/_paths.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@
44

55
from fastapi import APIRouter, Depends
66
from fastapi_pagination import create_page
7-
from fastapi_pagination.cursor import CursorPage, CursorParams
87
from models_library.api_schemas_storage.storage_schemas import PathMetaDataGet
98
from models_library.users import UserID
9+
from servicelib.fastapi.rest_pagination import (
10+
CustomizedPathsCursorPage,
11+
CustomizedPathsCursorPageParams,
12+
)
1013

1114
from ...dsm_factory import BaseDataManager
1215
from .dependencies.dsm_prodiver import get_data_manager
@@ -22,10 +25,10 @@
2225

2326
@router.get(
2427
"/locations/{location_id}/paths",
25-
response_model=CursorPage[PathMetaDataGet],
28+
response_model=CustomizedPathsCursorPage[PathMetaDataGet],
2629
)
2730
async def list_paths(
28-
page_params: Annotated[CursorParams, Depends()],
31+
page_params: Annotated[CustomizedPathsCursorPageParams, Depends()],
2932
dsm: Annotated[BaseDataManager, Depends(get_data_manager)],
3033
user_id: UserID,
3134
file_filter: Path | None = None,

‎services/storage/tests/unit/test_handlers_paths.py‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from fastapi import FastAPI, status
2020
from fastapi_pagination.cursor import CursorPage
2121
from models_library.api_schemas_storage.storage_schemas import PathMetaDataGet
22+
from models_library.api_schemas_webserver.storage import MAX_NUMBER_OF_PATHS_PER_PAGE
2223
from models_library.projects_nodes_io import LocationID, NodeID, SimcoreS3FileID
2324
from models_library.users import UserID
2425
from pydantic import ByteSize, TypeAdapter
@@ -203,6 +204,48 @@ async def test_list_paths_pagination(
203204
)
204205

205206

207+
@pytest.mark.parametrize(
208+
"project_params",
209+
[
210+
ProjectWithFilesParams(
211+
num_nodes=1,
212+
allowed_file_sizes=(TypeAdapter(ByteSize).validate_python("0b"),),
213+
workspace_files_count=MAX_NUMBER_OF_PATHS_PER_PAGE,
214+
)
215+
],
216+
ids=str,
217+
)
218+
async def test_list_paths_pagination_large_page(
219+
initialized_app: FastAPI,
220+
client: httpx.AsyncClient,
221+
location_id: LocationID,
222+
user_id: UserID,
223+
with_random_project_with_files: tuple[
224+
dict[str, Any],
225+
dict[NodeID, dict[SimcoreS3FileID, FileIDDict]],
226+
],
227+
):
228+
project, list_of_files = with_random_project_with_files
229+
selected_node_id = NodeID(random.choice(list(project["workbench"]))) # noqa: S311
230+
selected_node_s3_keys = [
231+
Path(s3_object_id) for s3_object_id in list_of_files[selected_node_id]
232+
]
233+
workspace_file_filter = Path(project["uuid"]) / f"{selected_node_id}" / "workspace"
234+
expected_paths = _filter_and_group_paths_one_level_deeper(
235+
selected_node_s3_keys, workspace_file_filter
236+
)
237+
await _assert_list_paths(
238+
initialized_app,
239+
client,
240+
location_id,
241+
user_id,
242+
file_filter=workspace_file_filter,
243+
expected_paths=expected_paths,
244+
check_total=False,
245+
limit=MAX_NUMBER_OF_PATHS_PER_PAGE,
246+
)
247+
248+
206249
@pytest.mark.parametrize(
207250
"project_params, num_projects",
208251
[

‎services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6009,9 +6009,9 @@ paths:
60096009
type: integer
60106010
minimum: 1
60116011
exclusiveMaximum: true
6012-
default: 20
6012+
default: 50
60136013
title: Size
6014-
maximum: 50
6014+
maximum: 1000
60156015
- name: cursor
60166016
in: query
60176017
required: false
@@ -6035,7 +6035,7 @@ paths:
60356035
content:
60366036
application/json:
60376037
schema:
6038-
$ref: '#/components/schemas/CursorPage_PathMetaDataGet_'
6038+
$ref: '#/components/schemas/CursorPage___T_Customized_PathMetaDataGet_'
60396039
/v0/storage/locations/{location_id}/datasets:
60406040
get:
60416041
tags:
@@ -8404,7 +8404,7 @@ components:
84048404
- usdPerCredit
84058405
- minPaymentAmountUsd
84068406
title: CreditPriceGet
8407-
CursorPage_PathMetaDataGet_:
8407+
CursorPage___T_Customized_PathMetaDataGet_:
84088408
properties:
84098409
items:
84108410
items:
@@ -8444,7 +8444,7 @@ components:
84448444
type: object
84458445
required:
84468446
- items
8447-
title: CursorPage[PathMetaDataGet]
8447+
title: CursorPage[~_T]Customized[PathMetaDataGet]
84488448
DatCoreFileLink:
84498449
properties:
84508450
store:

0 commit comments

Comments
 (0)