🐛Datcore-Adapter: fix listing files in pennsieve pagination sized 1000 (#7464)

Author: sanderegg

services/datcore-adapter/src/simcore_service_datcore_adapter/api/rest/datasets.py

@@ -1,15 +1,20 @@
 import logging
-from typing import Annotated, Final
+from typing import Annotated, Final, TypeAlias, TypeVar
 
 from aiocache import cached  # type: ignore[import-untyped]
-from fastapi import APIRouter, Depends, Header, Request
-from fastapi_pagination import Page, Params
+from fastapi import APIRouter, Depends, Header, Query, Request
+from fastapi_pagination import LimitOffsetPage, Params
 from fastapi_pagination.api import create_page, resolve_params
 from fastapi_pagination.bases import RawParams
+from fastapi_pagination.customization import CustomizedPage, UseParamsFields
 from models_library.api_schemas_datcore_adapter.datasets import (
     DatasetMetaData,
     FileMetaData,
 )
+from models_library.api_schemas_storage.storage_schemas import (
+    DEFAULT_NUMBER_OF_PATHS_PER_PAGE,
+    MAX_NUMBER_OF_PATHS_PER_PAGE,
+)
 from servicelib.fastapi.requests_decorators import cancel_on_disconnect
 from starlette import status
 
@@ -25,35 +30,47 @@
 )  # NOTE: this caching time is arbitrary
 
 
+_T = TypeVar("_T")
+_CustomPage = CustomizedPage[
+    LimitOffsetPage[_T],
+    UseParamsFields(
+        limit=Query(
+            DEFAULT_NUMBER_OF_PATHS_PER_PAGE, ge=1, le=MAX_NUMBER_OF_PATHS_PER_PAGE
+        ),
+    ),
+]
+
+_CustomizedPageParams: TypeAlias = _CustomPage.__params_type__  # type: ignore
+
+
 @router.get(
     "/datasets",
     summary="list datasets",
     status_code=status.HTTP_200_OK,
-    response_model=Page[DatasetMetaData],
+    response_model=_CustomPage[DatasetMetaData],
 )
 @cancel_on_disconnect
 @cached(
     ttl=_PENNSIEVE_CACHING_TTL_S,
-    key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['params']}",
+    key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['page_params']}",
 )
 async def list_datasets(
     request: Request,
     x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
     x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
     pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
-    params: Annotated[Params, Depends()],
-) -> Page[DatasetMetaData]:
+    page_params: Annotated[_CustomizedPageParams, Depends()],
+):
     assert request  # nosec
-    raw_params: RawParams = resolve_params(params).to_raw_params()
-    assert raw_params.limit is not None  # nosec
-    assert raw_params.offset is not None  # nosec
+    assert page_params.limit is not None  # nosec
+    assert page_params.offset is not None  # nosec
     datasets, total = await pennsieve_client.list_datasets(
         api_key=x_datcore_api_key,
         api_secret=x_datcore_api_secret,
-        limit=raw_params.limit,
-        offset=raw_params.offset,
+        limit=page_params.limit,
+        offset=page_params.offset,
     )
-    return create_page(datasets, total=total, params=params)  # type: ignore[return-value]
+    return create_page(datasets, total=total, params=page_params)
 
 
 @router.get(
@@ -85,46 +102,45 @@ async def get_dataset(
     "/datasets/{dataset_id}/files",
     summary="list top level files/folders in a dataset",
     status_code=status.HTTP_200_OK,
-    response_model=Page[FileMetaData],
+    response_model=_CustomPage[FileMetaData],
 )
 @cancel_on_disconnect
 @cached(
     ttl=_PENNSIEVE_CACHING_TTL_S,
-    key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['dataset_id']}_{kwargs['params']}",
+    key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['dataset_id']}_{kwargs['page_params']}",
 )
 async def list_dataset_top_level_files(
     request: Request,
     dataset_id: str,
     x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
     x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
     pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
-    params: Annotated[Params, Depends()],
-) -> Page[FileMetaData]:
+    page_params: Annotated[_CustomizedPageParams, Depends()],
+):
     assert request  # nosec
-    raw_params: RawParams = resolve_params(params).to_raw_params()
 
-    assert raw_params.limit is not None  # nosec
-    assert raw_params.offset is not None  # nosec
+    assert page_params.limit is not None  # nosec
+    assert page_params.offset is not None  # nosec
     file_metas, total = await pennsieve_client.list_packages_in_dataset(
         api_key=x_datcore_api_key,
         api_secret=x_datcore_api_secret,
         dataset_id=dataset_id,
-        limit=raw_params.limit,
-        offset=raw_params.offset,
+        limit=page_params.limit,
+        offset=page_params.offset,
     )
-    return create_page(file_metas, total=total, params=params)  # type: ignore[return-value]
+    return create_page(file_metas, total=total, params=page_params)
 
 
 @router.get(
     "/datasets/{dataset_id}/files/{collection_id}",
     summary="list top level files/folders in a collection in a dataset",
     status_code=status.HTTP_200_OK,
-    response_model=Page[FileMetaData],
+    response_model=_CustomPage[FileMetaData],
 )
 @cancel_on_disconnect
 @cached(
     ttl=_PENNSIEVE_CACHING_TTL_S,
-    key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['dataset_id']}_{kwargs['collection_id']}_{kwargs['params']}",
+    key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['dataset_id']}_{kwargs['collection_id']}_{kwargs['page_params']}",
 )
 async def list_dataset_collection_files(
     request: Request,
@@ -133,21 +149,20 @@ async def list_dataset_collection_files(
     x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
     x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
     pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
-    params: Annotated[Params, Depends()],
-) -> Page[FileMetaData]:
+    page_params: Annotated[_CustomizedPageParams, Depends()],
+):
     assert request  # nosec
-    raw_params: RawParams = resolve_params(params).to_raw_params()
-    assert raw_params.limit is not None  # nosec
-    assert raw_params.offset is not None  # nosec
+    assert page_params.limit is not None  # nosec
+    assert page_params.offset is not None  # nosec
     file_metas, total = await pennsieve_client.list_packages_in_collection(
         api_key=x_datcore_api_key,
         api_secret=x_datcore_api_secret,
-        limit=raw_params.limit,
-        offset=raw_params.offset,
+        limit=page_params.limit,
+        offset=page_params.offset,
         dataset_id=dataset_id,
         collection_id=collection_id,
     )
-    return create_page(file_metas, total=total, params=params)  # type: ignore[return-value]
+    return create_page(file_metas, total=total, params=page_params)
 
 
 @router.get(

services/datcore-adapter/src/simcore_service_datcore_adapter/modules/pennsieve.py

@@ -347,7 +347,11 @@ async def list_datasets(
                 DatasetMetaData(
                     id=d["content"]["id"],
                     display_name=d["content"]["name"],
-                    size=ByteSize(d["storage"]) if d["storage"] > 0 else None,
+                    size=(
+                        ByteSize(sz)
+                        if (sz := d.get("storage", 0)) > 0  # NOSONAR
+                        else None
+                    ),
                 )
                 for d in dataset_page["datasets"]
             ],

services/datcore-adapter/tests/unit/test_route_datasets.py

@@ -5,7 +5,7 @@
 
 import httpx
 import respx
-from fastapi_pagination import Page
+from fastapi_pagination import LimitOffsetPage
 from models_library.api_schemas_datcore_adapter.datasets import (
     DatasetMetaData,
     FileMetaData,
@@ -44,7 +44,7 @@ async def test_list_datasets_entrypoint(
     assert response.status_code == status.HTTP_200_OK
     data = response.json()
     assert data
-    TypeAdapter(Page[DatasetMetaData]).validate_python(data)
+    TypeAdapter(LimitOffsetPage[DatasetMetaData]).validate_python(data)
 
 
 async def test_list_dataset_files_legacy_entrypoint(
@@ -80,7 +80,7 @@ async def test_list_dataset_top_level_files_entrypoint(
     assert response.status_code == status.HTTP_200_OK
     data = response.json()
     assert data
-    TypeAdapter(Page[FileMetaData]).validate_python(data)
+    TypeAdapter(LimitOffsetPage[FileMetaData]).validate_python(data)
 
 
 async def test_list_dataset_collection_files_entrypoint(
@@ -100,4 +100,4 @@ async def test_list_dataset_collection_files_entrypoint(
     assert response.status_code == status.HTTP_200_OK
     data = response.json()
     assert data
-    TypeAdapter(Page[FileMetaData]).validate_python(data)
+    TypeAdapter(LimitOffsetPage[FileMetaData]).validate_python(data)