Skip to content

🐛Datcore-Adapter: fix listing files in pennsieve pagination sized 1000 #7464

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 3, 2025
Merged

🐛Datcore-Adapter: fix listing files in pennsieve pagination sized 1000 #7464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
988139e
fix
sanderegg Apr 1, 2025
d673aef
mypy
sanderegg Apr 1, 2025
9996464
@pcrespov review: tip
sanderegg Apr 1, 2025
51e3a50
fix tests
sanderegg Apr 3, 2025
91910c7
test fixes
sanderegg Apr 3, 2025
9b38341
sonar
sanderegg Apr 3, 2025
f05cf5f
Merge branch 'master' into datcore-adapter/allow-for-1000-files
mergify[bot] Apr 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 48 additions & 33 deletions services/datcore-adapter/src/simcore_service_datcore_adapter/api/rest/datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,20 @@
import logging
from typing import Annotated, Final
from typing import Annotated, Final, TypeAlias, TypeVar

from aiocache import cached # type: ignore[import-untyped]
from fastapi import APIRouter, Depends, Header, Request
from fastapi_pagination import Page, Params
from fastapi import APIRouter, Depends, Header, Query, Request
from fastapi_pagination import LimitOffsetPage, Params
from fastapi_pagination.api import create_page, resolve_params
from fastapi_pagination.bases import RawParams
from fastapi_pagination.customization import CustomizedPage, UseParamsFields
from models_library.api_schemas_datcore_adapter.datasets import (
DatasetMetaData,
FileMetaData,
)
from models_library.api_schemas_storage.storage_schemas import (
DEFAULT_NUMBER_OF_PATHS_PER_PAGE,
MAX_NUMBER_OF_PATHS_PER_PAGE,
)
from servicelib.fastapi.requests_decorators import cancel_on_disconnect
from starlette import status

Expand All @@ -25,35 +30,47 @@
) # NOTE: this caching time is arbitrary


_T = TypeVar("_T")
_CustomPage = CustomizedPage[
LimitOffsetPage[_T],
UseParamsFields(
limit=Query(
DEFAULT_NUMBER_OF_PATHS_PER_PAGE, ge=1, le=MAX_NUMBER_OF_PATHS_PER_PAGE
),
),
]

_CustomizedPageParams: TypeAlias = _CustomPage.__params_type__ # type: ignore


@router.get(
"/datasets",
summary="list datasets",
status_code=status.HTTP_200_OK,
response_model=Page[DatasetMetaData],
response_model=_CustomPage[DatasetMetaData],
)
@cancel_on_disconnect
@cached(
ttl=_PENNSIEVE_CACHING_TTL_S,
key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['params']}",
key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['page_params']}",
)
async def list_datasets(
request: Request,
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
params: Annotated[Params, Depends()],
) -> Page[DatasetMetaData]:
page_params: Annotated[_CustomizedPageParams, Depends()],
):
assert request # nosec
raw_params: RawParams = resolve_params(params).to_raw_params()
assert raw_params.limit is not None # nosec
assert raw_params.offset is not None # nosec
assert page_params.limit is not None # nosec
assert page_params.offset is not None # nosec
datasets, total = await pennsieve_client.list_datasets(
api_key=x_datcore_api_key,
api_secret=x_datcore_api_secret,
limit=raw_params.limit,
offset=raw_params.offset,
limit=page_params.limit,
offset=page_params.offset,
)
return create_page(datasets, total=total, params=params) # type: ignore[return-value]
return create_page(datasets, total=total, params=page_params)


@router.get(
Expand Down Expand Up @@ -85,46 +102,45 @@ async def get_dataset(
"/datasets/{dataset_id}/files",
summary="list top level files/folders in a dataset",
status_code=status.HTTP_200_OK,
response_model=Page[FileMetaData],
response_model=_CustomPage[FileMetaData],
)
@cancel_on_disconnect
@cached(
ttl=_PENNSIEVE_CACHING_TTL_S,
key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['dataset_id']}_{kwargs['params']}",
key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['dataset_id']}_{kwargs['page_params']}",
)
async def list_dataset_top_level_files(
request: Request,
dataset_id: str,
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
params: Annotated[Params, Depends()],
) -> Page[FileMetaData]:
page_params: Annotated[_CustomizedPageParams, Depends()],
):
assert request # nosec
raw_params: RawParams = resolve_params(params).to_raw_params()

assert raw_params.limit is not None # nosec
assert raw_params.offset is not None # nosec
assert page_params.limit is not None # nosec
assert page_params.offset is not None # nosec
file_metas, total = await pennsieve_client.list_packages_in_dataset(
api_key=x_datcore_api_key,
api_secret=x_datcore_api_secret,
dataset_id=dataset_id,
limit=raw_params.limit,
offset=raw_params.offset,
limit=page_params.limit,
offset=page_params.offset,
)
return create_page(file_metas, total=total, params=params) # type: ignore[return-value]
return create_page(file_metas, total=total, params=page_params)


@router.get(
"/datasets/{dataset_id}/files/{collection_id}",
summary="list top level files/folders in a collection in a dataset",
status_code=status.HTTP_200_OK,
response_model=Page[FileMetaData],
response_model=_CustomPage[FileMetaData],
)
@cancel_on_disconnect
@cached(
ttl=_PENNSIEVE_CACHING_TTL_S,
key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['dataset_id']}_{kwargs['collection_id']}_{kwargs['params']}",
key_builder=lambda f, *args, **kwargs: f"{f.__name__}_{kwargs['x_datcore_api_key']}_{kwargs['x_datcore_api_secret']}_{kwargs['dataset_id']}_{kwargs['collection_id']}_{kwargs['page_params']}",
)
async def list_dataset_collection_files(
request: Request,
Expand All @@ -133,21 +149,20 @@ async def list_dataset_collection_files(
x_datcore_api_key: Annotated[str, Header(..., description="Datcore API Key")],
x_datcore_api_secret: Annotated[str, Header(..., description="Datcore API Secret")],
pennsieve_client: Annotated[PennsieveApiClient, Depends(get_pennsieve_api_client)],
params: Annotated[Params, Depends()],
) -> Page[FileMetaData]:
page_params: Annotated[_CustomizedPageParams, Depends()],
):
assert request # nosec
raw_params: RawParams = resolve_params(params).to_raw_params()
assert raw_params.limit is not None # nosec
assert raw_params.offset is not None # nosec
assert page_params.limit is not None # nosec
assert page_params.offset is not None # nosec
file_metas, total = await pennsieve_client.list_packages_in_collection(
api_key=x_datcore_api_key,
api_secret=x_datcore_api_secret,
limit=raw_params.limit,
offset=raw_params.offset,
limit=page_params.limit,
offset=page_params.offset,
dataset_id=dataset_id,
collection_id=collection_id,
)
return create_page(file_metas, total=total, params=params) # type: ignore[return-value]
return create_page(file_metas, total=total, params=page_params)


@router.get(
Expand Down
6 changes: 5 additions & 1 deletion services/datcore-adapter/src/simcore_service_datcore_adapter/modules/pennsieve.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,11 @@ async def list_datasets(
DatasetMetaData(
id=d["content"]["id"],
display_name=d["content"]["name"],
size=ByteSize(d["storage"]) if d["storage"] > 0 else None,
size=(
ByteSize(sz)
if (sz := d.get("storage", 0)) > 0 # NOSONAR
else None
),
)
for d in dataset_page["datasets"]
],
Expand Down
8 changes: 4 additions & 4 deletions services/datcore-adapter/tests/unit/test_route_datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import httpx
import respx
from fastapi_pagination import Page
from fastapi_pagination import LimitOffsetPage
from models_library.api_schemas_datcore_adapter.datasets import (
DatasetMetaData,
FileMetaData,
Expand Down Expand Up @@ -44,7 +44,7 @@ async def test_list_datasets_entrypoint(
assert response.status_code == status.HTTP_200_OK
data = response.json()
assert data
TypeAdapter(Page[DatasetMetaData]).validate_python(data)
TypeAdapter(LimitOffsetPage[DatasetMetaData]).validate_python(data)


async def test_list_dataset_files_legacy_entrypoint(
Expand Down Expand Up @@ -80,7 +80,7 @@ async def test_list_dataset_top_level_files_entrypoint(
assert response.status_code == status.HTTP_200_OK
data = response.json()
assert data
TypeAdapter(Page[FileMetaData]).validate_python(data)
TypeAdapter(LimitOffsetPage[FileMetaData]).validate_python(data)


async def test_list_dataset_collection_files_entrypoint(
Expand All @@ -100,4 +100,4 @@ async def test_list_dataset_collection_files_entrypoint(
assert response.status_code == status.HTTP_200_OK
data = response.json()
assert data
TypeAdapter(Page[FileMetaData]).validate_python(data)
TypeAdapter(LimitOffsetPage[FileMetaData]).validate_python(data)
Loading