diff --git a/Makefile b/Makefile index d772c8b0510e..61cae56cc0dd 100644 --- a/Makefile +++ b/Makefile @@ -624,8 +624,8 @@ auto-doc: .stack-simcore-version.yml ## Auto generates diagrams for README.md # Updating docs/img @mv --verbose $<.png docs/img/ -.PHONY: services.ignore.md -services.ignore.md: ## Auto generates service.md +.PHONY: SERVICES.md +SERVICES.md: ## Auto generates service.md # Making $@ scripts/echo_services_markdown.py > $@ diff --git a/SERVICES.md b/SERVICES.md new file mode 100644 index 000000000000..ff52fa0dd99e --- /dev/null +++ b/SERVICES.md @@ -0,0 +1,67 @@ +# services +> +> Auto generated on `2025-05-26 09:50:15` using +```cmd +cd osparc-simcore +python ./scripts/echo_services_markdown.py +``` +| Name|Files| | +| ----------|----------|---------- | +| **AGENT**|| | +| |[services/agent/Dockerfile](./services/agent/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/agent)](https://hub.docker.com/r/itisfoundation/agent/tags) | +| **API-SERVER**|| | +| |[services/api-server/openapi.json](./services/api-server/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/api-server/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/api-server/openapi.json) | +| |[services/api-server/Dockerfile](./services/api-server/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/api-server)](https://hub.docker.com/r/itisfoundation/api-server/tags) | +| **AUTOSCALING**|| | +| |[services/autoscaling/Dockerfile](./services/autoscaling/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/autoscaling)](https://hub.docker.com/r/itisfoundation/autoscaling/tags) | +| **CATALOG**|| | +| |[services/catalog/openapi.json](./services/catalog/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/catalog/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/catalog/openapi.json) | +| |[services/catalog/Dockerfile](./services/catalog/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/catalog)](https://hub.docker.com/r/itisfoundation/catalog/tags) | +| **CLUSTERS-KEEPER**|| | +| |[services/clusters-keeper/Dockerfile](./services/clusters-keeper/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/clusters-keeper)](https://hub.docker.com/r/itisfoundation/clusters-keeper/tags) | +| **DASK-SIDECAR**|| | +| |[services/dask-sidecar/Dockerfile](./services/dask-sidecar/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/dask-sidecar)](https://hub.docker.com/r/itisfoundation/dask-sidecar/tags) | +| **DATCORE-ADAPTER**|| | +| |[services/datcore-adapter/Dockerfile](./services/datcore-adapter/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/datcore-adapter)](https://hub.docker.com/r/itisfoundation/datcore-adapter/tags) | +| **DIRECTOR**|| | +| |[services/director/src/simcore_service_director/api/v0/openapi.yaml](./services/director/src/simcore_service_director/api/v0/openapi.yaml)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/director/src/simcore_service_director/api/v0/openapi.yaml) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/director/src/simcore_service_director/api/v0/openapi.yaml) | +| |[services/director/Dockerfile](./services/director/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/director)](https://hub.docker.com/r/itisfoundation/director/tags) | +| **DIRECTOR-V2**|| | +| |[services/director-v2/openapi.json](./services/director-v2/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/director-v2/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/director-v2/openapi.json) | +| |[services/director-v2/Dockerfile](./services/director-v2/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/director-v2)](https://hub.docker.com/r/itisfoundation/director-v2/tags) | +| **DOCKER-API-PROXY**|| | +| |[services/docker-api-proxy/Dockerfile](./services/docker-api-proxy/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/docker-api-proxy)](https://hub.docker.com/r/itisfoundation/docker-api-proxy/tags) | +| **DYNAMIC-SCHEDULER**|| | +| |[services/dynamic-scheduler/openapi.json](./services/dynamic-scheduler/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/dynamic-scheduler/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/dynamic-scheduler/openapi.json) | +| |[services/dynamic-scheduler/Dockerfile](./services/dynamic-scheduler/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/dynamic-scheduler)](https://hub.docker.com/r/itisfoundation/dynamic-scheduler/tags) | +| **DYNAMIC-SIDECAR**|| | +| |[services/dynamic-sidecar/openapi.json](./services/dynamic-sidecar/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/dynamic-sidecar/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/dynamic-sidecar/openapi.json) | +| |[services/dynamic-sidecar/Dockerfile](./services/dynamic-sidecar/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/dynamic-sidecar)](https://hub.docker.com/r/itisfoundation/dynamic-sidecar/tags) | +| **EFS-GUARDIAN**|| | +| |[services/efs-guardian/Dockerfile](./services/efs-guardian/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/efs-guardian)](https://hub.docker.com/r/itisfoundation/efs-guardian/tags) | +| **INVITATIONS**|| | +| |[services/invitations/openapi.json](./services/invitations/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/invitations/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/invitations/openapi.json) | +| |[services/invitations/Dockerfile](./services/invitations/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/invitations)](https://hub.docker.com/r/itisfoundation/invitations/tags) | +| **MIGRATION**|| | +| |[services/migration/Dockerfile](./services/migration/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/migration)](https://hub.docker.com/r/itisfoundation/migration/tags) | +| **NOTIFICATIONS**|| | +| |[services/notifications/openapi.json](./services/notifications/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/notifications/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/notifications/openapi.json) | +| |[services/notifications/Dockerfile](./services/notifications/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/notifications)](https://hub.docker.com/r/itisfoundation/notifications/tags) | +| **PAYMENTS**|| | +| |[services/payments/openapi.json](./services/payments/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/payments/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/payments/openapi.json) | +| |[services/payments/Dockerfile](./services/payments/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/payments)](https://hub.docker.com/r/itisfoundation/payments/tags) | +| **RESOURCE-USAGE-TRACKER**|| | +| |[services/resource-usage-tracker/openapi.json](./services/resource-usage-tracker/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/resource-usage-tracker/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/resource-usage-tracker/openapi.json) | +| |[services/resource-usage-tracker/Dockerfile](./services/resource-usage-tracker/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/resource-usage-tracker)](https://hub.docker.com/r/itisfoundation/resource-usage-tracker/tags) | +| **STATIC-WEBSERVER**|| | +| |[services/static-webserver/client/tools/qooxdoo-kit/builder/Dockerfile](./services/static-webserver/client/tools/qooxdoo-kit/builder/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/static-webserver)](https://hub.docker.com/r/itisfoundation/static-webserver/tags) | +| |[services/static-webserver/client/qx_packages/ITISFoundation_qx-iconfont-material_v0_1_7/Dockerfile](./services/static-webserver/client/qx_packages/ITISFoundation_qx-iconfont-material_v0_1_7/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/static-webserver)](https://hub.docker.com/r/itisfoundation/static-webserver/tags) | +| |[services/static-webserver/client/qx_packages/ITISFoundation_qx-iconfont-fontawesome5_v0_2_2/Dockerfile](./services/static-webserver/client/qx_packages/ITISFoundation_qx-iconfont-fontawesome5_v0_2_2/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/static-webserver)](https://hub.docker.com/r/itisfoundation/static-webserver/tags) | +| |[services/static-webserver/client/qx_packages/ITISFoundation_qx-osparc-theme_v0_5_6/Dockerfile](./services/static-webserver/client/qx_packages/ITISFoundation_qx-osparc-theme_v0_5_6/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/static-webserver)](https://hub.docker.com/r/itisfoundation/static-webserver/tags) | +| **STORAGE**|| | +| |[services/storage/openapi.json](./services/storage/openapi.json)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/storage/openapi.json) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/storage/openapi.json) | +| |[services/storage/Dockerfile](./services/storage/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/storage)](https://hub.docker.com/r/itisfoundation/storage/tags) | +| **WEB**|| | +| |[services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml](./services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml)|[![ReDoc](https://img.shields.io/badge/OpenAPI-ReDoc-85ea2d?logo=openapiinitiative)](https://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml) [![Swagger UI](https://img.shields.io/badge/OpenAPI-Swagger_UI-85ea2d?logo=swagger)](https://petstore.swagger.io/?url=https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml) | +| |[services/web/Dockerfile](./services/web/Dockerfile)|[![Docker Image Size](https://img.shields.io/docker/image-size/itisfoundation/webserver)](https://hub.docker.com/r/itisfoundation/webserver/tags) | +| || | diff --git a/packages/common-library/src/common_library/exclude.py b/packages/common-library/src/common_library/exclude.py index e24efb998c4e..502e99fa32c8 100644 --- a/packages/common-library/src/common_library/exclude.py +++ b/packages/common-library/src/common_library/exclude.py @@ -1,24 +1,32 @@ -from typing import Any +from typing import Any, Final -class UnSet: - VALUE: "UnSet" +class Unset: + """Sentinel value to indicate that a parameter is not set.""" + VALUE: "Unset" -UnSet.VALUE = UnSet() + +unuset: Final = Unset() +Unset.VALUE = Unset() def is_unset(v: Any) -> bool: - return isinstance(v, UnSet) + return isinstance(v, Unset) def is_set(v: Any) -> bool: - return not isinstance(v, UnSet) + return not isinstance(v, Unset) def as_dict_exclude_unset(**params) -> dict[str, Any]: - return {k: v for k, v in params.items() if not isinstance(v, UnSet)} + """Excludes parameters that are instances of UnSet.""" + return {k: v for k, v in params.items() if not isinstance(v, Unset)} def as_dict_exclude_none(**params) -> dict[str, Any]: + """Analogous to `as_dict_exclude_unset` but with None. + + Sometimes None is used as a sentinel value, use this function to exclude it. + """ return {k: v for k, v in params.items() if v is not None} diff --git a/packages/common-library/src/common_library/pagination_tools.py b/packages/common-library/src/common_library/pagination_tools.py index a30f654f6a16..f85482dbf1b7 100644 --- a/packages/common-library/src/common_library/pagination_tools.py +++ b/packages/common-library/src/common_library/pagination_tools.py @@ -29,10 +29,25 @@ def total_number_of_pages(self) -> NonNegativeInt: def iter_pagination_params( - offset: NonNegativeInt = 0, - limit: PositiveInt = 100, + *, + limit: PositiveInt, + offset: NonNegativeInt, total_number_of_items: NonNegativeInt | None = None, ) -> Iterable[PageParams]: + """Iterates through pages of a collection by yielding PageParams for each page. + + Args: + limit: The maximum number of items to return in a single page. + offset: The number of items to skip before starting to collect the items for the current page. + total_number_of_items: The total count of items in the collection being paginated. + Must be set during the first iteration if not provided initially. + + Yields: + PageParams for each page in the collection. + + Raises: + RuntimeError: If total_number_of_items is not set before first iteration or if it changes between iterations. + """ kwargs = {} if total_number_of_items: diff --git a/packages/common-library/tests/test_exclude.py b/packages/common-library/tests/test_exclude.py index 78f5712161ef..a708def57789 100644 --- a/packages/common-library/tests/test_exclude.py +++ b/packages/common-library/tests/test_exclude.py @@ -1,11 +1,11 @@ from typing import Any -from common_library.exclude import UnSet, as_dict_exclude_none, as_dict_exclude_unset +from common_library.exclude import Unset, as_dict_exclude_none, as_dict_exclude_unset def test_as_dict_exclude_unset(): def f( - par1: str | UnSet = UnSet.VALUE, par2: int | UnSet = UnSet.VALUE + par1: str | Unset = Unset.VALUE, par2: int | Unset = Unset.VALUE ) -> dict[str, Any]: return as_dict_exclude_unset(par1=par1, par2=par2) diff --git a/packages/common-library/tests/test_pagination_tools.py b/packages/common-library/tests/test_pagination_tools.py index 56127c038a3a..987eabc0ee2a 100644 --- a/packages/common-library/tests/test_pagination_tools.py +++ b/packages/common-library/tests/test_pagination_tools.py @@ -38,7 +38,9 @@ async def test_iter_pages_args( num_pages = 0 page_args = None - for page_index, page_args in enumerate(iter_pagination_params(offset, limit)): + for page_index, page_args in enumerate( + iter_pagination_params(offset=offset, limit=limit) + ): page_items, page_args.total_number_of_items = await get_page( page_args.offset_current, page_args.limit @@ -74,7 +76,7 @@ def test_fails_if_total_number_of_items_not_set(): RuntimeError, match="page_args.total_number_of_items = total_count", ): - for _ in iter_pagination_params(limit=2): + for _ in iter_pagination_params(offset=0, limit=2): pass @@ -83,6 +85,8 @@ def test_fails_if_total_number_of_items_changes(): RuntimeError, match="total_number_of_items cannot change on every iteration", ): - for page_params in iter_pagination_params(limit=2, total_number_of_items=4): + for page_params in iter_pagination_params( + offset=0, limit=2, total_number_of_items=4 + ): assert page_params.total_number_of_items == 4 page_params.total_number_of_items += 1 diff --git a/packages/models-library/src/models_library/api_schemas_catalog/services.py b/packages/models-library/src/models_library/api_schemas_catalog/services.py index 5b66f46351f8..f94d6be84efb 100644 --- a/packages/models-library/src/models_library/api_schemas_catalog/services.py +++ b/packages/models-library/src/models_library/api_schemas_catalog/services.py @@ -177,9 +177,7 @@ class ServiceSummary(CatalogOutputSchema): version: ServiceVersion name: str description: str - version_display: str | None = None - contact: LowerCaseEmailStr | None @staticmethod @@ -195,6 +193,14 @@ def _update_json_schema_extra(schema: JsonDict) -> None: "version_display": _EXAMPLE_SLEEPER["version_display"], "contact": _EXAMPLE_SLEEPER["contact"], }, + { + "key": _EXAMPLE_SLEEPER["key"], + "version": "100.0.0", + "name": "sleeper", + "description": "short description", + "version_display": "HUGE Release", + "contact": "contact@acme.com", + }, { "key": _EXAMPLE_FILEPICKER["key"], "version": _EXAMPLE_FILEPICKER["version"], diff --git a/packages/models-library/src/models_library/rest_pagination.py b/packages/models-library/src/models_library/rest_pagination.py index e4a815777420..2158d6ba4111 100644 --- a/packages/models-library/src/models_library/rest_pagination.py +++ b/packages/models-library/src/models_library/rest_pagination.py @@ -19,29 +19,40 @@ # Default limit values # - Using same values across all pagination entrypoints simplifies # interconnecting paginated calls +MINIMUM_NUMBER_OF_ITEMS_PER_PAGE: Final[int] = 1 MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE: Final[int] = 50 - PageLimitInt: TypeAlias = Annotated[ - int, Field(ge=1, lt=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE) + int, + Field( + ge=MINIMUM_NUMBER_OF_ITEMS_PER_PAGE, + le=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, + description="The maximum number of items to return in a single page.", + ), +] +PageOffsetInt: TypeAlias = Annotated[ + int, + Field( + ge=0, + description="The number of items to skip before starting to collect the items for the current pag", + ), ] - -PageOffsetInt: TypeAlias = NonNegativeInt - PageTotalCount: TypeAlias = NonNegativeInt + DEFAULT_NUMBER_OF_ITEMS_PER_PAGE: Final[PageLimitInt] = TypeAdapter( PageLimitInt ).validate_python(20) class CursorQueryParameters(RequestParameters): - """Use as pagination options in query parameters""" + """Query parameters for Cursor-Based Pagination + + SEE https://uriyyo-fastapi-pagination.netlify.app/learn/pagination/techniques/#cursor-based-pagination + """ size: PageLimitInt = Field( - default=TypeAdapter(PageLimitInt).validate_python( - DEFAULT_NUMBER_OF_ITEMS_PER_PAGE - ), + default=DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, description="maximum number of items to return (pagination)", ) cursor: Annotated[ @@ -53,21 +64,13 @@ class CursorQueryParameters(RequestParameters): class PageQueryParameters(RequestParameters): - """Use as pagination options in query parameters""" + """Query parameters for Limit-Offset Pagination - limit: Annotated[ - PageLimitInt, - Field( - default=TypeAdapter(PageLimitInt).validate_python( - DEFAULT_NUMBER_OF_ITEMS_PER_PAGE - ), - description="maximum number of items to return (pagination)", - ), - ] - offset: Annotated[ - PageOffsetInt, - Field(default=0, description="index to the first item to return (pagination)"), - ] + SEE https://uriyyo-fastapi-pagination.netlify.app/learn/pagination/techniques/#limit-offset-pagination + """ + + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE + offset: PageOffsetInt = 0 class PageMetaInfoLimitOffset(BaseModel): diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py index 0e06f7d261ed..aba756249830 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py @@ -19,7 +19,11 @@ from models_library.api_schemas_catalog.services_ports import ServicePortGet from models_library.products import ProductName from models_library.rest_pagination import PageOffsetInt -from models_library.rpc_pagination import PageLimitInt, PageRpc +from models_library.rpc_pagination import ( + DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + PageLimitInt, + PageRpc, +) from models_library.services_enums import ServiceType from models_library.services_history import ServiceRelease from models_library.services_regex import ( @@ -28,7 +32,7 @@ ) from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID -from pydantic import NonNegativeInt, TypeAdapter, validate_call +from pydantic import TypeAdapter, validate_call from pytest_mock import MockType from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient @@ -51,8 +55,8 @@ async def list_services_paginated( *, product_name: ProductName, user_id: UserID, - limit: PageLimitInt, - offset: NonNegativeInt, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + offset: PageOffsetInt = 0, filters: ServiceListFilters | None = None, ): assert rpc_client @@ -158,8 +162,8 @@ async def list_my_service_history_latest_first( product_name: ProductName, user_id: UserID, service_key: ServiceKey, - offset: PageOffsetInt, - limit: PageLimitInt, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + offset: PageOffsetInt = 0, filters: ServiceListFilters | None = None, ) -> PageRpc[ServiceRelease]: @@ -208,8 +212,8 @@ async def list_all_services_summaries_paginated( *, product_name: ProductName, user_id: UserID, - limit: PageLimitInt, - offset: NonNegativeInt, + limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + offset: PageOffsetInt = 0, filters: ServiceListFilters | None = None, ): assert rpc_client @@ -274,7 +278,7 @@ async def list_my_service_history_latest_first(self, *args, **kwargs): return PageRpc[ServiceRelease].create( [], total=0, - limit=10, + limit=DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, offset=0, ) @@ -282,6 +286,6 @@ async def list_all_services_summaries_paginated(self, *args, **kwargs): return PageRpc[ServiceSummary].create( [], total=0, - limit=10, + limit=DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, offset=0, ) diff --git a/scripts/echo_services_markdown.py b/scripts/echo_services_markdown.py index cdd384607de0..0a0ebb2dadff 100755 --- a/scripts/echo_services_markdown.py +++ b/scripts/echo_services_markdown.py @@ -1,8 +1,8 @@ #!/bin/env python -""" Usage +"""Usage - cd osparc-simcore - ./scripts/echo_services_markdown.py >services.md +cd osparc-simcore +./scripts/echo_services_markdown.py >services.md """ import itertools @@ -16,9 +16,9 @@ CURRENT_FILE = Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve() CURRENT_DIR = CURRENT_FILE.parent -_URL_PREFIX: Final[ - str -] = "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master" +_URL_PREFIX: Final[str] = ( + "https://raw.githubusercontent.com/ITISFoundation/osparc-simcore/refs/heads/master" +) _REDOC_URL_PREFIX: Final[str] = f"https://redocly.github.io/redoc/?url={_URL_PREFIX}" _SWAGGER_URL_PREFIX: Final[str] = f"https://petstore.swagger.io/?url={_URL_PREFIX}" @@ -110,12 +110,19 @@ def _to_tuple(file: Path): file.relative_to(repo_base_path), ) - dockerfiles_found = (_to_tuple(file) for file in services_path.rglob("Dockerfile")) + def _is_hidden(file: Path) -> bool: + return any(p.name.startswith(".") for p in file.parents) + + dockerfiles_found = ( + _to_tuple(file) + for file in services_path.rglob("Dockerfile") + if not _is_hidden(file) + ) openapi_files_found = ( _to_tuple(file) for file in services_path.rglob("openapi.*") - if file.suffix in {".json", ".yaml", ".yml"} + if file.suffix in {".json", ".yaml", ".yml"} and not _is_hidden(file) ) markdown_table = generate_markdown_table( diff --git a/services/api-server/src/simcore_service_api_server/_service_jobs.py b/services/api-server/src/simcore_service_api_server/_service_jobs.py index 630c55b6c510..faac214897f8 100644 --- a/services/api-server/src/simcore_service_api_server/_service_jobs.py +++ b/services/api-server/src/simcore_service_api_server/_service_jobs.py @@ -2,12 +2,12 @@ from collections.abc import Callable from dataclasses import dataclass +from common_library.exclude import as_dict_exclude_none from models_library.api_schemas_webserver.projects import ProjectCreateNew, ProjectGet from models_library.products import ProductName from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.rest_pagination import ( - MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, PageMetaInfoLimitOffset, PageOffsetInt, ) @@ -43,19 +43,22 @@ async def list_jobs( job_parent_resource_name: str, *, filter_any_custom_metadata: list[NameValueTuple] | None = None, - pagination_offset: PageOffsetInt = 0, - pagination_limit: PageLimitInt = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE - 1, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, ) -> tuple[list[Job], PageMetaInfoLimitOffset]: """Lists all jobs for a user with pagination based on resource name prefix""" + pagination_kwargs = as_dict_exclude_none( + pagination_offset=pagination_offset, pagination_limit=pagination_limit + ) + # 1. List projects marked as jobs projects_page = await self._web_rpc_client.list_projects_marked_as_jobs( product_name=self.product_name, user_id=self.user_id, - pagination_offset=pagination_offset, - pagination_limit=pagination_limit, filter_by_job_parent_resource_name_prefix=job_parent_resource_name, filter_any_custom_metadata=filter_any_custom_metadata, + **pagination_kwargs, ) # 2. Convert projects to jobs diff --git a/services/api-server/src/simcore_service_api_server/_service_programs.py b/services/api-server/src/simcore_service_api_server/_service_programs.py index 252edc3bb231..ebb9ae652d77 100644 --- a/services/api-server/src/simcore_service_api_server/_service_programs.py +++ b/services/api-server/src/simcore_service_api_server/_service_programs.py @@ -2,9 +2,12 @@ from models_library.api_schemas_catalog.services import ServiceListFilters from models_library.basic_types import VersionStr -from models_library.rest_pagination import PageMetaInfoLimitOffset +from models_library.rest_pagination import ( + PageLimitInt, + PageMetaInfoLimitOffset, + PageOffsetInt, +) from models_library.services_enums import ServiceType -from pydantic import NonNegativeInt, PositiveInt from .models.schemas.programs import Program, ProgramKeyId from .services_rpc.catalog import CatalogService @@ -31,8 +34,8 @@ async def get_program( async def list_latest_programs( self, *, - pagination_offset: NonNegativeInt, - pagination_limit: PositiveInt, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, ) -> tuple[list[Program], PageMetaInfoLimitOffset]: page, page_meta = await self.catalog_service.list_latest_releases( pagination_offset=pagination_offset, @@ -47,13 +50,13 @@ async def list_program_history( self, *, program_key: ProgramKeyId, - offset: NonNegativeInt, - limit: PositiveInt, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, ) -> tuple[list[Program], PageMetaInfoLimitOffset]: page, page_meta = await self.catalog_service.list_release_history_latest_first( filter_by_service_key=program_key, - pagination_offset=offset, - pagination_limit=limit, + pagination_offset=pagination_offset, + pagination_limit=pagination_limit, ) if len(page) == 0: return [], page_meta diff --git a/services/api-server/src/simcore_service_api_server/_service_solvers.py b/services/api-server/src/simcore_service_api_server/_service_solvers.py index 28c767cbeea8..3c0aac79721a 100644 --- a/services/api-server/src/simcore_service_api_server/_service_solvers.py +++ b/services/api-server/src/simcore_service_api_server/_service_solvers.py @@ -4,14 +4,12 @@ from models_library.basic_types import VersionStr from models_library.products import ProductName from models_library.rest_pagination import ( - MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, PageMetaInfoLimitOffset, PageOffsetInt, ) from models_library.rpc_pagination import PageLimitInt from models_library.services_enums import ServiceType from models_library.users import UserID -from pydantic import NonNegativeInt, PositiveInt from simcore_service_api_server.models.basic_types import NameValueTuple from ._service_jobs import JobService @@ -27,8 +25,6 @@ from .models.schemas.solvers import Solver, SolverKeyId from .services_rpc.catalog import CatalogService -DEFAULT_PAGINATION_LIMIT = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE - 1 - @dataclass(frozen=True, kw_only=True) class SolverService: @@ -91,11 +87,11 @@ async def get_latest_release( async def list_jobs( self, *, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, filter_by_solver_key: SolverKeyId | None = None, filter_by_solver_version: VersionStr | None = None, filter_any_custom_metadata: list[NameValueTuple] | None = None, - pagination_offset: PageOffsetInt = 0, - pagination_limit: PageLimitInt = DEFAULT_PAGINATION_LIMIT, ) -> tuple[list[Job], PageMetaInfoLimitOffset]: """Lists all solver jobs for a user with pagination""" @@ -125,8 +121,8 @@ async def solver_release_history( self, *, solver_key: SolverKeyId, - pagination_offset: NonNegativeInt, - pagination_limit: PositiveInt, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, ) -> tuple[list[Solver], PageMetaInfoLimitOffset]: releases, page_meta = ( @@ -156,8 +152,8 @@ async def solver_release_history( async def list_all_solvers( self, *, - pagination_offset: NonNegativeInt, - pagination_limit: PositiveInt, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, filter_by_solver_key_pattern: str | None = None, filter_by_version_display_pattern: str | None = None, ) -> tuple[list[Solver], PageMetaInfoLimitOffset]: @@ -197,8 +193,8 @@ async def list_all_solvers( async def latest_solvers( self, *, - pagination_offset: NonNegativeInt, - pagination_limit: PositiveInt, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, filter_by_solver_key_pattern: str | None = None, filter_by_version_display_pattern: str | None = None, ) -> tuple[list[Solver], PageMetaInfoLimitOffset]: diff --git a/services/api-server/src/simcore_service_api_server/_service_studies.py b/services/api-server/src/simcore_service_api_server/_service_studies.py index 0fc31e7d4bb7..733315f33305 100644 --- a/services/api-server/src/simcore_service_api_server/_service_studies.py +++ b/services/api-server/src/simcore_service_api_server/_service_studies.py @@ -36,8 +36,8 @@ async def list_jobs( self, *, filter_by_study_id: StudyID | None = None, - pagination_offset: PageOffsetInt = 0, - pagination_limit: PageLimitInt = DEFAULT_PAGINATION_LIMIT, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, ) -> tuple[list[Job], PageMetaInfoLimitOffset]: """Lists all solver jobs for a user with pagination""" diff --git a/services/api-server/src/simcore_service_api_server/api/dependencies/_utils.py b/services/api-server/src/simcore_service_api_server/api/dependencies/_utils.py index 2982df786ad4..112d7c861d5c 100644 --- a/services/api-server/src/simcore_service_api_server/api/dependencies/_utils.py +++ b/services/api-server/src/simcore_service_api_server/api/dependencies/_utils.py @@ -1,15 +1,11 @@ from typing import Any +from common_library.exclude import as_dict_exclude_none from pydantic.fields import FieldInfo -def _get_query_params(field: FieldInfo) -> dict[str, Any]: - params = {} - - if field.description: - params["description"] = field.description - if field.examples: - params["example"] = next( - (example for example in field.examples if "*" in example), field.examples[0] - ) - return params +def get_query_params(field: FieldInfo) -> dict[str, Any]: + return as_dict_exclude_none( + description=field.description, + examples=field.examples or None, + ) diff --git a/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_function_filters.py b/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_function_filters.py index 640d0f9b743d..bd0e4e49cbee 100644 --- a/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_function_filters.py +++ b/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_function_filters.py @@ -3,7 +3,7 @@ from fastapi import Query from models_library.functions import FunctionIDString, FunctionJobCollectionsListFilters -from ._utils import _get_query_params +from ._utils import get_query_params def get_function_job_collections_filters( @@ -11,7 +11,7 @@ def get_function_job_collections_filters( has_function_id: Annotated[ FunctionIDString | None, Query( - **_get_query_params( + **get_query_params( FunctionJobCollectionsListFilters.model_fields["has_function_id"] ) ), diff --git a/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_solvers_filters.py b/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_solvers_filters.py index 4f09d90947cb..814b6003264d 100644 --- a/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_solvers_filters.py +++ b/services/api-server/src/simcore_service_api_server/api/dependencies/models_schemas_solvers_filters.py @@ -4,18 +4,18 @@ from models_library.basic_types import SafeQueryStr from ...models.schemas.solvers_filters import SolversListFilters -from ._utils import _get_query_params +from ._utils import get_query_params def get_solvers_filters( # pylint: disable=unsubscriptable-object solver_id: Annotated[ SafeQueryStr | None, - Query(**_get_query_params(SolversListFilters.model_fields["solver_id"])), + Query(**get_query_params(SolversListFilters.model_fields["solver_id"])), ] = None, version_display: Annotated[ SafeQueryStr | None, - Query(**_get_query_params(SolversListFilters.model_fields["version_display"])), + Query(**get_query_params(SolversListFilters.model_fields["version_display"])), ] = None, ) -> SolversListFilters: return SolversListFilters( diff --git a/services/api-server/src/simcore_service_api_server/api/routes/programs.py b/services/api-server/src/simcore_service_api_server/api/routes/programs.py index 4bfce2dd9d4c..0f3de6341935 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/programs.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/programs.py @@ -66,7 +66,7 @@ async def list_programs( return create_page( programs, - total=len(programs), + total=page_meta.total, params=page_params, ) @@ -90,8 +90,8 @@ async def list_program_history( ): programs, page_meta = await program_service.list_program_history( program_key=program_key, - offset=page_params.offset, - limit=page_params.limit, + pagination_offset=page_params.offset, + pagination_limit=page_params.limit, ) page_params.limit = page_meta.limit page_params.offset = page_meta.offset @@ -103,7 +103,7 @@ async def list_program_history( return create_page( programs, - total=len(programs), + total=page_meta.total, params=page_params, ) diff --git a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py index 3f4942499d24..c162e3594229 100644 --- a/services/api-server/src/simcore_service_api_server/api/routes/solvers.py +++ b/services/api-server/src/simcore_service_api_server/api/routes/solvers.py @@ -32,9 +32,6 @@ create_route_description, ) -DEFAULT_PAGINATION_LIMIT = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE - 1 - - _logger = logging.getLogger(__name__) _SOLVER_STATUS_CODES: dict[int | str, dict[str, Any]] = { @@ -109,9 +106,14 @@ async def list_all_solvers_paginated( "get_solver_release", solver_key=solver.id, version=solver.version ) - page_params.limit = page_meta.limit - page_params.offset = page_meta.offset - return create_page(solvers, total=len(solvers), params=page_params) + assert page_params.limit == page_meta.limit # nosec + assert page_params.offset == page_meta.offset # nosec + + return create_page( + solvers, + total=page_meta.total, + params=page_params, + ) @router.get( @@ -134,7 +136,9 @@ async def list_solvers_releases( ): latest_solvers: list[Solver] = [] - for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): solvers, page_meta = await solver_service.latest_solvers( pagination_offset=page_params.offset, pagination_limit=page_params.limit, @@ -144,7 +148,9 @@ async def list_solvers_releases( all_solvers = [] for solver in latest_solvers: - for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): solvers, page_meta = await solver_service.solver_release_history( solver_key=solver.id, pagination_offset=page_params.offset, @@ -215,7 +221,9 @@ async def list_solver_releases( url_for: Annotated[Callable, Depends(get_reverse_url_mapper)], ): all_releases: list[Solver] = [] - for page_params in iter_pagination_params(limit=DEFAULT_PAGINATION_LIMIT): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): solvers, page_meta = await solver_service.solver_release_history( solver_key=solver_key, pagination_offset=page_params.offset, @@ -263,7 +271,7 @@ async def list_solver_releases_paginated( page_params.offset = page_meta.offset return create_page( solvers, - total=len(solvers), + total=page_meta.total, params=page_params, ) diff --git a/services/api-server/src/simcore_service_api_server/models/pagination.py b/services/api-server/src/simcore_service_api_server/models/pagination.py index 935cd044f203..1c00ecccf427 100644 --- a/services/api-server/src/simcore_service_api_server/models/pagination.py +++ b/services/api-server/src/simcore_service_api_server/models/pagination.py @@ -15,6 +15,7 @@ from models_library.rest_pagination import ( DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, + MINIMUM_NUMBER_OF_ITEMS_PER_PAGE, ) from pydantic import ( BaseModel, @@ -32,8 +33,9 @@ # Customizes the default and maximum to fit those of the web-server. It simplifies interconnection UseParamsFields( limit=Query( + # NOTE: in sync with PageLimitInt DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, - ge=1, + ge=MINIMUM_NUMBER_OF_ITEMS_PER_PAGE, le=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, description="Page size limit", ) diff --git a/services/api-server/src/simcore_service_api_server/services_http/storage.py b/services/api-server/src/simcore_service_api_server/services_http/storage.py index 316902976124..ae82609a58aa 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/storage.py +++ b/services/api-server/src/simcore_service_api_server/services_http/storage.py @@ -20,7 +20,8 @@ ) from models_library.basic_types import SHA256Str from models_library.generics import Envelope -from pydantic import AnyUrl, PositiveInt +from models_library.rest_pagination import PageLimitInt, PageOffsetInt +from pydantic import AnyUrl from settings_library.tracing import TracingSettings from starlette.datastructures import URL @@ -97,8 +98,8 @@ async def search_owned_files( user_id: int, file_id: UUID | None, sha256_checksum: SHA256Str | None = None, - limit: PositiveInt | None = None, - offset: PositiveInt | None = None, + limit: PageLimitInt | None = None, + offset: PageOffsetInt | None = None, ) -> list[StorageFileMetaData]: # NOTE: can NOT use /locations/0/files/metadata with uuid_filter=api/ because # logic in storage 'wrongly' assumes that all data is associated to a project and diff --git a/services/api-server/src/simcore_service_api_server/services_http/webserver.py b/services/api-server/src/simcore_service_api_server/services_http/webserver.py index 8b84ba1af6c5..7751ffb382fb 100644 --- a/services/api-server/src/simcore_service_api_server/services_http/webserver.py +++ b/services/api-server/src/simcore_service_api_server/services_http/webserver.py @@ -38,7 +38,7 @@ from models_library.generics import Envelope from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID -from models_library.rest_pagination import Page +from models_library.rest_pagination import Page, PageLimitInt, PageOffsetInt from models_library.utils.fastapi_encoders import jsonable_encoder from pydantic import PositiveInt from servicelib.aiohttp.long_running_tasks.server import TaskStatus @@ -353,7 +353,7 @@ async def get_project(self, *, project_id: UUID) -> ProjectGet: return data async def get_projects_w_solver_page( - self, *, solver_name: str, limit: int, offset: int + self, *, solver_name: str, limit: PageLimitInt, offset: PageOffsetInt ) -> Page[ProjectGet]: assert not solver_name.endswith("/") # nosec @@ -364,7 +364,7 @@ async def get_projects_w_solver_page( search_by_project_name=solver_name, ) - async def get_projects_page(self, *, limit: int, offset: int): + async def get_projects_page(self, *, limit: PageLimitInt, offset: PageOffsetInt): return await self._page_projects( limit=limit, offset=offset, diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py index 32a4b9ce17e0..f8dbe035e6bc 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/catalog.py @@ -1,6 +1,8 @@ from dataclasses import dataclass from functools import partial +from typing import Any +from common_library.exclude import as_dict_exclude_none from models_library.api_schemas_catalog.services import ( LatestServiceGet, ServiceGetV2, @@ -10,7 +12,6 @@ from models_library.api_schemas_catalog.services_ports import ServicePortGet from models_library.products import ProductName from models_library.rest_pagination import ( - DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, PageLimitInt, PageMetaInfoLimitOffset, PageOffsetInt, @@ -36,6 +37,16 @@ _exception_mapper = partial(service_exception_mapper, service_name="CatalogService") +def _create_page_meta_info(page: Any) -> PageMetaInfoLimitOffset: + """Creates a PageMetaInfoLimitOffset from an RPC response page.""" + return PageMetaInfoLimitOffset( + limit=page.meta.limit, + offset=page.meta.offset, + total=page.meta.total, + count=page.meta.count, + ) + + @dataclass(frozen=True, kw_only=True) class CatalogService: _rpc_client: RabbitMQRPCClient @@ -45,56 +56,53 @@ class CatalogService: async def list_latest_releases( self, *, - pagination_offset: PageOffsetInt = 0, - pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, filters: ServiceListFilters | None = None, ) -> tuple[list[LatestServiceGet], PageMetaInfoLimitOffset]: + pagination_kwargs = as_dict_exclude_none( + offset=pagination_offset, limit=pagination_limit + ) + page = await catalog_rpc.list_services_paginated( self._rpc_client, product_name=self.product_name, user_id=self.user_id, - offset=pagination_offset, - limit=pagination_limit, filters=filters, + **pagination_kwargs, ) - meta = PageMetaInfoLimitOffset( - limit=page.meta.limit, - offset=page.meta.offset, - total=page.meta.total, - count=page.meta.count, - ) + meta = _create_page_meta_info(page) return page.data, meta async def list_release_history_latest_first( self, *, filter_by_service_key: ServiceKey, - pagination_offset: PageOffsetInt = 0, - pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, ) -> tuple[list[ServiceRelease], PageMetaInfoLimitOffset]: + pagination_kwargs = as_dict_exclude_none( + offset=pagination_offset, limit=pagination_limit + ) + page = await catalog_rpc.list_my_service_history_latest_first( self._rpc_client, product_name=self.product_name, user_id=self.user_id, service_key=filter_by_service_key, - offset=pagination_offset, - limit=pagination_limit, - ) - meta = PageMetaInfoLimitOffset( - limit=page.meta.limit, - offset=page.meta.offset, - total=page.meta.total, - count=page.meta.count, + **pagination_kwargs, ) + + meta = _create_page_meta_info(page) return page.data, meta async def list_all_services_summaries( self, *, - pagination_offset: PageOffsetInt = 0, - pagination_limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, + pagination_offset: PageOffsetInt | None = None, + pagination_limit: PageLimitInt | None = None, filters: ServiceListFilters | None = None, ) -> tuple[list[ServiceSummary], PageMetaInfoLimitOffset]: """Lists all services with pagination, including all versions of each service. @@ -109,20 +117,19 @@ async def list_all_services_summaries( Returns: Tuple containing list of service summaries and pagination metadata """ + + pagination_kwargs = as_dict_exclude_none( + offset=pagination_offset, limit=pagination_limit + ) + page = await catalog_rpc.list_all_services_summaries_paginated( self._rpc_client, product_name=self.product_name, user_id=self.user_id, - offset=pagination_offset, - limit=pagination_limit, filters=filters, + **pagination_kwargs, ) - meta = PageMetaInfoLimitOffset( - limit=page.meta.limit, - offset=page.meta.offset, - total=page.meta.total, - count=page.meta.count, - ) + meta = _create_page_meta_info(page) return page.data, meta @_exception_mapper( diff --git a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py index c448e0cdec8b..f5f4534e71e0 100644 --- a/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py +++ b/services/api-server/src/simcore_service_api_server/services_rpc/wb_api_server.py @@ -4,6 +4,7 @@ from functools import partial from typing import cast +from common_library.exclude import as_dict_exclude_none from fastapi import FastAPI from fastapi_pagination import create_page from models_library.api_schemas_api_server.functions import ( @@ -254,6 +255,10 @@ async def list_projects_marked_as_jobs( filter_by_job_parent_resource_name_prefix: str | None, filter_any_custom_metadata: list[NameValueTuple] | None, ): + pagination_kwargs = as_dict_exclude_none( + offset=pagination_offset, limit=pagination_limit + ) + filters = ListProjectsMarkedAsJobRpcFilters( job_parent_resource_name_prefix=filter_by_job_parent_resource_name_prefix, any_custom_metadata=( @@ -270,9 +275,8 @@ async def list_projects_marked_as_jobs( rpc_client=self._client, product_name=product_name, user_id=user_id, - offset=pagination_offset, - limit=pagination_limit, filters=filters, + **pagination_kwargs, ) async def register_function(self, *, function: Function) -> RegisteredFunction: diff --git a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py index 83fa6a0ce523..300bfaab9632 100644 --- a/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py +++ b/services/api-server/tests/unit/api_solvers/test_api_routers_solvers.py @@ -172,3 +172,65 @@ async def test_list_solver_ports_again( ) assert response.status_code == status.HTTP_200_OK assert TypeAdapter(OnePage[SolverPort]).validate_python(response.json()) + + +async def test_solvers_page_pagination_links( + mocked_catalog_rpc_api: dict[str, MockType], + client: httpx.AsyncClient, + auth: httpx.BasicAuth, +): + # Use a small limit to ensure pagination is needed + limit = 2 + response = await client.get(f"/{API_VTAG}/solvers/page?limit={limit}", auth=auth) + + assert response.status_code == status.HTTP_200_OK + + response_data = response.json() + assert "links" in response_data, "Response should contain links section" + + links = response_data["links"] + assert "next" in links, "Pagination should include 'next' link" + assert "prev" in links, "Pagination should include 'prev' link" + assert "first" in links, "Pagination should include 'first' link" + assert "last" in links, "Pagination should include 'last' link" + assert "self" in links, "Pagination should include 'self' link" + + # Verify the self link contains the correct limit parameter + assert ( + f"limit={limit}" in links["self"] + ), "Self link should reflect the requested limit" + + +async def test_solvers_page_pagination_last_page( + mocked_catalog_rpc_api: dict[str, MockType], + client: httpx.AsyncClient, + auth: httpx.BasicAuth, +): + # Get total count first + response = await client.get(f"/{API_VTAG}/solvers/page", auth=auth) + assert response.status_code == status.HTTP_200_OK + total_items = response.json()["total"] + + assert ( + total_items > 1 + ), "Total items in MOCK examples should be greater than 1 for pagination test since we need 'prev', 'self' and 'prev' links" + last_item = total_items - 1 + page_size = 1 + + # Request the last page by using the total count as offset + response = await client.get( + f"/{API_VTAG}/solvers/page?limit={page_size}&offset={last_item}", auth=auth + ) + assert response.status_code == status.HTTP_200_OK + + response_data = response.json() + assert "links" in response_data, "Response should contain links section" + + links = response_data["links"] + assert links["next"] is None, "Next link should be None for the last page (size=1)" + assert ( + links["prev"] is not None + ), "Prev link should be present for the last page (size=1)" + assert ( + links["last"] == links["self"] + ), "Last link should be the same as self link for the last page" diff --git a/services/web/server/VERSION b/services/web/server/VERSION index e40e4fc339c6..328185caaeb7 100644 --- a/services/web/server/VERSION +++ b/services/web/server/VERSION @@ -1 +1 @@ -0.66.0 +0.67.0 diff --git a/services/web/server/setup.cfg b/services/web/server/setup.cfg index 86f86bf7dcb5..13e5b2c3d4e7 100644 --- a/services/web/server/setup.cfg +++ b/services/web/server/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.66.0 +current_version = 0.67.0 commit = True message = services/webserver api version: {current_version} → {new_version} tag = False @@ -13,13 +13,13 @@ commit_args = --no-verify addopts = --strict-markers asyncio_mode = auto asyncio_default_fixture_loop_scope = function -markers = +markers = slow: marks tests as slow (deselect with '-m "not slow"') acceptance_test: "marks tests as 'acceptance tests' i.e. does the system do what the user expects? Typically those are workflows." testit: "marks test to run during development" heavy_load: "mark tests that require large amount of data" [mypy] -plugins = +plugins = pydantic.mypy sqlalchemy.ext.mypy.plugin diff --git a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml index 44347cae8c56..62cabc53c13f 100644 --- a/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml +++ b/services/web/server/src/simcore_service_webserver/api/v0/openapi.yaml @@ -2,7 +2,7 @@ openapi: 3.1.0 info: title: simcore-service-webserver description: Main service with an interface (http-API & websockets) to the web front-end - version: 0.66.0 + version: 0.67.0 servers: - url: '' description: webserver @@ -1616,11 +1616,10 @@ paths: required: false schema: type: integer + maximum: 50 minimum: 1 - exclusiveMaximum: true default: 20 title: Limit - maximum: 50 - name: offset in: query required: false @@ -2062,11 +2061,10 @@ paths: required: false schema: type: integer + maximum: 50 minimum: 1 - exclusiveMaximum: true default: 20 title: Limit - maximum: 50 - name: offset in: query required: false @@ -5075,11 +5073,10 @@ paths: required: false schema: type: integer + maximum: 50 minimum: 1 - exclusiveMaximum: true default: 20 title: Limit - maximum: 50 - name: offset in: query required: false @@ -5243,11 +5240,10 @@ paths: required: false schema: type: integer + maximum: 50 minimum: 1 - exclusiveMaximum: true default: 20 title: Limit - maximum: 50 - name: offset in: query required: false diff --git a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py index 0076c48ae0e6..140f0594b779 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py +++ b/services/web/server/src/simcore_service_webserver/folders/_folders_repository.py @@ -5,7 +5,7 @@ import sqlalchemy as sa from aiohttp import web -from common_library.exclude import UnSet, as_dict_exclude_unset, is_set +from common_library.exclude import Unset, as_dict_exclude_unset, is_set from models_library.folders import ( FolderDB, FolderID, @@ -265,9 +265,9 @@ async def list_folders_db_as_admin( connection: AsyncConnection | None = None, *, # filter - trashed_explicitly: bool | UnSet = UnSet.VALUE, - trashed_before: datetime | UnSet = UnSet.VALUE, - shared_workspace_id: WorkspaceID | UnSet = UnSet.VALUE, # <-- Workspace filter + trashed_explicitly: bool | Unset = Unset.VALUE, + trashed_before: datetime | Unset = Unset.VALUE, + shared_workspace_id: WorkspaceID | Unset = Unset.VALUE, # <-- Workspace filter # pagination offset: NonNegativeInt, limit: int, @@ -382,13 +382,13 @@ async def update( folders_id_or_ids: FolderID | set[FolderID], product_name: ProductName, # updatable columns - name: str | UnSet = UnSet.VALUE, - parent_folder_id: FolderID | None | UnSet = UnSet.VALUE, - trashed: datetime | None | UnSet = UnSet.VALUE, - trashed_explicitly: bool | UnSet = UnSet.VALUE, - trashed_by: UserID | None | UnSet = UnSet.VALUE, # who trashed - workspace_id: WorkspaceID | None | UnSet = UnSet.VALUE, - user_id: UserID | None | UnSet = UnSet.VALUE, # ownership + name: str | Unset = Unset.VALUE, + parent_folder_id: FolderID | None | Unset = Unset.VALUE, + trashed: datetime | None | Unset = Unset.VALUE, + trashed_explicitly: bool | Unset = Unset.VALUE, + trashed_by: UserID | None | Unset = Unset.VALUE, # who trashed + workspace_id: WorkspaceID | None | Unset = Unset.VALUE, + user_id: UserID | None | Unset = Unset.VALUE, # ownership ) -> FolderDB: """ Batch/single patch of folder/s diff --git a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py index 6f5765bfce95..266e1447a112 100644 --- a/services/web/server/src/simcore_service_webserver/folders/_trash_service.py +++ b/services/web/server/src/simcore_service_webserver/folders/_trash_service.py @@ -225,7 +225,9 @@ async def list_explicitly_trashed_folders( ) -> list[FolderID]: trashed_folder_ids: list[FolderID] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( folders, page_params.total_number_of_items, @@ -301,7 +303,9 @@ async def batch_delete_trashed_folders_as_admin( """ errors: list[tuple[FolderID, Exception]] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( page_params.total_number_of_items, expired_trashed_folders, @@ -349,7 +353,9 @@ async def batch_delete_folders_with_content_in_root_workspace_as_admin( deleted_folder_ids: list[FolderID] = [] errors: list[tuple[FolderID, Exception]] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( page_params.total_number_of_items, folders_for_deletion, diff --git a/services/web/server/src/simcore_service_webserver/projects/_folders_repository.py b/services/web/server/src/simcore_service_webserver/projects/_folders_repository.py index f9c4e689eac5..6c1755eb2450 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_folders_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_folders_repository.py @@ -2,7 +2,7 @@ from datetime import datetime from aiohttp import web -from common_library.exclude import UnSet, as_dict_exclude_unset +from common_library.exclude import Unset, as_dict_exclude_unset from models_library.folders import FolderID from models_library.projects import ProjectID from models_library.users import UserID @@ -118,7 +118,7 @@ async def update_project_to_folder( *, folders_id_or_ids: FolderID | set[FolderID], # updatable columns - user_id: UserID | None | UnSet = UnSet.VALUE, + user_id: UserID | None | Unset = Unset.VALUE, ) -> None: """ Batch/single patch of project to folders diff --git a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py index 3460bc733180..705e4970ee7a 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py +++ b/services/web/server/src/simcore_service_webserver/projects/_projects_repository.py @@ -5,7 +5,7 @@ import sqlalchemy as sa from aiohttp import web -from common_library.exclude import UnSet, is_set +from common_library.exclude import Unset, is_set from models_library.basic_types import IDStr from models_library.groups import GroupID from models_library.projects import ProjectID @@ -54,9 +54,9 @@ async def list_projects_db_get_as_admin( connection: AsyncConnection | None = None, *, # filter - trashed_explicitly: bool | UnSet = UnSet.VALUE, - trashed_before: datetime | UnSet = UnSet.VALUE, - shared_workspace_id: WorkspaceID | UnSet = UnSet.VALUE, + trashed_explicitly: bool | Unset = Unset.VALUE, + trashed_before: datetime | Unset = Unset.VALUE, + shared_workspace_id: WorkspaceID | Unset = Unset.VALUE, # pagination offset: NonNegativeInt = 0, limit: PositiveInt = MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE, diff --git a/services/web/server/src/simcore_service_webserver/projects/_trash_service.py b/services/web/server/src/simcore_service_webserver/projects/_trash_service.py index dab8f3aafa42..926345a3a356 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_trash_service.py +++ b/services/web/server/src/simcore_service_webserver/projects/_trash_service.py @@ -161,7 +161,9 @@ async def list_explicitly_trashed_projects( """ trashed_projects: list[ProjectID] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( projects, page_params.total_number_of_items, @@ -238,7 +240,9 @@ async def batch_delete_trashed_projects_as_admin( deleted_project_ids: list[ProjectID] = [] errors: list[tuple[ProjectID, Exception]] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( page_params.total_number_of_items, expired_trashed_projects, @@ -291,7 +295,9 @@ async def batch_delete_projects_in_root_workspace_as_admin( deleted_project_ids: list[ProjectID] = [] errors: list[tuple[ProjectID, Exception]] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( page_params.total_number_of_items, projects_for_deletion, diff --git a/services/web/server/src/simcore_service_webserver/users/_users_repository.py b/services/web/server/src/simcore_service_webserver/users/_users_repository.py index af7e3232b45c..41c358cb520a 100644 --- a/services/web/server/src/simcore_service_webserver/users/_users_repository.py +++ b/services/web/server/src/simcore_service_webserver/users/_users_repository.py @@ -4,7 +4,7 @@ import sqlalchemy as sa from aiohttp import web -from common_library.exclude import UnSet, is_unset +from common_library.exclude import Unset, is_unset from common_library.users_enums import AccountRequestStatus, UserRole from models_library.groups import GroupID from models_library.products import ProductName @@ -565,7 +565,7 @@ async def list_user_pre_registrations( connection: AsyncConnection | None = None, *, filter_by_pre_email: str | None = None, - filter_by_product_name: ProductName | UnSet = UnSet.VALUE, + filter_by_product_name: ProductName | Unset = Unset.VALUE, filter_by_account_request_status: AccountRequestStatus | None = None, pagination_limit: int = 50, pagination_offset: int = 0, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_trash_service.py b/services/web/server/src/simcore_service_webserver/workspaces/_trash_service.py index a0558863ab1e..249b1f0d21d0 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_trash_service.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_trash_service.py @@ -63,7 +63,9 @@ async def _list_root_child_folders( ) -> list[FolderID]: child_folders: list[FolderID] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( folders, page_params.total_number_of_items, @@ -93,7 +95,9 @@ async def _list_root_child_projects( ) -> list[ProjectID]: child_projects: list[ProjectID] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( projects, page_params.total_number_of_items, @@ -285,7 +289,9 @@ async def list_trashed_workspaces( ) -> list[WorkspaceID]: trashed_workspace_ids: list[WorkspaceID] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( page_params.total_number_of_items, workspaces, @@ -326,7 +332,9 @@ async def batch_delete_trashed_workspaces_as_admin( deleted_workspace_ids: list[WorkspaceID] = [] errors: list[tuple[WorkspaceID, Exception]] = [] - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( page_params.total_number_of_items, expired_trashed_workspaces, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py index 4e3744331503..b08e86284e7b 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_repository.py @@ -3,7 +3,7 @@ from typing import cast from aiohttp import web -from common_library.exclude import UnSet, is_set +from common_library.exclude import Unset, is_set from models_library.groups import GroupID from models_library.products import ProductName from models_library.rest_ordering import OrderBy, OrderDirection @@ -277,7 +277,7 @@ async def list_workspaces_db_get_as_admin( connection: AsyncConnection | None = None, *, # filter - trashed_before: datetime | UnSet = UnSet.VALUE, + trashed_before: datetime | Unset = Unset.VALUE, # pagination offset: NonNegativeInt, limit: int, diff --git a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py index cbb376e41307..4779e39fbd97 100644 --- a/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py +++ b/services/web/server/src/simcore_service_webserver/workspaces/_workspaces_service.py @@ -99,7 +99,9 @@ async def delete_workspace_with_all_content( ) # Get all root projects - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( projects, page_params.total_number_of_items, @@ -131,7 +133,9 @@ async def delete_workspace_with_all_content( ) # Get all root folders - for page_params in iter_pagination_params(limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE): + for page_params in iter_pagination_params( + offset=0, limit=MAXIMUM_NUMBER_OF_ITEMS_PER_PAGE + ): ( folders, page_params.total_number_of_items,