Merge branch 'master' into fix/path-size

Author: odeimaiz

packages/aws-library/src/aws_library/s3/_client.py

@@ -85,21 +85,30 @@ async def create(
         cls, settings: S3Settings, s3_max_concurrency: int = _S3_MAX_CONCURRENCY_DEFAULT
     ) -> "SimcoreS3API":
         session = aioboto3.Session()
-        session_client = session.client(  # type: ignore[call-overload]
-            "s3",
-            endpoint_url=f"{settings.S3_ENDPOINT}",
-            aws_access_key_id=settings.S3_ACCESS_KEY,
-            aws_secret_access_key=settings.S3_SECRET_KEY,
-            region_name=settings.S3_REGION,
-            config=Config(signature_version="s3v4"),
-        )
-        assert isinstance(session_client, ClientCreatorContext)  # nosec
+        session_client = None
         exit_stack = contextlib.AsyncExitStack()
-        s3_client = cast(S3Client, await exit_stack.enter_async_context(session_client))
-        # NOTE: this triggers a botocore.exception.ClientError in case the connection is not made to the S3 backend
-        await s3_client.list_buckets()
+        try:
+            session_client = session.client(  # type: ignore[call-overload]
+                "s3",
+                endpoint_url=f"{settings.S3_ENDPOINT}",
+                aws_access_key_id=settings.S3_ACCESS_KEY,
+                aws_secret_access_key=settings.S3_SECRET_KEY,
+                region_name=settings.S3_REGION,
+                config=Config(signature_version="s3v4"),
+            )
+            assert isinstance(session_client, ClientCreatorContext)  # nosec
 
-        return cls(s3_client, session, exit_stack, s3_max_concurrency)
+            s3_client = cast(
+                S3Client, await exit_stack.enter_async_context(session_client)
+            )
+            # NOTE: this triggers a botocore.exception.ClientError in case the connection is not made to the S3 backend
+            await s3_client.list_buckets()
+
+            return cls(s3_client, session, exit_stack, s3_max_concurrency)
+        except Exception:
+            await exit_stack.aclose()
+
+            raise
 
     async def close(self) -> None:
         await self._exit_stack.aclose()

packages/models-library/src/models_library/api_schemas_catalog/services.py

@@ -323,11 +323,20 @@ def _update_json_schema_extra(schema: JsonDict) -> None:
     )
 
 
-PageRpcServicesGetV2: TypeAlias = PageRpc[
+PageRpcLatestServiceGet: TypeAlias = PageRpc[
     # WARNING: keep this definition in models_library and not in the RPC interface
+    # otherwise the metaclass PageRpc[*] will create *different* classes in server/client side
+    # and will fail to serialize/deserialize these parameters when transmitted/received
     LatestServiceGet
 ]
 
+PageRpcServiceRelease: TypeAlias = PageRpc[
+    # WARNING: keep this definition in models_library and not in the RPC interface
+    # otherwise the metaclass PageRpc[*] will create *different* classes in server/client side
+    # and will fail to serialize/deserialize these parameters when transmitted/received
+    ServiceRelease
+]
+
 ServiceResourcesGet: TypeAlias = ServiceResourcesDict
 
 
@@ -365,3 +374,6 @@ class MyServiceGet(CatalogOutputSchema):
 
     owner: GroupID | None
     my_access_rights: ServiceGroupAccessRightsV2
+
+
+__all__: tuple[str, ...] = ("ServiceRelease",)

packages/models-library/src/models_library/rest_pagination.py

@@ -28,6 +28,8 @@
 
 PageOffsetInt: TypeAlias = NonNegativeInt
 
+PageTotalCount: TypeAlias = NonNegativeInt
+
 DEFAULT_NUMBER_OF_ITEMS_PER_PAGE: Final[PageLimitInt] = TypeAdapter(
     PageLimitInt
 ).validate_python(20)
@@ -70,7 +72,7 @@ class PageQueryParameters(RequestParameters):
 
 class PageMetaInfoLimitOffset(BaseModel):
     limit: PositiveInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE
-    total: NonNegativeInt
+    total: PageTotalCount
     offset: NonNegativeInt = 0
     count: NonNegativeInt
 

packages/pytest-simcore/src/pytest_simcore/helpers/catalog_rpc_server.py

@@ -0,0 +1,117 @@
+# pylint: disable=not-context-manager
+# pylint: disable=protected-access
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+
+from models_library.api_schemas_catalog.services import LatestServiceGet, ServiceGetV2
+from models_library.api_schemas_webserver.catalog import (
+    CatalogServiceUpdate,
+)
+from models_library.products import ProductName
+from models_library.rest_pagination import PageOffsetInt
+from models_library.rpc_pagination import PageLimitInt, PageRpc
+from models_library.services_history import ServiceRelease
+from models_library.services_types import ServiceKey, ServiceVersion
+from models_library.users import UserID
+from pydantic import NonNegativeInt, TypeAdapter
+from servicelib.rabbitmq._client_rpc import RabbitMQRPCClient
+
+
+class CatalogRpcSideEffects:
+    # pylint: disable=no-self-use
+    async def list_services_paginated(
+        self,
+        rpc_client: RabbitMQRPCClient,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        limit: PageLimitInt,
+        offset: NonNegativeInt,
+    ):
+        assert rpc_client
+        assert product_name
+        assert user_id
+
+        items = TypeAdapter(list[LatestServiceGet]).validate_python(
+            LatestServiceGet.model_json_schema()["examples"],
+        )
+        total_count = len(items)
+
+        return PageRpc[LatestServiceGet].create(
+            items[offset : offset + limit],
+            total=total_count,
+            limit=limit,
+            offset=offset,
+        )
+
+    async def get_service(
+        self,
+        rpc_client: RabbitMQRPCClient,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        service_key: ServiceKey,
+        service_version: ServiceVersion,
+    ):
+        assert rpc_client
+        assert product_name
+        assert user_id
+
+        got = ServiceGetV2.model_validate(
+            ServiceGetV2.model_json_schema()["examples"][0]
+        )
+        got.version = service_version
+        got.key = service_key
+
+        return got
+
+    async def update_service(
+        self,
+        rpc_client: RabbitMQRPCClient,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        service_key: ServiceKey,
+        service_version: ServiceVersion,
+        update: CatalogServiceUpdate,
+    ):
+        assert rpc_client
+        assert product_name
+        assert user_id
+
+        got = ServiceGetV2.model_validate(
+            ServiceGetV2.model_json_schema()["examples"][0]
+        )
+        got.version = service_version
+        got.key = service_key
+        return got.model_copy(update=update.model_dump(exclude_unset=True))
+
+    async def list_my_service_history_paginated(
+        self,
+        rpc_client: RabbitMQRPCClient,
+        *,
+        product_name: ProductName,
+        user_id: UserID,
+        service_key: ServiceKey,
+        offset: PageOffsetInt,
+        limit: PageLimitInt,
+    ) -> PageRpc[ServiceRelease]:
+
+        assert rpc_client
+        assert product_name
+        assert user_id
+        assert service_key
+
+        items = TypeAdapter(list[ServiceRelease]).validate_python(
+            ServiceRelease.model_json_schema()["examples"],
+        )
+        total_count = len(items)
+
+        return PageRpc[ServiceRelease].create(
+            items[offset : offset + limit],
+            total=total_count,
+            limit=limit,
+            offset=offset,
+        )

packages/service-library/src/servicelib/fastapi/http_error.py

@@ -1,3 +1,4 @@
+import logging
 from collections.abc import Awaitable, Callable
 from typing import TypeVar
 
@@ -10,6 +11,9 @@
 from fastapi.responses import JSONResponse
 from pydantic import ValidationError
 
+from ..logging_errors import create_troubleshotting_log_kwargs
+from ..status_codes_utils import is_5xx_server_error
+
 validation_error_response_definition["properties"] = {
     "errors": {
         "title": "Validation errors",
@@ -21,6 +25,8 @@
 
 TException = TypeVar("TException")
 
+_logger = logging.getLogger(__name__)
+
 
 def make_http_error_handler_for_exception(
     status_code: int,
@@ -36,12 +42,24 @@ def make_http_error_handler_for_exception(
     SEE https://docs.python.org/3/library/exceptions.html#concrete-exceptions
     """
 
-    async def _http_error_handler(_: Request, exc: Exception) -> JSONResponse:
+    async def _http_error_handler(request: Request, exc: Exception) -> JSONResponse:
         assert isinstance(exc, exception_cls)  # nosec
         error_content = {
             "errors": error_extractor(exc) if error_extractor else [f"{exc}"]
         }
 
+        if is_5xx_server_error(status_code):
+            _logger.exception(
+                create_troubleshotting_log_kwargs(
+                    "Unexpected error happened in the Resource Usage Tracker. Please contact support.",
+                    error=exc,
+                    error_context={
+                        "request": request,
+                        "request.method": f"{request.method}",
+                    },
+                )
+            )
+
         return JSONResponse(
             content=jsonable_encoder(
                 {"error": error_content} if envelope_error else error_content

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/catalog/services.py

@@ -7,19 +7,23 @@
 from models_library.api_schemas_catalog.services import (
     LatestServiceGet,
     MyServiceGet,
+    PageRpcLatestServiceGet,
+    PageRpcServiceRelease,
     ServiceGetV2,
+    ServiceRelease,
     ServiceUpdateV2,
 )
 from models_library.products import ProductName
 from models_library.rabbitmq_basic_types import RPCMethodName
+from models_library.rest_pagination import PageOffsetInt
 from models_library.rpc_pagination import (
     DEFAULT_NUMBER_OF_ITEMS_PER_PAGE,
     PageLimitInt,
     PageRpc,
 )
 from models_library.services_types import ServiceKey, ServiceVersion
 from models_library.users import UserID
-from pydantic import NonNegativeInt, TypeAdapter, validate_call
+from pydantic import TypeAdapter, validate_call
 from servicelib.logging_utils import log_decorator
 from servicelib.rabbitmq._constants import RPC_REQUEST_DEFAULT_TIMEOUT_S
 
@@ -34,8 +38,8 @@ async def list_services_paginated(  # pylint: disable=too-many-arguments
     product_name: ProductName,
     user_id: UserID,
     limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE,
-    offset: NonNegativeInt = 0,
-) -> PageRpc[LatestServiceGet]:
+    offset: PageOffsetInt = 0,
+) -> PageRpcLatestServiceGet:
     """
     Raises:
         ValidationError: on invalid arguments
@@ -47,7 +51,7 @@ async def _call(
         product_name: ProductName,
         user_id: UserID,
         limit: PageLimitInt,
-        offset: NonNegativeInt,
+        offset: PageOffsetInt,
     ):
         return await rpc_client.request(
             CATALOG_RPC_NAMESPACE,
@@ -235,3 +239,51 @@ async def _call(
     result = await _call(product_name=product_name, user_id=user_id, ids=ids)
     assert TypeAdapter(list[MyServiceGet]).validate_python(result) is not None  # nosec
     return cast(list[MyServiceGet], result)
+
+
+async def list_my_service_history_paginated(  # pylint: disable=too-many-arguments
+    rpc_client: RabbitMQRPCClient,
+    *,
+    product_name: ProductName,
+    user_id: UserID,
+    service_key: ServiceKey,
+    limit: PageLimitInt = DEFAULT_NUMBER_OF_ITEMS_PER_PAGE,
+    offset: PageOffsetInt = 0,
+) -> PageRpcServiceRelease:
+    """
+    Raises:
+        ValidationError: on invalid arguments
+    """
+
+    @validate_call()
+    async def _call(
+        product_name: ProductName,
+        user_id: UserID,
+        service_key: ServiceKey,
+        limit: PageLimitInt,
+        offset: PageOffsetInt,
+    ):
+        return await rpc_client.request(
+            CATALOG_RPC_NAMESPACE,
+            TypeAdapter(RPCMethodName).validate_python(
+                "list_my_service_history_paginated"
+            ),
+            product_name=product_name,
+            user_id=user_id,
+            service_key=service_key,
+            limit=limit,
+            offset=offset,
+        )
+
+    result = await _call(
+        product_name=product_name,
+        user_id=user_id,
+        service_key=service_key,
+        limit=limit,
+        offset=offset,
+    )
+
+    assert (  # nosec
+        TypeAdapter(PageRpcServiceRelease).validate_python(result) is not None
+    )
+    return cast(PageRpc[ServiceRelease], result)

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/storage/paths.py

@@ -5,6 +5,7 @@
     AsyncJobNameData,
 )
 from models_library.api_schemas_storage import STORAGE_RPC_NAMESPACE
+from models_library.products import ProductName
 from models_library.projects_nodes_io import LocationID
 from models_library.rabbitmq_basic_types import RPCMethodName
 from models_library.users import UserID
@@ -17,7 +18,7 @@ async def compute_path_size(
     client: RabbitMQRPCClient,
     *,
     user_id: UserID,
-    product_name: str,
+    product_name: ProductName,
     location_id: LocationID,
     path: Path,
 ) -> tuple[AsyncJobGet, AsyncJobNameData]: