Merge remote-tracking branch 'upstream/master' into extract-celery-code

Author: giancarloromeo

packages/models-library/src/models_library/functions_errors.py

@@ -2,84 +2,100 @@
 
 
 class FunctionBaseError(OsparcErrorMixin, Exception):
-    pass
+    status_code: int
 
 
 class FunctionJobReadAccessDeniedError(FunctionBaseError):
     msg_template: str = (
         "Function job {function_job_id} read access denied for user {user_id}"
     )
+    status_code: int = 403  # Forbidden
 
 
 class FunctionIDNotFoundError(FunctionBaseError):
     msg_template: str = "Function {function_id} not found"
+    status_code: int = 404  # Not Found
 
 
 class FunctionJobIDNotFoundError(FunctionBaseError):
     msg_template: str = "Function job {function_job_id} not found"
+    status_code: int = 404  # Not Found
 
 
 class FunctionInputsValidationError(FunctionBaseError):
     msg_template: str = "Function inputs validation failed: {error}"
+    status_code: int = 422  # Unprocessable Entity
 
 
 class FunctionReadAccessDeniedError(FunctionBaseError):
     msg_template: str = "Function {function_id} read access denied for user {user_id}"
+    status_code: int = 403  # Forbidden
 
 
 class FunctionJobCollectionIDNotFoundError(FunctionBaseError):
     msg_template: str = "Function job collection {function_job_collection_id} not found"
+    status_code: int = 404  # Not Found
 
 
 class UnsupportedFunctionClassError(FunctionBaseError):
     msg_template: str = "Function class {function_class} is not supported"
+    status_code: int = 400  # Bad Request
 
 
 class UnsupportedFunctionJobClassError(FunctionBaseError):
     msg_template: str = "Function job class {function_job_class} is not supported"
+    status_code: int = 400  # Bad Request
 
 
 class UnsupportedFunctionFunctionJobClassCombinationError(FunctionBaseError):
     msg_template: str = (
         "Function class {function_class} and function job class {function_job_class} combination is not supported"
     )
+    status_code: int = 400  # Bad Request
 
 
 class FunctionJobCollectionReadAccessDeniedError(FunctionBaseError):
     msg_template: str = (
         "Function job collection {function_job_collection_id} read access denied for user {user_id}"
     )
+    status_code: int = 403  # Forbidden
 
 
 class FunctionWriteAccessDeniedError(FunctionBaseError):
     msg_template: str = "Function {function_id} write access denied for user {user_id}"
+    status_code: int = 403  # Forbidden
 
 
 class FunctionJobWriteAccessDeniedError(FunctionBaseError):
     msg_template: str = (
         "Function job {function_job_id} write access denied for user {user_id}"
     )
+    status_code: int = 403  # Forbidden
 
 
 class FunctionJobCollectionWriteAccessDeniedError(FunctionBaseError):
     msg_template: str = (
         "Function job collection {function_job_collection_id} write access denied for user {user_id}"
     )
+    status_code: int = 403  # Forbidden
 
 
 class FunctionExecuteAccessDeniedError(FunctionBaseError):
     msg_template: str = (
         "Function {function_id} execute access denied for user {user_id}"
     )
+    status_code: int = 403  # Forbidden
 
 
 class FunctionJobExecuteAccessDeniedError(FunctionBaseError):
     msg_template: str = (
         "Function job {function_job_id} execute access denied for user {user_id}"
     )
+    status_code: int = 403  # Forbidden
 
 
 class FunctionJobCollectionExecuteAccessDeniedError(FunctionBaseError):
     msg_template: str = (
         "Function job collection {function_job_collection_id} execute access denied for user {user_id}"
     )
+    status_code: int = 403  # Forbidden

packages/simcore-sdk/requirements/_base.in

@@ -13,9 +13,10 @@
 aiocache
 aiofiles
 aiohttp
+httpx
 packaging
 pint
-sqlalchemy[asyncio]
 pydantic[email]
+sqlalchemy[asyncio]
 tenacity
 tqdm

packages/simcore-sdk/requirements/_base.txt

@@ -44,6 +44,7 @@ anyio==4.8.0
     # via
     #   fast-depends
     #   faststream
+    #   httpx
 arrow==1.3.0
     # via
     #   -r requirements/../../../packages/models-library/requirements/_base.in
@@ -72,6 +73,8 @@ certifi==2025.1.31
     #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
+    #   httpcore
+    #   httpx
     #   requests
 charset-normalizer==3.4.1
     # via requests
@@ -109,10 +112,32 @@ greenlet==3.1.1
     # via sqlalchemy
 grpcio==1.70.0
     # via opentelemetry-exporter-otlp-proto-grpc
+h11==0.16.0
+    # via httpcore
+httpcore==1.0.9
+    # via httpx
+httpx==0.28.1
+    # via
+    #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/postgres-database/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -r requirements/_base.in
 idna==3.10
     # via
     #   anyio
     #   email-validator
+    #   httpx
     #   requests
     #   yarl
 importlib-metadata==8.5.0

packages/simcore-sdk/src/simcore_sdk/node_ports_common/file_io_utils.py

@@ -7,10 +7,10 @@
 from typing import IO, Any, Final, Protocol, runtime_checkable
 
 import aiofiles
+import httpx
 from aiohttp import (
     ClientConnectionError,
     ClientError,
-    ClientPayloadError,
     ClientResponse,
     ClientResponseError,
     ClientSession,
@@ -39,6 +39,7 @@
 from tqdm.contrib.logging import tqdm_logging_redirect
 from yarl import URL
 
+from ..config.http_clients import client_request_settings
 from . import exceptions
 from .constants import CHUNK_SIZE
 
@@ -148,13 +149,13 @@ async def __call__(self, log: str) -> None: ...
 
 async def _file_chunk_writer(
     file: Path,
-    response: ClientResponse,
+    response: httpx.Response,
     pbar: tqdm,
     io_log_redirect_cb: LogRedirectCB | None,
     progress_bar: ProgressBarData,
 ):
     async with aiofiles.open(file, "wb") as file_pointer:
-        while chunk := await response.content.read(CHUNK_SIZE):
+        async for chunk in response.aiter_bytes(CHUNK_SIZE):
             await file_pointer.write(chunk)
             if io_log_redirect_cb and pbar.update(len(chunk)):
                 with log_catch(_logger, reraise=False):
@@ -172,7 +173,6 @@ async def _file_chunk_writer(
 
 
 async def download_link_to_file(
-    session: ClientSession,
     url: URL,
     file_path: Path,
     *,
@@ -185,16 +185,25 @@ async def download_link_to_file(
         reraise=True,
         wait=wait_exponential(min=1, max=10),
         stop=stop_after_attempt(num_retries),
-        retry=retry_if_exception_type(ClientConnectionError),
+        retry=retry_if_exception_type(httpx.TransportError),
         before_sleep=before_sleep_log(_logger, logging.WARNING, exc_info=True),
         after=after_log(_logger, log_level=logging.ERROR),
     ):
         with attempt:
             async with AsyncExitStack() as stack:
-                response = await stack.enter_async_context(session.get(url))
-                if response.status == status.HTTP_404_NOT_FOUND:
+                client = await stack.enter_async_context(
+                    httpx.AsyncClient(
+                        timeout=httpx.Timeout(
+                            client_request_settings.HTTP_CLIENT_REQUEST_TOTAL_TIMEOUT
+                        )
+                    )
+                )
+                response = await stack.enter_async_context(
+                    client.stream("GET", f"{url}")
+                )
+                if response.status_code == status.HTTP_404_NOT_FOUND:
                     raise exceptions.InvalidDownloadLinkError(url)
-                if response.status > _VALID_HTTP_STATUS_CODES:
+                if response.status_code > _VALID_HTTP_STATUS_CODES:
                     raise exceptions.TransferError(url)
                 file_path.parent.mkdir(parents=True, exist_ok=True)
                 # SEE https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Length
@@ -231,7 +240,7 @@ async def download_link_to_file(
                         sub_progress,
                     )
                     _logger.debug("Download complete")
-                except ClientPayloadError as exc:
+                except httpx.HTTPError as exc:
                     raise exceptions.TransferError(url) from exc
 
 

packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py

@@ -217,7 +217,6 @@ async def download_path_from_s3(
         return await download_file_from_link(
             download_link,
             local_path,
-            client_session=session,
             io_log_redirect_cb=io_log_redirect_cb,
             progress_bar=progress_bar,
         )
@@ -229,7 +228,6 @@ async def download_file_from_link(
     *,
     io_log_redirect_cb: LogRedirectCB | None,
     file_name: str | None = None,
-    client_session: ClientSession | None = None,
     progress_bar: ProgressBarData,
 ) -> Path:
     # a download link looks something like:
@@ -242,15 +240,14 @@ async def download_file_from_link(
 
     if io_log_redirect_cb:
         await io_log_redirect_cb(f"downloading {local_file_path}, please wait...")
-    async with ClientSessionContextManager(client_session) as session:
-        await download_link_to_file(
-            session,
-            download_link,
-            local_file_path,
-            num_retries=NodePortsSettings.create_from_envs().NODE_PORTS_IO_NUM_RETRY_ATTEMPTS,
-            io_log_redirect_cb=io_log_redirect_cb,
-            progress_bar=progress_bar,
-        )
+
+    await download_link_to_file(
+        download_link,
+        local_file_path,
+        num_retries=NodePortsSettings.create_from_envs().NODE_PORTS_IO_NUM_RETRY_ATTEMPTS,
+        io_log_redirect_cb=io_log_redirect_cb,
+        progress_bar=progress_bar,
+    )
     if io_log_redirect_cb:
         await io_log_redirect_cb(f"download of {local_file_path} complete.")
     return local_file_path

services/api-server/src/simcore_service_api_server/exceptions/handlers/__init__.py

@@ -1,6 +1,7 @@
 from fastapi import FastAPI
 from fastapi.exceptions import RequestValidationError
 from httpx import HTTPError as HttpxException
+from models_library.functions_errors import FunctionBaseError
 from starlette import status
 from starlette.exceptions import HTTPException
 
@@ -9,6 +10,7 @@
 from ..custom_errors import CustomBaseError
 from ..log_streaming_errors import LogStreamingBaseError
 from ._custom_errors import custom_error_handler
+from ._handler_function_errors import function_error_handler
 from ._handlers_backend_errors import backend_error_handler
 from ._handlers_factory import make_handler_for_exception
 from ._http_exceptions import http_exception_handler
@@ -24,6 +26,7 @@ def setup(app: FastAPI, *, is_debug: bool = False):
     app.add_exception_handler(LogStreamingBaseError, log_handling_error_handler)
     app.add_exception_handler(CustomBaseError, custom_error_handler)
     app.add_exception_handler(BaseBackEndError, backend_error_handler)
+    app.add_exception_handler(FunctionBaseError, function_error_handler)
 
     # SEE https://docs.python.org/3/library/exceptions.html#exception-hierarchy
     app.add_exception_handler(

services/api-server/src/simcore_service_api_server/exceptions/handlers/_handler_function_errors.py

@@ -0,0 +1,11 @@
+from fastapi import Request
+from models_library.functions_errors import FunctionBaseError
+
+from ._utils import create_error_json_response
+
+
+async def function_error_handler(request: Request, exc: Exception):
+    assert request  # nosec
+    assert isinstance(exc, FunctionBaseError)
+
+    return create_error_json_response(f"{exc}", status_code=exc.status_code)

services/api-server/tests/unit/api_functions/test_api_routers_functions.py

@@ -7,7 +7,6 @@
 from uuid import uuid4
 
 import httpx
-import pytest
 from httpx import AsyncClient
 from models_library.api_schemas_webserver.functions import (
     FunctionJobCollection,
@@ -17,7 +16,10 @@
     RegisteredProjectFunction,
     RegisteredProjectFunctionJob,
 )
-from models_library.functions_errors import FunctionIDNotFoundError
+from models_library.functions_errors import (
+    FunctionIDNotFoundError,
+    FunctionReadAccessDeniedError,
+)
 from models_library.rest_pagination import PageMetaInfoLimitOffset
 from servicelib.aiohttp import status
 from simcore_service_api_server._meta import API_VTAG
@@ -92,8 +94,35 @@ async def test_get_function_not_found(
         None,
         FunctionIDNotFoundError(function_id=non_existent_function_id),
     )
-    with pytest.raises(FunctionIDNotFoundError):
-        await client.get(f"{API_VTAG}/functions/{non_existent_function_id}", auth=auth)
+    response = await client.get(
+        f"{API_VTAG}/functions/{non_existent_function_id}", auth=auth
+    )
+    assert response.status_code == status.HTTP_404_NOT_FOUND
+
+
+async def test_get_function_read_access_denied(
+    client: AsyncClient,
+    mock_handler_in_functions_rpc_interface: Callable[
+        [str, Any, Exception | None], None
+    ],
+    mock_registered_function: RegisteredProjectFunction,
+    auth: httpx.BasicAuth,
+) -> None:
+    unauthorized_user_id = "unauthorized user"
+    mock_handler_in_functions_rpc_interface(
+        "get_function",
+        None,
+        FunctionReadAccessDeniedError(
+            function_id=mock_registered_function.uid, user_id=unauthorized_user_id
+        ),
+    )
+    response = await client.get(
+        f"{API_VTAG}/functions/{mock_registered_function.uid}", auth=auth
+    )
+    assert response.status_code == status.HTTP_403_FORBIDDEN
+    assert response.json()["errors"][0] == (
+        f"Function {mock_registered_function.uid} read access denied for user {unauthorized_user_id}"
+    )
 
 
 async def test_list_functions(