Merge branch 'master' into is7581/wait-for-asyncio-tasks-when-aborting

Author: giancarloromeo

packages/aws-library/src/aws_library/s3/__init__.py

@@ -22,10 +22,10 @@
 )
 
 __all__: tuple[str, ...] = (
-    "CopiedBytesTransferredCallback",
-    "MultiPartUploadLinks",
     "PRESIGNED_LINK_MAX_SIZE",
     "S3_MAX_FILE_SIZE",
+    "CopiedBytesTransferredCallback",
+    "MultiPartUploadLinks",
     "S3AccessError",
     "S3BucketInvalidError",
     "S3DestinationNotEmptyError",
@@ -37,8 +37,8 @@
     "S3RuntimeError",
     "S3UploadNotFoundError",
     "SimcoreS3API",
-    "UploadedBytesTransferredCallback",
     "UploadID",
+    "UploadedBytesTransferredCallback",
 )
 
 # nopycln: file

packages/aws-library/src/aws_library/s3/_client.py

@@ -3,6 +3,7 @@
 import functools
 import logging
 import urllib.parse
+import warnings
 from collections.abc import AsyncGenerator, Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -11,6 +12,7 @@
 import aioboto3
 from aiobotocore.session import ClientCreatorContext
 from boto3.s3.transfer import TransferConfig
+from botocore import __version__ as botocore_version
 from botocore import exceptions as botocore_exc
 from botocore.client import Config
 from models_library.api_schemas_storage.storage_schemas import (
@@ -20,6 +22,7 @@
 )
 from models_library.basic_types import SHA256Str
 from models_library.bytes_iters import BytesIter, DataSize
+from packaging import version
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.bytes_iters import DEFAULT_READ_CHUNK_SIZE, BytesStreamer
 from servicelib.logging_utils import log_catch, log_context
@@ -51,6 +54,22 @@
 )
 from ._utils import compute_num_file_chunks, create_final_prefix
 
+_BOTOCORE_VERSION: Final[version.Version] = version.parse(botocore_version)
+_MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3: Final[version.Version] = version.parse(
+    "1.36.0"
+)
+
+
+def _check_botocore_version() -> None:
+    if _BOTOCORE_VERSION >= _MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3:
+        warnings.warn(
+            f"Botocore version {botocore_version} is not supported for file uploads with CEPH S3 until CEPH is updated. "
+            "Please use a version < 1.36.0. The upload operation will likely fail.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
+
 _logger = logging.getLogger(__name__)
 
 _S3_MAX_CONCURRENCY_DEFAULT: Final[int] = 10
@@ -504,6 +523,9 @@ async def upload_file(
         bytes_transfered_cb: UploadedBytesTransferredCallback | None,
     ) -> None:
         """upload a file using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)"""
+
+        _check_botocore_version()
+
         upload_options: dict[str, Any] = {
             "Bucket": bucket,
             "Key": object_key,
@@ -528,6 +550,9 @@ async def copy_object(
         object_metadata: S3MetaData | None = None,
     ) -> None:
         """copy a file in S3 using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)"""
+
+        _check_botocore_version()
+
         copy_options: dict[str, Any] = {
             "CopySource": {"Bucket": bucket, "Key": src_object_key},
             "Bucket": bucket,
@@ -634,6 +659,7 @@ async def upload_object_from_file_like(
         file_like_reader: FileLikeReader,
     ) -> None:
         """streams write an object in S3 from an AsyncIterable[bytes]"""
+        _check_botocore_version()
         await self._client.upload_fileobj(file_like_reader, bucket_name, object_key)  # type: ignore[arg-type]
 
     @staticmethod

packages/models-library/src/models_library/projects.py

@@ -113,8 +113,8 @@ class ProjectAtDB(BaseProjectModel):
 
     published: Annotated[
         bool | None,
-        Field(default=False, description="Defines if a study is available publicly"),
-    ]
+        Field(description="Defines if a study is available publicly"),
+    ] = False
 
     @field_validator("project_type", mode="before")
     @classmethod

packages/models-library/src/models_library/projects_nodes.py

@@ -1,5 +1,5 @@
 """
-    Models Node as a central element in a project's pipeline
+Models Node as a central element in a project's pipeline
 """
 
 from typing import Annotated, Any, TypeAlias, Union
@@ -17,6 +17,7 @@
     StringConstraints,
     field_validator,
 )
+from pydantic.config import JsonDict
 
 from .basic_types import EnvVarKey, KeyIDStr
 from .projects_access import AccessEnum
@@ -71,26 +72,41 @@
 
 
 class NodeState(BaseModel):
-    modified: bool = Field(
-        default=True, description="true if the node's outputs need to be re-computed"
-    )
-    dependencies: set[NodeID] = Field(
-        default_factory=set,
-        description="contains the node inputs dependencies if they need to be computed first",
-    )
-    current_status: RunningState = Field(
-        default=RunningState.NOT_STARTED,
-        description="the node's current state",
-        alias="currentStatus",
-    )
-    progress: float | None = Field(
-        default=0,
-        ge=0.0,
-        le=1.0,
-        description="current progress of the task if available (None if not started or not a computational task)",
-    )
+    modified: Annotated[
+        bool,
+        Field(
+            description="true if the node's outputs need to be re-computed",
+        ),
+    ] = True
+
+    dependencies: Annotated[
+        set[NodeID],
+        Field(
+            default_factory=set,
+            description="contains the node inputs dependencies if they need to be computed first",
+        ),
+    ] = DEFAULT_FACTORY
+
+    current_status: Annotated[
+        RunningState,
+        Field(
+            description="the node's current state",
+            alias="currentStatus",
+        ),
+    ] = RunningState.NOT_STARTED
+
+    progress: Annotated[
+        float | None,
+        Field(
+            ge=0.0,
+            le=1.0,
+            description="current progress of the task if available (None if not started or not a computational task)",
+        ),
+    ] = 0
+
     model_config = ConfigDict(
         extra="forbid",
+        populate_by_name=True,
         json_schema_extra={
             "examples": [
                 {
@@ -113,24 +129,35 @@ class NodeState(BaseModel):
     )
 
 
+def _convert_old_enum_name(v) -> RunningState:
+    if v == "FAILURE":
+        return RunningState.FAILED
+    return RunningState(v)
+
+
 class Node(BaseModel):
-    key: ServiceKey = Field(
-        ...,
-        description="distinctive name for the node based on the docker registry path",
-        examples=[
-            "simcore/services/comp/itis/sleeper",
-            "simcore/services/dynamic/3dviewer",
-            "simcore/services/frontend/file-picker",
-        ],
-    )
-    version: ServiceVersion = Field(
-        ...,
-        description="semantic version number of the node",
-        examples=["1.0.0", "0.0.1"],
-    )
-    label: str = Field(
-        ..., description="The short name of the node", examples=["JupyterLab"]
-    )
+    key: Annotated[
+        ServiceKey,
+        Field(
+            description="distinctive name for the node based on the docker registry path",
+            examples=[
+                "simcore/services/comp/itis/sleeper",
+                "simcore/services/dynamic/3dviewer",
+                "simcore/services/frontend/file-picker",
+            ],
+        ),
+    ]
+    version: Annotated[
+        ServiceVersion,
+        Field(
+            description="semantic version number of the node",
+            examples=["1.0.0", "0.0.1"],
+        ),
+    ]
+    label: Annotated[
+        str,
+        Field(description="The short name of the node", examples=["JupyterLab"]),
+    ]
     progress: Annotated[
         float | None,
         Field(
@@ -204,9 +231,9 @@ class Node(BaseModel):
         Field(default_factory=dict, description="values of output properties"),
     ] = DEFAULT_FACTORY
 
-    output_node: Annotated[
-        bool | None, Field(deprecated=True, alias="outputNode")
-    ] = None
+    output_node: Annotated[bool | None, Field(deprecated=True, alias="outputNode")] = (
+        None
+    )
 
     output_nodes: Annotated[
         list[NodeID] | None,
@@ -255,24 +282,109 @@ def _convert_empty_str_to_none(cls, v):
             return None
         return v
 
-    @classmethod
-    def _convert_old_enum_name(cls, v) -> RunningState:
-        if v == "FAILURE":
-            return RunningState.FAILED
-        return RunningState(v)
-
     @field_validator("state", mode="before")
     @classmethod
     def _convert_from_enum(cls, v):
         if isinstance(v, str):
+
             # the old version of state was a enum of RunningState
-            running_state_value = cls._convert_old_enum_name(v)
-            return NodeState(currentStatus=running_state_value)
+            running_state_value = _convert_old_enum_name(v)
+            return NodeState(current_status=running_state_value)
         return v
 
+    @staticmethod
+    def _update_json_schema_extra(schema: JsonDict) -> None:
+        schema.update(
+            {
+                "examples": [
+                    # Minimal example with only required fields
+                    {
+                        "key": "simcore/services/comp/no_ports",
+                        "version": "1.0.0",
+                        "label": "Sleep",
+                    },
+                    # Complete example with optional fields
+                    {
+                        "key": "simcore/services/comp/only_inputs",
+                        "version": "1.0.0",
+                        "label": "Only INputs",
+                        "inputs": {
+                            "input_1": 1,
+                            "input_2": 2,
+                            "input_3": 3,
+                        },
+                    },
+                    # Complete example with optional fields
+                    {
+                        "key": "simcore/services/comp/only_outputs",
+                        "version": "1.0.0",
+                        "label": "Only Outputs",
+                        "outputs": {
+                            "output_1": 1,
+                            "output_2": 2,
+                            "output_3": 3,
+                        },
+                    },
+                    # Example with all possible input and output types
+                    {
+                        "key": "simcore/services/comp/itis/all-types",
+                        "version": "1.0.0",
+                        "label": "All Types Demo",
+                        "inputs": {
+                            "boolean_input": True,
+                            "integer_input": 42,
+                            "float_input": 3.14159,
+                            "string_input": "text value",
+                            "json_input": {"key": "value", "nested": {"data": 123}},
+                            "port_link_input": {
+                                "nodeUuid": "f2700a54-adcf-45d4-9881-01ec30fd75a2",
+                                "output": "out_1",
+                            },
+                            "simcore_file_link": {
+                                "store": "simcore.s3",
+                                "path": "123e4567-e89b-12d3-a456-426614174000/test.csv",
+                            },
+                            "datcore_file_link": {
+                                "store": "datcore",
+                                "dataset": "N:dataset:123",
+                                "path": "path/to/file.txt",
+                            },
+                            "download_link": {
+                                "downloadLink": "https://example.com/downloadable/file.txt"
+                            },
+                            "array_input": [1, 2, 3, 4, 5],
+                            "object_input": {"name": "test", "value": 42},
+                        },
+                        "outputs": {
+                            "boolean_output": False,
+                            "integer_output": 100,
+                            "float_output": 2.71828,
+                            "string_output": "result text",
+                            "json_output": {"status": "success", "data": [1, 2, 3]},
+                            "simcore_file_output": {
+                                "store": "simcore.s3",
+                                "path": "987e6543-e21b-12d3-a456-426614174000/result.csv",
+                            },
+                            "datcore_file_output": {
+                                "store": "datcore",
+                                "dataset": "N:dataset:456",
+                                "path": "results/output.txt",
+                            },
+                            "download_link_output": {
+                                "downloadLink": "https://example.com/results/download.txt"
+                            },
+                            "array_output": ["a", "b", "c", "d"],
+                            "object_output": {"status": "complete", "count": 42},
+                        },
+                    },
+                ],
+            }
+        )
+
     model_config = ConfigDict(
         extra="forbid",
         populate_by_name=True,
+        json_schema_extra=_update_json_schema_extra,
     )