ITISFoundation · sanderegg · Apr 25, 2025 · Apr 25, 2025 · Apr 25, 2025 · Apr 25, 2025
diff --git a/packages/aws-library/src/aws_library/s3/__init__.py b/packages/aws-library/src/aws_library/s3/__init__.py
@@ -22,10 +22,10 @@
 )
 
 __all__: tuple[str, ...] = (
-    "CopiedBytesTransferredCallback",
-    "MultiPartUploadLinks",
     "PRESIGNED_LINK_MAX_SIZE",
     "S3_MAX_FILE_SIZE",
+    "CopiedBytesTransferredCallback",
+    "MultiPartUploadLinks",
     "S3AccessError",
     "S3BucketInvalidError",
     "S3DestinationNotEmptyError",
@@ -37,8 +37,8 @@
     "S3RuntimeError",
     "S3UploadNotFoundError",
     "SimcoreS3API",
-    "UploadedBytesTransferredCallback",
     "UploadID",
+    "UploadedBytesTransferredCallback",
 )
 
 # nopycln: file
diff --git a/packages/aws-library/src/aws_library/s3/_client.py b/packages/aws-library/src/aws_library/s3/_client.py
@@ -3,6 +3,7 @@
 import functools
 import logging
 import urllib.parse
+import warnings
 from collections.abc import AsyncGenerator, Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -11,6 +12,7 @@
 import aioboto3
 from aiobotocore.session import ClientCreatorContext
 from boto3.s3.transfer import TransferConfig
+from botocore import __version__ as botocore_version
 from botocore import exceptions as botocore_exc
 from botocore.client import Config
 from models_library.api_schemas_storage.storage_schemas import (
@@ -20,6 +22,7 @@
 )
 from models_library.basic_types import SHA256Str
 from models_library.bytes_iters import BytesIter, DataSize
+from packaging import version
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.bytes_iters import DEFAULT_READ_CHUNK_SIZE, BytesStreamer
 from servicelib.logging_utils import log_catch, log_context
@@ -51,6 +54,22 @@
 )
 from ._utils import compute_num_file_chunks, create_final_prefix
 
+_BOTOCORE_VERSION: Final[version.Version] = version.parse(botocore_version)
+_MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3: Final[version.Version] = version.parse(
+    "1.36.0"
+)
+
+
+def _check_botocore_version() -> None:
+    if _BOTOCORE_VERSION >= _MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3:
+        warnings.warn(
+            f"Botocore version {botocore_version} is not supported for file uploads with CEPH S3 until CEPH is updated. "
+            "Please use a version < 1.36.0. The upload operation will likely fail.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
+
 _logger = logging.getLogger(__name__)
 
 _S3_MAX_CONCURRENCY_DEFAULT: Final[int] = 10
@@ -504,6 +523,9 @@ async def upload_file(
         bytes_transfered_cb: UploadedBytesTransferredCallback | None,
     ) -> None:
         """upload a file using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)"""
+
+        _check_botocore_version()
+
         upload_options: dict[str, Any] = {
             "Bucket": bucket,
             "Key": object_key,
@@ -528,6 +550,9 @@ async def copy_object(
         object_metadata: S3MetaData | None = None,
     ) -> None:
         """copy a file in S3 using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)"""
+
+        _check_botocore_version()
+
         copy_options: dict[str, Any] = {
             "CopySource": {"Bucket": bucket, "Key": src_object_key},
             "Bucket": bucket,
@@ -634,6 +659,7 @@ async def upload_object_from_file_like(
         file_like_reader: FileLikeReader,
     ) -> None:
         """streams write an object in S3 from an AsyncIterable[bytes]"""
+        _check_botocore_version()
         await self._client.upload_fileobj(file_like_reader, bucket_name, object_key)  # type: ignore[arg-type]
 
     @staticmethod

diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt
@@ -1,6 +1,6 @@
 aio-pika==9.5.3
     # via -r requirements/../../../packages/service-library/requirements/_base.in
-aiobotocore==2.21.1
+aiobotocore==2.17.0
     # via s3fs
 aiocache==0.12.3
     # via -r requirements/../../../packages/service-library/requirements/_base.in
@@ -67,8 +67,10 @@ blosc==1.11.2
     # via -r requirements/_base.in
 bokeh==3.6.2
     # via dask
-botocore==1.37.1
-    # via aiobotocore
+botocore==1.35.93
+    # via
+    #   -c requirements/constraints.txt
+    #   aiobotocore
 certifi==2024.8.30
     # via
     #   -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -567,6 +569,7 @@ urllib3==2.2.3
     #   -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
+    #   aiobotocore
     #   botocore
     #   distributed
     #   requests

diff --git a/services/dask-sidecar/requirements/_test.txt b/services/dask-sidecar/requirements/_test.txt
@@ -15,11 +15,11 @@ aws-xray-sdk==2.14.0
     # via moto
 blinker==1.9.0
     # via flask
-boto3==1.37.1
+boto3==1.35.93
     # via
     #   aws-sam-translator
     #   moto
-botocore==1.37.1
+botocore==1.35.93
     # via
     #   -c requirements/_base.txt
     #   aws-xray-sdk
@@ -226,7 +226,7 @@ rpds-py==0.22.3
     #   -c requirements/_base.txt
     #   jsonschema
     #   referencing
-s3transfer==0.11.3
+s3transfer==0.10.4
     # via boto3
 setuptools==75.8.2
     # via moto

diff --git a/services/dask-sidecar/requirements/constraints.txt b/services/dask-sidecar/requirements/constraints.txt
@@ -13,3 +13,6 @@ dask[distributed]>=2024.4.2 # issue with publish_dataset: https://github.com/das
 #
 # Compatibility/coordination
 #
+# botocore does not always add the checksums to the s3 object metadata leading to issues with CEPH S3
+# see https://github.com/ITISFoundation/osparc-simcore/issues/7585
+botocore<1.36.0
@@ -3,6 +3,7 @@
 #
 
 --constraint ../../../requirements/constraints.txt
+--constraint ./constraints.txt
 
 
 --requirement ../../../packages/aws-library/requirements/_base.in

@@ -2,11 +2,11 @@ aio-pika==9.5.4
     # via
     #   -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
-aioboto3==14.1.0
+aioboto3==13.3.0
     # via
     #   -r requirements/../../../packages/aws-library/requirements/_base.in
     #   -r requirements/_base.in
-aiobotocore==2.21.1
+aiobotocore==2.16.0
     # via aioboto3
 aiocache==0.12.3
     # via
@@ -102,10 +102,11 @@ attrs==25.1.0
     #   referencing
 billiard==4.2.1
     # via celery
-boto3==1.37.1
+boto3==1.35.81
     # via aiobotocore
-botocore==1.37.1
+botocore==1.35.81
     # via
+    #   -c requirements/./constraints.txt
     #   aiobotocore
     #   boto3
     #   s3transfer
@@ -285,7 +286,6 @@ jinja2==3.1.5
     #   fastapi
 jmespath==1.0.1
     # via
-    #   aiobotocore
     #   boto3
     #   botocore
 jsonschema==4.23.0
@@ -337,7 +337,6 @@ mdurl==0.1.2
     # via markdown-it-py
 multidict==6.1.0
     # via
-    #   aiobotocore
     #   aiohttp
     #   yarl
 opentelemetry-api==1.30.0
@@ -637,7 +636,6 @@ pyinstrument==5.0.1
     #   -r requirements/../../../packages/service-library/requirements/_base.in
 python-dateutil==2.9.0.post0
     # via
-    #   aiobotocore
     #   arrow
     #   botocore
     #   celery
@@ -756,7 +754,7 @@ rpds-py==0.22.3
     # via
     #   jsonschema
     #   referencing
-s3transfer==0.11.3
+s3transfer==0.10.4
     # via boto3
 sh==2.2.1
     # via -r requirements/../../../packages/aws-library/requirements/_base.in

@@ -47,12 +47,12 @@ billiard==4.2.1
     #   celery
 blinker==1.9.0
     # via flask
-boto3==1.37.1
+boto3==1.35.81
     # via
     #   -c requirements/_base.txt
     #   aws-sam-translator
     #   moto
-botocore==1.37.1
+botocore==1.35.81
     # via
     #   -c requirements/_base.txt
     #   aws-xray-sdk
@@ -362,7 +362,7 @@ rpds-py==0.22.3
     #   -c requirements/_base.txt
     #   jsonschema
     #   referencing
-s3transfer==0.11.3
+s3transfer==0.10.4
     # via
     #   -c requirements/_base.txt
     #   boto3

@@ -0,0 +1,3 @@
+# botocore does not always add the checksums to the s3 object metadata leading to issues with CEPH S3
+# see https://github.com/ITISFoundation/osparc-simcore/issues/7585
+botocore<1.36.0
-Original file line number
+Diff line change
@@ Expand Up / @@ -3,6 +3,7 @@ @@
     #
     --constraint ../../../requirements/constraints.txt
+    --constraint ./constraints.txt
     --requirement ../../../packages/aws-library/requirements/_base.in
@@ Expand Down @@