added warning

Author: sanderegg

packages/aws-library/src/aws_library/s3/__init__.py

@@ -22,10 +22,10 @@
 )
 
 __all__: tuple[str, ...] = (
-    "CopiedBytesTransferredCallback",
-    "MultiPartUploadLinks",
     "PRESIGNED_LINK_MAX_SIZE",
     "S3_MAX_FILE_SIZE",
+    "CopiedBytesTransferredCallback",
+    "MultiPartUploadLinks",
     "S3AccessError",
     "S3BucketInvalidError",
     "S3DestinationNotEmptyError",
@@ -37,8 +37,8 @@
     "S3RuntimeError",
     "S3UploadNotFoundError",
     "SimcoreS3API",
-    "UploadedBytesTransferredCallback",
     "UploadID",
+    "UploadedBytesTransferredCallback",
 )
 
 # nopycln: file

packages/aws-library/src/aws_library/s3/_client.py

@@ -3,6 +3,7 @@
 import functools
 import logging
 import urllib.parse
+import warnings
 from collections.abc import AsyncGenerator, Sequence
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -11,6 +12,7 @@
 import aioboto3
 from aiobotocore.session import ClientCreatorContext
 from boto3.s3.transfer import TransferConfig
+from botocore import __version__ as botocore_version
 from botocore import exceptions as botocore_exc
 from botocore.client import Config
 from models_library.api_schemas_storage.storage_schemas import (
@@ -20,6 +22,7 @@
 )
 from models_library.basic_types import SHA256Str
 from models_library.bytes_iters import BytesIter, DataSize
+from packaging import version
 from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.bytes_iters import DEFAULT_READ_CHUNK_SIZE, BytesStreamer
 from servicelib.logging_utils import log_catch, log_context
@@ -51,6 +54,22 @@
 )
 from ._utils import compute_num_file_chunks, create_final_prefix
 
+_BOTOCORE_VERSION: Final[version.Version] = version.parse(botocore_version)
+_MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3: Final[version.Version] = version.parse(
+    "1.36.0"
+)
+
+
+def _check_botocore_version() -> None:
+    if _BOTOCORE_VERSION >= _MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3:
+        warnings.warn(
+            f"Botocore version {botocore_version} is not supported for file uploads with CEPH S3 until CEPH is updated. "
+            "Please use a version < 1.36.0. The upload operation will likely fail.",
+            RuntimeWarning,
+            stacklevel=2,
+        )
+
+
 _logger = logging.getLogger(__name__)
 
 _S3_MAX_CONCURRENCY_DEFAULT: Final[int] = 10
@@ -504,6 +523,9 @@ async def upload_file(
         bytes_transfered_cb: UploadedBytesTransferredCallback | None,
     ) -> None:
         """upload a file using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)"""
+
+        _check_botocore_version()
+
         upload_options: dict[str, Any] = {
             "Bucket": bucket,
             "Key": object_key,
@@ -528,6 +550,9 @@ async def copy_object(
         object_metadata: S3MetaData | None = None,
     ) -> None:
         """copy a file in S3 using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)"""
+
+        _check_botocore_version()
+
         copy_options: dict[str, Any] = {
             "CopySource": {"Bucket": bucket, "Key": src_object_key},
             "Bucket": bucket,
@@ -634,6 +659,7 @@ async def upload_object_from_file_like(
         file_like_reader: FileLikeReader,
     ) -> None:
         """streams write an object in S3 from an AsyncIterable[bytes]"""
+        _check_botocore_version()
         await self._client.upload_fileobj(file_like_reader, bucket_name, object_key)  # type: ignore[arg-type]
 
     @staticmethod