diff --git a/packages/aws-library/src/aws_library/s3/__init__.py b/packages/aws-library/src/aws_library/s3/__init__.py index ea8f6264d604..8a9a85f1279e 100644 --- a/packages/aws-library/src/aws_library/s3/__init__.py +++ b/packages/aws-library/src/aws_library/s3/__init__.py @@ -22,10 +22,10 @@ ) __all__: tuple[str, ...] = ( - "CopiedBytesTransferredCallback", - "MultiPartUploadLinks", "PRESIGNED_LINK_MAX_SIZE", "S3_MAX_FILE_SIZE", + "CopiedBytesTransferredCallback", + "MultiPartUploadLinks", "S3AccessError", "S3BucketInvalidError", "S3DestinationNotEmptyError", @@ -37,8 +37,8 @@ "S3RuntimeError", "S3UploadNotFoundError", "SimcoreS3API", - "UploadedBytesTransferredCallback", "UploadID", + "UploadedBytesTransferredCallback", ) # nopycln: file diff --git a/packages/aws-library/src/aws_library/s3/_client.py b/packages/aws-library/src/aws_library/s3/_client.py index 351291318e2d..9d308a9d0fb2 100644 --- a/packages/aws-library/src/aws_library/s3/_client.py +++ b/packages/aws-library/src/aws_library/s3/_client.py @@ -3,6 +3,7 @@ import functools import logging import urllib.parse +import warnings from collections.abc import AsyncGenerator, Sequence from dataclasses import dataclass, field from pathlib import Path @@ -11,6 +12,7 @@ import aioboto3 from aiobotocore.session import ClientCreatorContext from boto3.s3.transfer import TransferConfig +from botocore import __version__ as botocore_version from botocore import exceptions as botocore_exc from botocore.client import Config from models_library.api_schemas_storage.storage_schemas import ( @@ -20,6 +22,7 @@ ) from models_library.basic_types import SHA256Str from models_library.bytes_iters import BytesIter, DataSize +from packaging import version from pydantic import AnyUrl, ByteSize, TypeAdapter from servicelib.bytes_iters import DEFAULT_READ_CHUNK_SIZE, BytesStreamer from servicelib.logging_utils import log_catch, log_context @@ -51,6 +54,22 @@ ) from ._utils import compute_num_file_chunks, create_final_prefix +_BOTOCORE_VERSION: Final[version.Version] = version.parse(botocore_version) +_MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3: Final[version.Version] = version.parse( + "1.36.0" +) + + +def _check_botocore_version() -> None: + if _BOTOCORE_VERSION >= _MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3: + warnings.warn( + f"Botocore version {botocore_version} is not supported for file uploads with CEPH S3 until CEPH is updated. " + "Please use a version < 1.36.0. The upload operation will likely fail.", + RuntimeWarning, + stacklevel=2, + ) + + _logger = logging.getLogger(__name__) _S3_MAX_CONCURRENCY_DEFAULT: Final[int] = 10 @@ -504,6 +523,9 @@ async def upload_file( bytes_transfered_cb: UploadedBytesTransferredCallback | None, ) -> None: """upload a file using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)""" + + _check_botocore_version() + upload_options: dict[str, Any] = { "Bucket": bucket, "Key": object_key, @@ -528,6 +550,9 @@ async def copy_object( object_metadata: S3MetaData | None = None, ) -> None: """copy a file in S3 using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)""" + + _check_botocore_version() + copy_options: dict[str, Any] = { "CopySource": {"Bucket": bucket, "Key": src_object_key}, "Bucket": bucket, @@ -634,6 +659,7 @@ async def upload_object_from_file_like( file_like_reader: FileLikeReader, ) -> None: """streams write an object in S3 from an AsyncIterable[bytes]""" + _check_botocore_version() await self._client.upload_fileobj(file_like_reader, bucket_name, object_key) # type: ignore[arg-type] @staticmethod diff --git a/services/dask-sidecar/requirements/_base.txt b/services/dask-sidecar/requirements/_base.txt index f59d2014f251..96c7590e1df0 100644 --- a/services/dask-sidecar/requirements/_base.txt +++ b/services/dask-sidecar/requirements/_base.txt @@ -1,6 +1,6 @@ aio-pika==9.5.3 # via -r requirements/../../../packages/service-library/requirements/_base.in -aiobotocore==2.21.1 +aiobotocore==2.17.0 # via s3fs aiocache==0.12.3 # via -r requirements/../../../packages/service-library/requirements/_base.in @@ -67,8 +67,10 @@ blosc==1.11.2 # via -r requirements/_base.in bokeh==3.6.2 # via dask -botocore==1.37.1 - # via aiobotocore +botocore==1.35.93 + # via + # -c requirements/constraints.txt + # aiobotocore certifi==2024.8.30 # via # -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt @@ -567,6 +569,7 @@ urllib3==2.2.3 # -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt + # aiobotocore # botocore # distributed # requests diff --git a/services/dask-sidecar/requirements/_test.txt b/services/dask-sidecar/requirements/_test.txt index d72662cb7530..d9e95c7c297f 100644 --- a/services/dask-sidecar/requirements/_test.txt +++ b/services/dask-sidecar/requirements/_test.txt @@ -15,11 +15,11 @@ aws-xray-sdk==2.14.0 # via moto blinker==1.9.0 # via flask -boto3==1.37.1 +boto3==1.35.93 # via # aws-sam-translator # moto -botocore==1.37.1 +botocore==1.35.93 # via # -c requirements/_base.txt # aws-xray-sdk @@ -226,7 +226,7 @@ rpds-py==0.22.3 # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.11.3 +s3transfer==0.10.4 # via boto3 setuptools==75.8.2 # via moto diff --git a/services/dask-sidecar/requirements/constraints.txt b/services/dask-sidecar/requirements/constraints.txt index d8a10d684a0d..9bb961b3dbd2 100644 --- a/services/dask-sidecar/requirements/constraints.txt +++ b/services/dask-sidecar/requirements/constraints.txt @@ -13,3 +13,6 @@ dask[distributed]>=2024.4.2 # issue with publish_dataset: https://github.com/das # # Compatibility/coordination # +# botocore does not always add the checksums to the s3 object metadata leading to issues with CEPH S3 +# see https://github.com/ITISFoundation/osparc-simcore/issues/7585 +botocore<1.36.0 diff --git a/services/storage/requirements/_base.in b/services/storage/requirements/_base.in index cf0ccfdba899..99f71f50afff 100644 --- a/services/storage/requirements/_base.in +++ b/services/storage/requirements/_base.in @@ -3,6 +3,7 @@ # --constraint ../../../requirements/constraints.txt +--constraint ./constraints.txt --requirement ../../../packages/aws-library/requirements/_base.in diff --git a/services/storage/requirements/_base.txt b/services/storage/requirements/_base.txt index f5eea0f34d05..7ac5068587f9 100644 --- a/services/storage/requirements/_base.txt +++ b/services/storage/requirements/_base.txt @@ -2,11 +2,11 @@ aio-pika==9.5.4 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aioboto3==14.1.0 +aioboto3==13.3.0 # via # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/_base.in -aiobotocore==2.21.1 +aiobotocore==2.16.0 # via aioboto3 aiocache==0.12.3 # via @@ -102,10 +102,11 @@ attrs==25.1.0 # referencing billiard==4.2.1 # via celery -boto3==1.37.1 +boto3==1.35.81 # via aiobotocore -botocore==1.37.1 +botocore==1.35.81 # via + # -c requirements/./constraints.txt # aiobotocore # boto3 # s3transfer @@ -285,7 +286,6 @@ jinja2==3.1.5 # fastapi jmespath==1.0.1 # via - # aiobotocore # boto3 # botocore jsonschema==4.23.0 @@ -337,7 +337,6 @@ mdurl==0.1.2 # via markdown-it-py multidict==6.1.0 # via - # aiobotocore # aiohttp # yarl opentelemetry-api==1.30.0 @@ -637,7 +636,6 @@ pyinstrument==5.0.1 # -r requirements/../../../packages/service-library/requirements/_base.in python-dateutil==2.9.0.post0 # via - # aiobotocore # arrow # botocore # celery @@ -756,7 +754,7 @@ rpds-py==0.22.3 # via # jsonschema # referencing -s3transfer==0.11.3 +s3transfer==0.10.4 # via boto3 sh==2.2.1 # via -r requirements/../../../packages/aws-library/requirements/_base.in diff --git a/services/storage/requirements/_test.txt b/services/storage/requirements/_test.txt index 221cc0b62eec..6dfb56937e35 100644 --- a/services/storage/requirements/_test.txt +++ b/services/storage/requirements/_test.txt @@ -47,12 +47,12 @@ billiard==4.2.1 # celery blinker==1.9.0 # via flask -boto3==1.37.1 +boto3==1.35.81 # via # -c requirements/_base.txt # aws-sam-translator # moto -botocore==1.37.1 +botocore==1.35.81 # via # -c requirements/_base.txt # aws-xray-sdk @@ -362,7 +362,7 @@ rpds-py==0.22.3 # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.11.3 +s3transfer==0.10.4 # via # -c requirements/_base.txt # boto3 diff --git a/services/storage/requirements/constraints.txt b/services/storage/requirements/constraints.txt new file mode 100644 index 000000000000..d8bb7d732246 --- /dev/null +++ b/services/storage/requirements/constraints.txt @@ -0,0 +1,3 @@ +# botocore does not always add the checksums to the s3 object metadata leading to issues with CEPH S3 +# see https://github.com/ITISFoundation/osparc-simcore/issues/7585 +botocore<1.36.0