Skip to content

Commit 955e0b4

Browse files
sandereggsanderegg
committed
Revert "šŸ›lock botocore<1.36.0 until CEPH S3 is updated to be compatible with AWS S3 (#7587)"
This reverts commit 1b6b386.
1 parent bea53b7 commit 955e0b4

File tree

9 files changed

+20
-54
lines changed

9 files changed

+20
-54
lines changed

ā€Žpackages/aws-library/src/aws_library/s3/__init__.pyā€Ž

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@
2222
)
2323

2424
__all__: tuple[str, ...] = (
25-
"PRESIGNED_LINK_MAX_SIZE",
26-
"S3_MAX_FILE_SIZE",
2725
"CopiedBytesTransferredCallback",
2826
"MultiPartUploadLinks",
27+
"PRESIGNED_LINK_MAX_SIZE",
28+
"S3_MAX_FILE_SIZE",
2929
"S3AccessError",
3030
"S3BucketInvalidError",
3131
"S3DestinationNotEmptyError",
@@ -37,8 +37,8 @@
3737
"S3RuntimeError",
3838
"S3UploadNotFoundError",
3939
"SimcoreS3API",
40-
"UploadID",
4140
"UploadedBytesTransferredCallback",
41+
"UploadID",
4242
)
4343

4444
# nopycln: file

ā€Žpackages/aws-library/src/aws_library/s3/_client.pyā€Ž

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
import functools
44
import logging
55
import urllib.parse
6-
import warnings
76
from collections.abc import AsyncGenerator, Sequence
87
from dataclasses import dataclass, field
98
from pathlib import Path
@@ -12,7 +11,6 @@
1211
import aioboto3
1312
from aiobotocore.session import ClientCreatorContext
1413
from boto3.s3.transfer import TransferConfig
15-
from botocore import __version__ as botocore_version
1614
from botocore import exceptions as botocore_exc
1715
from botocore.client import Config
1816
from models_library.api_schemas_storage.storage_schemas import (
@@ -22,7 +20,6 @@
2220
)
2321
from models_library.basic_types import SHA256Str
2422
from models_library.bytes_iters import BytesIter, DataSize
25-
from packaging import version
2623
from pydantic import AnyUrl, ByteSize, TypeAdapter
2724
from servicelib.bytes_iters import DEFAULT_READ_CHUNK_SIZE, BytesStreamer
2825
from servicelib.logging_utils import log_catch, log_context
@@ -54,22 +51,6 @@
5451
)
5552
from ._utils import compute_num_file_chunks, create_final_prefix
5653

57-
_BOTOCORE_VERSION: Final[version.Version] = version.parse(botocore_version)
58-
_MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3: Final[version.Version] = version.parse(
59-
"1.36.0"
60-
)
61-
62-
63-
def _check_botocore_version() -> None:
64-
if _BOTOCORE_VERSION >= _MAX_BOTOCORE_VERSION_COMPATIBLE_WITH_CEPH_S3:
65-
warnings.warn(
66-
f"Botocore version {botocore_version} is not supported for file uploads with CEPH S3 until CEPH is updated. "
67-
"Please use a version < 1.36.0. The upload operation will likely fail.",
68-
RuntimeWarning,
69-
stacklevel=2,
70-
)
71-
72-
7354
_logger = logging.getLogger(__name__)
7455

7556
_S3_MAX_CONCURRENCY_DEFAULT: Final[int] = 10
@@ -526,9 +507,6 @@ async def upload_file(
526507
bytes_transfered_cb: UploadedBytesTransferredCallback | None,
527508
) -> None:
528509
"""upload a file using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)"""
529-
530-
_check_botocore_version()
531-
532510
upload_options: dict[str, Any] = {
533511
"Bucket": bucket,
534512
"Key": object_key,
@@ -553,9 +531,6 @@ async def copy_object(
553531
object_metadata: S3MetaData | None = None,
554532
) -> None:
555533
"""copy a file in S3 using aioboto3 transfer manager (e.g. works >5Gb and creates multiple threads)"""
556-
557-
_check_botocore_version()
558-
559534
copy_options: dict[str, Any] = {
560535
"CopySource": {"Bucket": bucket, "Key": src_object_key},
561536
"Bucket": bucket,
@@ -662,7 +637,6 @@ async def upload_object_from_file_like(
662637
file_like_reader: FileLikeReader,
663638
) -> None:
664639
"""streams write an object in S3 from an AsyncIterable[bytes]"""
665-
_check_botocore_version()
666640
await self._client.upload_fileobj(file_like_reader, bucket_name, object_key) # type: ignore[arg-type]
667641

668642
@staticmethod

ā€Žservices/dask-sidecar/requirements/_base.txtā€Ž

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
aio-pika==9.5.3
22
# via -r requirements/../../../packages/service-library/requirements/_base.in
3-
aiobotocore==2.17.0
3+
aiobotocore==2.21.1
44
# via s3fs
55
aiocache==0.12.3
66
# via -r requirements/../../../packages/service-library/requirements/_base.in
@@ -67,10 +67,8 @@ blosc==1.11.2
6767
# via -r requirements/_base.in
6868
bokeh==3.6.2
6969
# via dask
70-
botocore==1.35.93
71-
# via
72-
# -c requirements/constraints.txt
73-
# aiobotocore
70+
botocore==1.37.1
71+
# via aiobotocore
7472
certifi==2024.8.30
7573
# via
7674
# -c requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
@@ -569,7 +567,6 @@ urllib3==2.2.3
569567
# -c requirements/../../../packages/settings-library/requirements/../../../packages/common-library/requirements/../../../requirements/constraints.txt
570568
# -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
571569
# -c requirements/../../../requirements/constraints.txt
572-
# aiobotocore
573570
# botocore
574571
# distributed
575572
# requests

ā€Žservices/dask-sidecar/requirements/_test.txtā€Ž

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,11 @@ aws-xray-sdk==2.14.0
1515
# via moto
1616
blinker==1.9.0
1717
# via flask
18-
boto3==1.35.93
18+
boto3==1.37.1
1919
# via
2020
# aws-sam-translator
2121
# moto
22-
botocore==1.35.93
22+
botocore==1.37.1
2323
# via
2424
# -c requirements/_base.txt
2525
# aws-xray-sdk
@@ -226,7 +226,7 @@ rpds-py==0.22.3
226226
# -c requirements/_base.txt
227227
# jsonschema
228228
# referencing
229-
s3transfer==0.10.4
229+
s3transfer==0.11.3
230230
# via boto3
231231
setuptools==75.8.2
232232
# via moto

ā€Žservices/dask-sidecar/requirements/constraints.txtā€Ž

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,3 @@ dask[distributed]>=2024.4.2 # issue with publish_dataset: https://github.com/das
1313
#
1414
# Compatibility/coordination
1515
#
16-
# botocore does not always add the checksums to the s3 object metadata leading to issues with CEPH S3
17-
# see https://github.com/ITISFoundation/osparc-simcore/issues/7585
18-
botocore<1.36.0

ā€Žservices/storage/requirements/_base.inā€Ž

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
#
44

55
--constraint ../../../requirements/constraints.txt
6-
--constraint ./constraints.txt
76

87

98
--requirement ../../../packages/aws-library/requirements/_base.in

ā€Žservices/storage/requirements/_base.txtā€Ž

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ aio-pika==9.5.4
22
# via
33
# -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in
44
# -r requirements/../../../packages/service-library/requirements/_base.in
5-
aioboto3==13.3.0
5+
aioboto3==14.1.0
66
# via
77
# -r requirements/../../../packages/aws-library/requirements/_base.in
88
# -r requirements/_base.in
9-
aiobotocore==2.16.0
9+
aiobotocore==2.21.1
1010
# via aioboto3
1111
aiocache==0.12.3
1212
# via
@@ -102,11 +102,10 @@ attrs==25.1.0
102102
# referencing
103103
billiard==4.2.1
104104
# via celery
105-
boto3==1.35.81
105+
boto3==1.37.1
106106
# via aiobotocore
107-
botocore==1.35.81
107+
botocore==1.37.1
108108
# via
109-
# -c requirements/./constraints.txt
110109
# aiobotocore
111110
# boto3
112111
# s3transfer
@@ -286,6 +285,7 @@ jinja2==3.1.5
286285
# fastapi
287286
jmespath==1.0.1
288287
# via
288+
# aiobotocore
289289
# boto3
290290
# botocore
291291
jsonschema==4.23.0
@@ -337,6 +337,7 @@ mdurl==0.1.2
337337
# via markdown-it-py
338338
multidict==6.1.0
339339
# via
340+
# aiobotocore
340341
# aiohttp
341342
# yarl
342343
opentelemetry-api==1.30.0
@@ -636,6 +637,7 @@ pyinstrument==5.0.1
636637
# -r requirements/../../../packages/service-library/requirements/_base.in
637638
python-dateutil==2.9.0.post0
638639
# via
640+
# aiobotocore
639641
# arrow
640642
# botocore
641643
# celery
@@ -754,7 +756,7 @@ rpds-py==0.22.3
754756
# via
755757
# jsonschema
756758
# referencing
757-
s3transfer==0.10.4
759+
s3transfer==0.11.3
758760
# via boto3
759761
sh==2.2.1
760762
# via -r requirements/../../../packages/aws-library/requirements/_base.in

ā€Žservices/storage/requirements/_test.txtā€Ž

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,12 +47,12 @@ billiard==4.2.1
4747
# celery
4848
blinker==1.9.0
4949
# via flask
50-
boto3==1.35.81
50+
boto3==1.37.1
5151
# via
5252
# -c requirements/_base.txt
5353
# aws-sam-translator
5454
# moto
55-
botocore==1.35.81
55+
botocore==1.37.1
5656
# via
5757
# -c requirements/_base.txt
5858
# aws-xray-sdk
@@ -362,7 +362,7 @@ rpds-py==0.22.3
362362
# -c requirements/_base.txt
363363
# jsonschema
364364
# referencing
365-
s3transfer==0.10.4
365+
s3transfer==0.11.3
366366
# via
367367
# -c requirements/_base.txt
368368
# boto3

ā€Žservices/storage/requirements/constraints.txtā€Ž

Lines changed: 0 additions & 3 deletions
This file was deleted.

0 commit comments

Comments
Ā (0)