Skip to content

Commit c5aa314

Browse files
matusdrobuliak66matusdrobuliak66
authored
♻️ reusing the same S3 client (ITISFoundation#5289)
1 parent 3fa4101 commit c5aa314

File tree

21 files changed

+166
-193
lines changed

21 files changed

+166
-193
lines changed

‎packages/aws-library/src/aws_library/s3/client.py‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -4,27 +4,31 @@
44
from typing import cast
55

66
import aioboto3
7-
import botocore.exceptions
87
from aiobotocore.session import ClientCreatorContext
98
from botocore.client import Config
109
from models_library.api_schemas_storage import S3BucketName
1110
from pydantic import AnyUrl, parse_obj_as
1211
from settings_library.s3 import S3Settings
1312
from types_aiobotocore_s3 import S3Client
1413

15-
from .errors import S3RuntimeError
14+
from .errors import s3_exception_handler
1615

1716
_logger = logging.getLogger(__name__)
1817

18+
_S3_MAX_CONCURRENCY_DEFAULT = 10
19+
1920

2021
@dataclass(frozen=True)
2122
class SimcoreS3API:
2223
client: S3Client
2324
session: aioboto3.Session
2425
exit_stack: contextlib.AsyncExitStack
26+
transfer_max_concurrency: int
2527

2628
@classmethod
27-
async def create(cls, settings: S3Settings) -> "SimcoreS3API":
29+
async def create(
30+
cls, settings: S3Settings, s3_max_concurrency: int = _S3_MAX_CONCURRENCY_DEFAULT
31+
) -> "SimcoreS3API":
2832
session = aioboto3.Session()
2933
session_client = session.client(
3034
"s3",
@@ -37,10 +41,11 @@ async def create(cls, settings: S3Settings) -> "SimcoreS3API":
3741
)
3842
assert isinstance(session_client, ClientCreatorContext) # nosec
3943
exit_stack = contextlib.AsyncExitStack()
40-
s3_client = cast(
41-
S3Settings, await exit_stack.enter_async_context(session_client)
42-
)
43-
return cls(s3_client, session, exit_stack)
44+
s3_client = cast(S3Client, await exit_stack.enter_async_context(session_client))
45+
# NOTE: this triggers a botocore.exception.ClientError in case the connection is not made to the S3 backend
46+
await s3_client.list_buckets()
47+
48+
return cls(s3_client, session, exit_stack, s3_max_concurrency)
4449

4550
async def close(self) -> None:
4651
await self.exit_stack.aclose()
@@ -53,21 +58,19 @@ async def http_check_bucket_connected(self, bucket: S3BucketName) -> bool:
5358
except Exception: # pylint: disable=broad-except
5459
return False
5560

56-
async def create_presigned_download_link(
61+
@s3_exception_handler(_logger)
62+
async def create_single_presigned_download_link(
5763
self,
5864
bucket_name: S3BucketName,
5965
object_key: str,
6066
expiration_secs: int,
6167
) -> AnyUrl:
62-
try:
63-
# NOTE: ensure the bucket/object exists, this will raise if not
64-
await self.client.head_bucket(Bucket=bucket_name)
65-
generated_link = await self.client.generate_presigned_url(
66-
"get_object",
67-
Params={"Bucket": bucket_name, "Key": object_key},
68-
ExpiresIn=expiration_secs,
69-
)
70-
url: AnyUrl = parse_obj_as(AnyUrl, generated_link)
71-
return url
72-
except botocore.exceptions.ClientError as exc:
73-
raise S3RuntimeError from exc # pragma: no cover
68+
# NOTE: ensure the bucket/object exists, this will raise if not
69+
await self.client.head_bucket(Bucket=bucket_name)
70+
generated_link = await self.client.generate_presigned_url(
71+
"get_object",
72+
Params={"Bucket": bucket_name, "Key": object_key},
73+
ExpiresIn=expiration_secs,
74+
)
75+
url: AnyUrl = parse_obj_as(AnyUrl, generated_link)
76+
return url

‎packages/aws-library/src/aws_library/s3/errors.py‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
import functools
2+
import logging
3+
4+
from botocore import exceptions as botocore_exc
15
from pydantic.errors import PydanticErrorMixin
26

37

@@ -7,3 +11,56 @@ class S3RuntimeError(PydanticErrorMixin, RuntimeError):
711

812
class S3NotConnectedError(S3RuntimeError):
913
msg_template: str = "Cannot connect with s3 server"
14+
15+
16+
class S3AccessError(S3RuntimeError):
17+
code = "s3_access.error"
18+
msg_template: str = "Unexpected error while accessing S3 backend"
19+
20+
21+
class S3BucketInvalidError(S3AccessError):
22+
code = "s3_bucket.invalid_error"
23+
msg_template: str = "The bucket '{bucket}' is invalid"
24+
25+
26+
class S3KeyNotFoundError(S3AccessError):
27+
code = "s3_key.not_found_error"
28+
msg_template: str = "The file {key} in {bucket} was not found"
29+
30+
31+
def s3_exception_handler(log: logging.Logger):
32+
"""converts typical aiobotocore/boto exceptions to storage exceptions
33+
NOTE: this is a work in progress as more exceptions might arise in different
34+
use-cases
35+
"""
36+
37+
def decorator(func): # noqa: C901
38+
@functools.wraps(func)
39+
async def wrapper(self, *args, **kwargs):
40+
try:
41+
return await func(self, *args, **kwargs)
42+
except self.client.exceptions.NoSuchBucket as exc:
43+
raise S3BucketInvalidError(
44+
bucket=exc.response.get("Error", {}).get("BucketName", "undefined")
45+
) from exc
46+
except botocore_exc.ClientError as exc:
47+
status_code = int(exc.response.get("Error", {}).get("Code", -1))
48+
operation_name = exc.operation_name
49+
50+
match status_code, operation_name:
51+
case 404, "HeadObject":
52+
raise S3KeyNotFoundError(bucket=args[0], key=args[1]) from exc
53+
case (404, "HeadBucket") | (403, "HeadBucket"):
54+
raise S3BucketInvalidError(bucket=args[0]) from exc
55+
case _:
56+
raise S3AccessError from exc
57+
except botocore_exc.EndpointConnectionError as exc:
58+
raise S3AccessError from exc
59+
60+
except botocore_exc.BotoCoreError as exc:
61+
log.exception("Unexpected error in s3 client: ")
62+
raise S3AccessError from exc
63+
64+
return wrapper
65+
66+
return decorator

‎packages/aws-library/tests/test_s3_client.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ async def upload_file_to_bucket(
127127
async def test_create_single_presigned_download_link(
128128
simcore_s3_api: SimcoreS3API, upload_file_to_bucket: None, create_s3_bucket
129129
):
130-
download_url = await simcore_s3_api.create_presigned_download_link(
130+
download_url = await simcore_s3_api.create_single_presigned_download_link(
131131
create_s3_bucket, "test.csv", 50
132132
)
133133
assert isinstance(download_url, AnyUrl)

‎services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ async def export_service_runs(
168168
)
169169

170170
# Create presigned S3 link
171-
generated_url: AnyUrl = await s3_client.create_presigned_download_link(
171+
generated_url: AnyUrl = await s3_client.create_single_presigned_download_link(
172172
bucket_name=s3_bucket_name,
173173
object_key=s3_object_key,
174174
expiration_secs=_PRESIGNED_LINK_EXPIRATION_SEC,

‎services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ async def mocked_export(mocker: MockerFixture):
3636
@pytest.fixture
3737
async def mocked_presigned_link(mocker: MockerFixture):
3838
mock_presigned_link = mocker.patch(
39-
"simcore_service_resource_usage_tracker.services.resource_tracker_service_runs.SimcoreS3API.create_presigned_download_link",
39+
"simcore_service_resource_usage_tracker.services.resource_tracker_service_runs.SimcoreS3API.create_single_presigned_download_link",
4040
return_value=parse_obj_as(
4141
AnyUrl,
4242
"https://www.testing.com/",

‎services/storage/requirements/_base.in‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
--constraint ../../../requirements/constraints.txt
66

77

8+
--requirement ../../../packages/aws-library/requirements/_base.in
89
--requirement ../../../packages/models-library/requirements/_base.in
910
--requirement ../../../packages/postgres-database/requirements/_base.in
1011
--requirement ../../../packages/settings-library/requirements/_base.in

‎services/storage/requirements/_base.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,7 @@ typing-extensions==4.10.0
191191
# pydantic
192192
# typer-slim
193193
# types-aiobotocore
194+
# types-aiobotocore-ec2
194195
# types-aiobotocore-s3
195196
ujson==5.9.0
196197
# via aiohttp-swagger

‎services/storage/requirements/ci.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
--requirement _test.txt
1212

1313
# installs this repo's packages
14+
simcore-aws-library @ ../../packages/aws-library/
1415
simcore-models-library @ ../../packages/models-library/
1516
simcore-postgres-database @ ../../packages/postgres-database/
1617
pytest-simcore @ ../../packages/pytest-simcore/

‎services/storage/requirements/dev.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
--requirement _tools.txt
1313

1414
# installs this repo's packages
15+
--editable ../../packages/aws-library/
1516
--editable ../../packages/models-library
1617
--editable ../../packages/postgres-database/
1718
--editable ../../packages/pytest-simcore/

‎services/storage/requirements/prod.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
--requirement _base.txt
1111

1212
# installs this repo's packages
13+
simcore-aws-library @ ../../packages/aws-library/
1314
simcore-models-library @ ../../packages/models-library/
1415
simcore-postgres-database @ ../../packages/postgres-database/
1516
simcore-service-library[aiohttp] @ ../../packages/service-library

0 commit comments

Comments
 (0)