Skip to content

Commit cea85f8

Browse files
Merge branch 'is4481/upgrade-libs' into is4481/upgrade-api-server
2 parents 5e42c7c + e5f9263 commit cea85f8

File tree

333 files changed

+4871
-4309
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

333 files changed

+4871
-4309
lines changed

packages/aws-library/requirements/_base.txt

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ aiormq==6.8.0
3232
# via aio-pika
3333
aiosignal==1.3.1
3434
# via aiohttp
35+
annotated-types==0.7.0
36+
# via pydantic
3537
anyio==4.4.0
3638
# via
3739
# fast-depends
@@ -105,7 +107,7 @@ orjson==3.10.7
105107
# -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
106108
pamqp==3.3.0
107109
# via aiormq
108-
pydantic==1.10.17
110+
pydantic==2.9.1
109111
# via
110112
# -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
111113
# -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -120,6 +122,20 @@ pydantic==1.10.17
120122
# -r requirements/../../../packages/settings-library/requirements/_base.in
121123
# -r requirements/_base.in
122124
# fast-depends
125+
# pydantic-extra-types
126+
# pydantic-settings
127+
pydantic-core==2.23.3
128+
# via pydantic
129+
pydantic-extra-types==2.9.0
130+
# via
131+
# -r requirements/../../../packages/models-library/requirements/_base.in
132+
# -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
133+
pydantic-settings==2.5.2
134+
# via
135+
# -r requirements/../../../packages/models-library/requirements/_base.in
136+
# -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
137+
# -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
138+
# -r requirements/../../../packages/settings-library/requirements/_base.in
123139
pygments==2.18.0
124140
# via rich
125141
pyinstrument==4.7.2
@@ -128,6 +144,8 @@ python-dateutil==2.9.0.post0
128144
# via
129145
# arrow
130146
# botocore
147+
python-dotenv==1.0.1
148+
# via pydantic-settings
131149
pyyaml==6.0.2
132150
# via
133151
# -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -198,6 +216,7 @@ typing-extensions==4.12.2
198216
# aiodebug
199217
# faststream
200218
# pydantic
219+
# pydantic-core
201220
# typer
202221
# types-aiobotocore
203222
# types-aiobotocore-ec2

packages/aws-library/requirements/_test.txt

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
annotated-types==0.7.0
2+
# via
3+
# -c requirements/_base.txt
4+
# pydantic
15
antlr4-python3-runtime==4.13.2
26
# via moto
37
appdirs==1.4.4
@@ -151,11 +155,15 @@ py-partiql-parser==0.5.5
151155
# via moto
152156
pycparser==2.22
153157
# via cffi
154-
pydantic==1.10.17
158+
pydantic==2.9.1
155159
# via
156160
# -c requirements/../../../requirements/constraints.txt
157161
# -c requirements/_base.txt
158162
# aws-sam-translator
163+
pydantic-core==2.23.3
164+
# via
165+
# -c requirements/_base.txt
166+
# pydantic
159167
pyparsing==3.1.2
160168
# via moto
161169
pytest==8.3.2
@@ -193,7 +201,9 @@ python-dateutil==2.9.0.post0
193201
# faker
194202
# moto
195203
python-dotenv==1.0.1
196-
# via -r requirements/_test.in
204+
# via
205+
# -c requirements/_base.txt
206+
# -r requirements/_test.in
197207
pyyaml==6.0.2
198208
# via
199209
# -c requirements/../../../requirements/constraints.txt
@@ -268,6 +278,7 @@ typing-extensions==4.12.2
268278
# flexparser
269279
# pint
270280
# pydantic
281+
# pydantic-core
271282
# types-aioboto3
272283
# types-aiobotocore
273284
urllib3==2.2.2

packages/aws-library/src/aws_library/ec2/_client.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ async def create(cls, settings: EC2Settings) -> "SimcoreEC2API":
4141
session = aioboto3.Session()
4242
session_client = session.client(
4343
"ec2",
44-
endpoint_url=settings.EC2_ENDPOINT,
44+
endpoint_url=str(settings.EC2_ENDPOINT),
4545
aws_access_key_id=settings.EC2_ACCESS_KEY_ID,
4646
aws_secret_access_key=settings.EC2_SECRET_ACCESS_KEY,
4747
region_name=settings.EC2_REGION_NAME,

packages/aws-library/src/aws_library/ec2/_models.py

Lines changed: 47 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,19 @@
22
import re
33
import tempfile
44
from dataclasses import dataclass
5-
from typing import Any, ClassVar, TypeAlias
5+
from typing import Annotated, TypeAlias
66

77
import sh # type: ignore[import-untyped]
88
from models_library.docker import DockerGenericTag
99
from pydantic import (
1010
BaseModel,
1111
ByteSize,
12-
ConstrainedStr,
12+
ConfigDict,
1313
Field,
1414
NonNegativeFloat,
1515
NonNegativeInt,
16-
validator,
16+
StringConstraints,
17+
field_validator,
1718
)
1819
from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType
1920

@@ -33,26 +34,26 @@ def __gt__(self, other: "Resources") -> bool:
3334
return self.cpus > other.cpus or self.ram > other.ram
3435

3536
def __add__(self, other: "Resources") -> "Resources":
36-
return Resources.construct(
37+
return Resources.model_construct(
3738
**{
3839
key: a + b
3940
for (key, a), b in zip(
40-
self.dict().items(), other.dict().values(), strict=True
41+
self.model_dump().items(), other.model_dump().values(), strict=True
4142
)
4243
}
4344
)
4445

4546
def __sub__(self, other: "Resources") -> "Resources":
46-
return Resources.construct(
47+
return Resources.model_construct(
4748
**{
4849
key: a - b
4950
for (key, a), b in zip(
50-
self.dict().items(), other.dict().values(), strict=True
51+
self.model_dump().items(), other.model_dump().values(), strict=True
5152
)
5253
}
5354
)
5455

55-
@validator("cpus", pre=True)
56+
@field_validator("cpus", mode="before")
5657
@classmethod
5758
def _floor_cpus_to_0(cls, v: float) -> float:
5859
return max(v, 0)
@@ -67,19 +68,26 @@ class EC2InstanceType:
6768
InstancePrivateDNSName: TypeAlias = str
6869

6970

70-
class AWSTagKey(ConstrainedStr):
71-
# see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
72-
regex = re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]+$")
73-
min_length = 1
74-
max_length = 128
75-
76-
77-
class AWSTagValue(ConstrainedStr):
78-
# see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
79-
# quotes []{} were added as it allows to json encode. it seems to be accepted as a value
80-
regex = re.compile(r"^[a-zA-Z0-9\s\+\-=\.,_:/@\"\'\[\]\{\}]*$")
81-
min_length = 0
82-
max_length = 256
71+
# see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
72+
AWSTagKey: TypeAlias = Annotated[
73+
str,
74+
StringConstraints(
75+
min_length=1,
76+
max_length=128,
77+
pattern=re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]+$"),
78+
),
79+
]
80+
81+
# see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
82+
# quotes []{} were added as it allows to json encode. it seems to be accepted as a value
83+
AWSTagValue: TypeAlias = Annotated[
84+
str,
85+
StringConstraints(
86+
min_length=0,
87+
max_length=256,
88+
pattern=r"^[a-zA-Z0-9\s\+\-=\.,_:/@\"\'\[\]\{\}]*$",
89+
),
90+
]
8391

8492

8593
EC2Tags: TypeAlias = dict[AWSTagKey, AWSTagValue]
@@ -148,8 +156,23 @@ class EC2InstanceBootSpecific(BaseModel):
148156
default=0, description="number of buffer EC2s to keep (defaults to 0)"
149157
)
150158

151-
class Config:
152-
schema_extra: ClassVar[dict[str, Any]] = {
159+
@field_validator("custom_boot_scripts")
160+
@classmethod
161+
def validate_bash_calls(cls, v):
162+
try:
163+
with tempfile.NamedTemporaryFile(mode="wt", delete=True) as temp_file:
164+
temp_file.writelines(v)
165+
temp_file.flush()
166+
# NOTE: this will not capture runtime errors, but at least some syntax errors such as invalid quotes
167+
sh.bash("-n", temp_file.name)
168+
except sh.ErrorReturnCode as exc:
169+
msg = f"Invalid bash call in custom_boot_scripts: {v}, Error: {exc.stderr}"
170+
raise ValueError(msg) from exc
171+
172+
return v
173+
174+
model_config = ConfigDict(
175+
json_schema_extra={
153176
"examples": [
154177
{
155178
# just AMI
@@ -205,18 +228,4 @@ class Config:
205228
},
206229
]
207230
}
208-
209-
@validator("custom_boot_scripts")
210-
@classmethod
211-
def validate_bash_calls(cls, v):
212-
try:
213-
with tempfile.NamedTemporaryFile(mode="wt", delete=True) as temp_file:
214-
temp_file.writelines(v)
215-
temp_file.flush()
216-
# NOTE: this will not capture runtime errors, but at least some syntax errors such as invalid quotes
217-
sh.bash("-n", temp_file.name)
218-
except sh.ErrorReturnCode as exc:
219-
msg = f"Invalid bash call in custom_boot_scripts: {v}, Error: {exc.stderr}"
220-
raise ValueError(msg) from exc
221-
222-
return v
231+
)

packages/aws-library/src/aws_library/s3/_client.py

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
from botocore.client import Config
1616
from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart
1717
from models_library.basic_types import SHA256Str
18-
from pydantic import AnyUrl, ByteSize, parse_obj_as
18+
from pydantic import AnyUrl, ByteSize, TypeAdapter
1919
from servicelib.logging_utils import log_catch, log_context
2020
from servicelib.utils import limited_gather
2121
from settings_library.s3 import S3Settings
@@ -70,7 +70,7 @@ async def create(
7070
session = aioboto3.Session()
7171
session_client = session.client(
7272
"s3",
73-
endpoint_url=settings.S3_ENDPOINT,
73+
endpoint_url=str(settings.S3_ENDPOINT),
7474
aws_access_key_id=settings.S3_ACCESS_KEY,
7575
aws_secret_access_key=settings.S3_SECRET_KEY,
7676
region_name=settings.S3_REGION,
@@ -260,7 +260,7 @@ async def create_single_presigned_download_link(
260260
Params={"Bucket": bucket, "Key": object_key},
261261
ExpiresIn=expiration_secs,
262262
)
263-
url: AnyUrl = parse_obj_as(AnyUrl, generated_link)
263+
url: AnyUrl = TypeAdapter(AnyUrl).validate_python(generated_link)
264264
return url
265265

266266
@s3_exception_handler(_logger)
@@ -274,7 +274,7 @@ async def create_single_presigned_upload_link(
274274
Params={"Bucket": bucket, "Key": object_key},
275275
ExpiresIn=expiration_secs,
276276
)
277-
url: AnyUrl = parse_obj_as(AnyUrl, generated_link)
277+
url: AnyUrl = TypeAdapter(AnyUrl).validate_python(generated_link)
278278
return url
279279

280280
@s3_exception_handler(_logger)
@@ -298,8 +298,7 @@ async def create_multipart_upload_links(
298298
# compute the number of links, based on the announced file size
299299
num_upload_links, chunk_size = compute_num_file_chunks(file_size)
300300
# now create the links
301-
upload_links = parse_obj_as(
302-
list[AnyUrl],
301+
upload_links = TypeAdapter(list[AnyUrl]).validate_python(
303302
await asyncio.gather(
304303
*(
305304
self._client.generate_presigned_url(
@@ -473,7 +472,7 @@ def is_multipart(file_size: ByteSize) -> bool:
473472

474473
@staticmethod
475474
def compute_s3_url(*, bucket: S3BucketName, object_key: S3ObjectKey) -> AnyUrl:
476-
url: AnyUrl = parse_obj_as(
477-
AnyUrl, f"s3://{bucket}/{urllib.parse.quote(object_key)}"
475+
url: AnyUrl = TypeAdapter(AnyUrl).validate_python(
476+
f"s3://{bucket}/{urllib.parse.quote(object_key)}"
478477
)
479478
return url
Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
from typing import Final
22

3-
from pydantic import ByteSize, parse_obj_as
3+
from pydantic import ByteSize, TypeAdapter
44

55
# NOTE: AWS S3 upload limits https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
6-
MULTIPART_UPLOADS_MIN_TOTAL_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "100MiB")
7-
MULTIPART_COPY_THRESHOLD: Final[ByteSize] = parse_obj_as(ByteSize, "100MiB")
6+
MULTIPART_UPLOADS_MIN_TOTAL_SIZE: Final[ByteSize] = TypeAdapter(
7+
ByteSize
8+
).validate_python("100MiB")
9+
MULTIPART_COPY_THRESHOLD: Final[ByteSize] = TypeAdapter(ByteSize).validate_python(
10+
"100MiB"
11+
)
812

9-
PRESIGNED_LINK_MAX_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "5GiB")
10-
S3_MAX_FILE_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "5TiB")
13+
PRESIGNED_LINK_MAX_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5GiB")
14+
S3_MAX_FILE_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5TiB")
Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
from pydantic.errors import PydanticErrorMixin
1+
from models_library.errors_classes import OsparcErrorMixin
22

33

4-
class S3RuntimeError(PydanticErrorMixin, RuntimeError):
4+
class S3RuntimeError(OsparcErrorMixin, RuntimeError):
55
msg_template: str = "S3 client unexpected error"
66

77

@@ -10,25 +10,25 @@ class S3NotConnectedError(S3RuntimeError):
1010

1111

1212
class S3AccessError(S3RuntimeError):
13-
code = "s3_access.error"
13+
code = "s3_access.error" # type: ignore[assignment]
1414
msg_template: str = "Unexpected error while accessing S3 backend"
1515

1616

1717
class S3BucketInvalidError(S3AccessError):
18-
code = "s3_bucket.invalid_error"
18+
code = "s3_bucket.invalid_error" # type: ignore[assignment]
1919
msg_template: str = "The bucket '{bucket}' is invalid"
2020

2121

2222
class S3KeyNotFoundError(S3AccessError):
23-
code = "s3_key.not_found_error"
23+
code = "s3_key.not_found_error" # type: ignore[assignment]
2424
msg_template: str = "The file {key} in {bucket} was not found"
2525

2626

2727
class S3UploadNotFoundError(S3AccessError):
28-
code = "s3_upload.not_found_error"
28+
code = "s3_upload.not_found_error" # type: ignore[assignment]
2929
msg_template: str = "The upload for {key} in {bucket} was not found"
3030

3131

3232
class S3DestinationNotEmptyError(S3AccessError):
33-
code = "s3_destination.not_empty_error"
33+
code = "s3_destination.not_empty_error" # type: ignore[assignment]
3434
msg_template: str = "The destination {dst_prefix} is not empty"

packages/aws-library/src/aws_library/s3/_utils.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
from typing import Final
22

3-
from pydantic import ByteSize, parse_obj_as
3+
from pydantic import ByteSize, TypeAdapter
44

55
_MULTIPART_MAX_NUMBER_OF_PARTS: Final[int] = 10000
66

77
# this is artifically defined, if possible we keep a maximum number of requests for parallel
88
# uploading. If that is not possible then we create as many upload part as the max part size allows
99
_MULTIPART_UPLOADS_TARGET_MAX_PART_SIZE: Final[list[ByteSize]] = [
10-
parse_obj_as(ByteSize, x)
10+
TypeAdapter(ByteSize).validate_python(x)
1111
for x in [
1212
"10Mib",
1313
"50Mib",

0 commit comments

Comments
 (0)