⬆️ WIP: Upgrade api-server (pydantic v2)

packages/aws-library/requirements/_base.txt

@@ -32,6 +32,8 @@ aiormq==6.8.0
     # via aio-pika
 aiosignal==1.3.1
     # via aiohttp
+annotated-types==0.7.0
+    # via pydantic
 anyio==4.4.0
     # via
     #   fast-depends
@@ -105,7 +107,7 @@ orjson==3.10.7
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
 pamqp==3.3.0
     # via aiormq
-pydantic==1.10.17
+pydantic==2.9.1
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -120,6 +122,20 @@ pydantic==1.10.17
     #   -r requirements/../../../packages/settings-library/requirements/_base.in
     #   -r requirements/_base.in
     #   fast-depends
+    #   pydantic-extra-types
+    #   pydantic-settings
+pydantic-core==2.23.3
+    # via pydantic
+pydantic-extra-types==2.9.0
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+pydantic-settings==2.5.2
+    # via
+    #   -r requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
+    #   -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in
+    #   -r requirements/../../../packages/settings-library/requirements/_base.in
 pygments==2.18.0
     # via rich
 pyinstrument==4.7.2
@@ -128,6 +144,8 @@ python-dateutil==2.9.0.post0
     # via
     #   arrow
     #   botocore
+python-dotenv==1.0.1
+    # via pydantic-settings
 pyyaml==6.0.2
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -198,6 +216,7 @@ typing-extensions==4.12.2
     #   aiodebug
     #   faststream
     #   pydantic
+    #   pydantic-core
     #   typer
     #   types-aiobotocore
     #   types-aiobotocore-ec2

packages/aws-library/requirements/_test.txt

@@ -1,3 +1,7 @@
+annotated-types==0.7.0
+    # via
+    #   -c requirements/_base.txt
+    #   pydantic
 antlr4-python3-runtime==4.13.2
     # via moto
 appdirs==1.4.4
@@ -151,11 +155,15 @@ py-partiql-parser==0.5.5
     # via moto
 pycparser==2.22
     # via cffi
-pydantic==1.10.17
+pydantic==2.9.1
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   aws-sam-translator
+pydantic-core==2.23.3
+    # via
+    #   -c requirements/_base.txt
+    #   pydantic
 pyparsing==3.1.2
     # via moto
 pytest==8.3.2
@@ -193,7 +201,9 @@ python-dateutil==2.9.0.post0
     #   faker
     #   moto
 python-dotenv==1.0.1
-    # via -r requirements/_test.in
+    # via
+    #   -c requirements/_base.txt
+    #   -r requirements/_test.in
 pyyaml==6.0.2
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -268,6 +278,7 @@ typing-extensions==4.12.2
     #   flexparser
     #   pint
     #   pydantic
+    #   pydantic-core
     #   types-aioboto3
     #   types-aiobotocore
 urllib3==2.2.2

packages/aws-library/src/aws_library/ec2/_client.py

@@ -41,7 +41,7 @@ async def create(cls, settings: EC2Settings) -> "SimcoreEC2API":
         session = aioboto3.Session()
         session_client = session.client(
             "ec2",
-            endpoint_url=settings.EC2_ENDPOINT,
+            endpoint_url=str(settings.EC2_ENDPOINT),
             aws_access_key_id=settings.EC2_ACCESS_KEY_ID,
             aws_secret_access_key=settings.EC2_SECRET_ACCESS_KEY,
             region_name=settings.EC2_REGION_NAME,

packages/aws-library/src/aws_library/ec2/_models.py

@@ -2,18 +2,19 @@
 import re
 import tempfile
 from dataclasses import dataclass
-from typing import Any, ClassVar, TypeAlias
+from typing import Annotated, TypeAlias
 
 import sh  # type: ignore[import-untyped]
 from models_library.docker import DockerGenericTag
 from pydantic import (
     BaseModel,
     ByteSize,
-    ConstrainedStr,
+    ConfigDict,
     Field,
     NonNegativeFloat,
     NonNegativeInt,
-    validator,
+    StringConstraints,
+    field_validator,
 )
 from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType
 
@@ -33,26 +34,26 @@ def __gt__(self, other: "Resources") -> bool:
         return self.cpus > other.cpus or self.ram > other.ram
 
     def __add__(self, other: "Resources") -> "Resources":
-        return Resources.construct(
+        return Resources.model_construct(
             **{
                 key: a + b
                 for (key, a), b in zip(
-                    self.dict().items(), other.dict().values(), strict=True
+                    self.model_dump().items(), other.model_dump().values(), strict=True
                 )
             }
         )
 
     def __sub__(self, other: "Resources") -> "Resources":
-        return Resources.construct(
+        return Resources.model_construct(
             **{
                 key: a - b
                 for (key, a), b in zip(
-                    self.dict().items(), other.dict().values(), strict=True
+                    self.model_dump().items(), other.model_dump().values(), strict=True
                 )
             }
         )
 
-    @validator("cpus", pre=True)
+    @field_validator("cpus", mode="before")
     @classmethod
     def _floor_cpus_to_0(cls, v: float) -> float:
         return max(v, 0)
@@ -67,19 +68,26 @@ class EC2InstanceType:
 InstancePrivateDNSName: TypeAlias = str
 
 
-class AWSTagKey(ConstrainedStr):
-    # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
-    regex = re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]+$")
-    min_length = 1
-    max_length = 128
-
-
-class AWSTagValue(ConstrainedStr):
-    # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
-    # quotes []{} were added as it allows to json encode. it seems to be accepted as a value
-    regex = re.compile(r"^[a-zA-Z0-9\s\+\-=\.,_:/@\"\'\[\]\{\}]*$")
-    min_length = 0
-    max_length = 256
+# see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
+AWSTagKey: TypeAlias = Annotated[
+    str,
+    StringConstraints(
+        min_length=1,
+        max_length=128,
+        pattern=re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]+$"),
+    ),
+]
+
+# see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
+# quotes []{} were added as it allows to json encode. it seems to be accepted as a value
+AWSTagValue: TypeAlias = Annotated[
+    str,
+    StringConstraints(
+        min_length=0,
+        max_length=256,
+        pattern=r"^[a-zA-Z0-9\s\+\-=\.,_:/@\"\'\[\]\{\}]*$",
+    ),
+]
 
 
 EC2Tags: TypeAlias = dict[AWSTagKey, AWSTagValue]
@@ -148,8 +156,23 @@ class EC2InstanceBootSpecific(BaseModel):
         default=0, description="number of buffer EC2s to keep (defaults to 0)"
     )
 
-    class Config:
-        schema_extra: ClassVar[dict[str, Any]] = {
+    @field_validator("custom_boot_scripts")
+    @classmethod
+    def validate_bash_calls(cls, v):
+        try:
+            with tempfile.NamedTemporaryFile(mode="wt", delete=True) as temp_file:
+                temp_file.writelines(v)
+                temp_file.flush()
+                # NOTE: this will not capture runtime errors, but at least some syntax errors such as invalid quotes
+                sh.bash("-n", temp_file.name)
+        except sh.ErrorReturnCode as exc:
+            msg = f"Invalid bash call in custom_boot_scripts: {v}, Error: {exc.stderr}"
+            raise ValueError(msg) from exc
+
+        return v
+
+    model_config = ConfigDict(
+        json_schema_extra={
             "examples": [
                 {
                     # just AMI
@@ -205,18 +228,4 @@ class Config:
                 },
             ]
         }
-
-    @validator("custom_boot_scripts")
-    @classmethod
-    def validate_bash_calls(cls, v):
-        try:
-            with tempfile.NamedTemporaryFile(mode="wt", delete=True) as temp_file:
-                temp_file.writelines(v)
-                temp_file.flush()
-                # NOTE: this will not capture runtime errors, but at least some syntax errors such as invalid quotes
-                sh.bash("-n", temp_file.name)
-        except sh.ErrorReturnCode as exc:
-            msg = f"Invalid bash call in custom_boot_scripts: {v}, Error: {exc.stderr}"
-            raise ValueError(msg) from exc
-
-        return v
+    )

packages/aws-library/src/aws_library/s3/_client.py

@@ -15,7 +15,7 @@
 from botocore.client import Config
 from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart
 from models_library.basic_types import SHA256Str
-from pydantic import AnyUrl, ByteSize, parse_obj_as
+from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.logging_utils import log_catch, log_context
 from servicelib.utils import limited_gather
 from settings_library.s3 import S3Settings
@@ -70,7 +70,7 @@ async def create(
         session = aioboto3.Session()
         session_client = session.client(
             "s3",
-            endpoint_url=settings.S3_ENDPOINT,
+            endpoint_url=str(settings.S3_ENDPOINT),
             aws_access_key_id=settings.S3_ACCESS_KEY,
             aws_secret_access_key=settings.S3_SECRET_KEY,
             region_name=settings.S3_REGION,
@@ -260,7 +260,7 @@ async def create_single_presigned_download_link(
             Params={"Bucket": bucket, "Key": object_key},
             ExpiresIn=expiration_secs,
         )
-        url: AnyUrl = parse_obj_as(AnyUrl, generated_link)
+        url: AnyUrl = TypeAdapter(AnyUrl).validate_python(generated_link)
         return url
 
     @s3_exception_handler(_logger)
@@ -274,7 +274,7 @@ async def create_single_presigned_upload_link(
             Params={"Bucket": bucket, "Key": object_key},
             ExpiresIn=expiration_secs,
         )
-        url: AnyUrl = parse_obj_as(AnyUrl, generated_link)
+        url: AnyUrl = TypeAdapter(AnyUrl).validate_python(generated_link)
         return url
 
     @s3_exception_handler(_logger)
@@ -298,8 +298,7 @@ async def create_multipart_upload_links(
         # compute the number of links, based on the announced file size
         num_upload_links, chunk_size = compute_num_file_chunks(file_size)
         # now create the links
-        upload_links = parse_obj_as(
-            list[AnyUrl],
+        upload_links = TypeAdapter(list[AnyUrl]).validate_python(
             await asyncio.gather(
                 *(
                     self._client.generate_presigned_url(
@@ -473,7 +472,7 @@ def is_multipart(file_size: ByteSize) -> bool:
 
     @staticmethod
     def compute_s3_url(*, bucket: S3BucketName, object_key: S3ObjectKey) -> AnyUrl:
-        url: AnyUrl = parse_obj_as(
-            AnyUrl, f"s3://{bucket}/{urllib.parse.quote(object_key)}"
+        url: AnyUrl = TypeAdapter(AnyUrl).validate_python(
+            f"s3://{bucket}/{urllib.parse.quote(object_key)}"
         )
         return url

packages/aws-library/src/aws_library/s3/_constants.py

@@ -1,10 +1,14 @@
 from typing import Final
 
-from pydantic import ByteSize, parse_obj_as
+from pydantic import ByteSize, TypeAdapter
 
 # NOTE: AWS S3 upload limits https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
-MULTIPART_UPLOADS_MIN_TOTAL_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "100MiB")
-MULTIPART_COPY_THRESHOLD: Final[ByteSize] = parse_obj_as(ByteSize, "100MiB")
+MULTIPART_UPLOADS_MIN_TOTAL_SIZE: Final[ByteSize] = TypeAdapter(
+    ByteSize
+).validate_python("100MiB")
+MULTIPART_COPY_THRESHOLD: Final[ByteSize] = TypeAdapter(ByteSize).validate_python(
+    "100MiB"
+)
 
-PRESIGNED_LINK_MAX_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "5GiB")
-S3_MAX_FILE_SIZE: Final[ByteSize] = parse_obj_as(ByteSize, "5TiB")
+PRESIGNED_LINK_MAX_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5GiB")
+S3_MAX_FILE_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5TiB")

packages/aws-library/src/aws_library/s3/_errors.py

@@ -1,7 +1,7 @@
-from pydantic.errors import PydanticErrorMixin
+from models_library.errors_classes import OsparcErrorMixin
 
 
-class S3RuntimeError(PydanticErrorMixin, RuntimeError):
+class S3RuntimeError(OsparcErrorMixin, RuntimeError):
     msg_template: str = "S3 client unexpected error"
 
 
@@ -10,25 +10,25 @@ class S3NotConnectedError(S3RuntimeError):
 
 
 class S3AccessError(S3RuntimeError):
-    code = "s3_access.error"
+    code = "s3_access.error"    # type: ignore[assignment]
     msg_template: str = "Unexpected error while accessing S3 backend"
 
 
 class S3BucketInvalidError(S3AccessError):
-    code = "s3_bucket.invalid_error"
+    code = "s3_bucket.invalid_error"    # type: ignore[assignment]
     msg_template: str = "The bucket '{bucket}' is invalid"
 
 
 class S3KeyNotFoundError(S3AccessError):
-    code = "s3_key.not_found_error"
+    code = "s3_key.not_found_error" # type: ignore[assignment]
     msg_template: str = "The file {key}  in {bucket} was not found"
 
 
 class S3UploadNotFoundError(S3AccessError):
-    code = "s3_upload.not_found_error"
+    code = "s3_upload.not_found_error"  # type: ignore[assignment]
     msg_template: str = "The upload for {key}  in {bucket} was not found"
 
 
 class S3DestinationNotEmptyError(S3AccessError):
-    code = "s3_destination.not_empty_error"
+    code = "s3_destination.not_empty_error" # type: ignore[assignment]
     msg_template: str = "The destination {dst_prefix} is not empty"

packages/aws-library/src/aws_library/s3/_utils.py

@@ -1,13 +1,13 @@
 from typing import Final
 
-from pydantic import ByteSize, parse_obj_as
+from pydantic import ByteSize, TypeAdapter
 
 _MULTIPART_MAX_NUMBER_OF_PARTS: Final[int] = 10000
 
 # this is artifically defined, if possible we keep a maximum number of requests for parallel
 # uploading. If that is not possible then we create as many upload part as the max part size allows
 _MULTIPART_UPLOADS_TARGET_MAX_PART_SIZE: Final[list[ByteSize]] = [
-    parse_obj_as(ByteSize, x)
+    TypeAdapter(ByteSize).validate_python(x)
     for x in [
         "10Mib",
         "50Mib",