🎨 pydantic2 migration: fixed unit-tests for dynamic-sidecar

packages/aws-library/src/aws_library/s3/_constants.py

@@ -1,14 +1,13 @@
 from typing import Final
 
+from common_library.pydantic_type_adapters import ByteSizeAdapter
 from pydantic import ByteSize, TypeAdapter
 
 # NOTE: AWS S3 upload limits https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html
 MULTIPART_UPLOADS_MIN_TOTAL_SIZE: Final[ByteSize] = TypeAdapter(
     ByteSize
 ).validate_python("100MiB")
-MULTIPART_COPY_THRESHOLD: Final[ByteSize] = TypeAdapter(ByteSize).validate_python(
-    "100MiB"
-)
+MULTIPART_COPY_THRESHOLD: Final[ByteSize] = ByteSizeAdapter.validate_python("100MiB")
 
-PRESIGNED_LINK_MAX_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5GiB")
-S3_MAX_FILE_SIZE: Final[ByteSize] = TypeAdapter(ByteSize).validate_python("5TiB")
+PRESIGNED_LINK_MAX_SIZE: Final[ByteSize] = ByteSizeAdapter.validate_python("5GiB")
+S3_MAX_FILE_SIZE: Final[ByteSize] = ByteSizeAdapter.validate_python("5TiB")

packages/aws-library/src/aws_library/s3/_utils.py

@@ -1,13 +1,14 @@
 from typing import Final
 
-from pydantic import ByteSize, TypeAdapter
+from common_library.pydantic_type_adapters import ByteSizeAdapter
+from pydantic import ByteSize
 
 _MULTIPART_MAX_NUMBER_OF_PARTS: Final[int] = 10000
 
 # this is artifically defined, if possible we keep a maximum number of requests for parallel
 # uploading. If that is not possible then we create as many upload part as the max part size allows
 _MULTIPART_UPLOADS_TARGET_MAX_PART_SIZE: Final[list[ByteSize]] = [
-    TypeAdapter(ByteSize).validate_python(x)
+    ByteSizeAdapter.validate_python(x)
     for x in [
         "10Mib",
         "50Mib",

packages/aws-library/tests/test_s3_client.py

@@ -14,7 +14,7 @@
 from collections.abc import AsyncIterator, Awaitable, Callable
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Final
+from typing import Any
 
 import botocore.exceptions
 import pytest
@@ -28,6 +28,7 @@
     S3UploadNotFoundError,
 )
 from aws_library.s3._models import MultiPartUploadLinks
+from common_library.pydantic_type_adapters import ByteSizeAdapter
 from faker import Faker
 from models_library.api_schemas_storage import S3BucketName, UploadedPart
 from models_library.basic_types import SHA256Str
@@ -49,8 +50,6 @@
 from types_aiobotocore_s3 import S3Client
 from types_aiobotocore_s3.literals import BucketLocationConstraintType
 
-_BYTE_SIZE_ADAPTER: Final[TypeAdapter[ByteSize]] = TypeAdapter(ByteSize)
-
 
 @pytest.fixture
 async def simcore_s3_api(
@@ -113,7 +112,7 @@ async def _(
             file,
             MultiPartUploadLinks(
                 upload_id="fake",
-                chunk_size=_BYTE_SIZE_ADAPTER.validate_python(file.stat().st_size),
+                chunk_size=ByteSizeAdapter.validate_python(file.stat().st_size),
                 urls=[presigned_url],
             ),
         )
@@ -137,7 +136,7 @@ async def with_uploaded_file_on_s3(
     s3_client: S3Client,
     with_s3_bucket: S3BucketName,
 ) -> AsyncIterator[UploadedFile]:
-    test_file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("10Kib"))
+    test_file = create_file_of_size(ByteSizeAdapter.validate_python("10Kib"))
     await s3_client.upload_file(
         Filename=f"{test_file}",
         Bucket=with_s3_bucket,
@@ -592,7 +591,7 @@ async def test_undelete_file(
     assert file_metadata.size == with_uploaded_file_on_s3.local_path.stat().st_size
 
     # upload another file on top of the existing one
-    new_file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("5Kib"))
+    new_file = create_file_of_size(ByteSizeAdapter.validate_python("5Kib"))
     await s3_client.upload_file(
         Filename=f"{new_file}",
         Bucket=with_s3_bucket,
@@ -747,7 +746,7 @@ async def test_create_single_presigned_upload_link(
         [Path, str, S3BucketName, S3ObjectKey], Awaitable[None]
     ],
 ):
-    file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1Mib"))
+    file = create_file_of_size(ByteSizeAdapter.validate_python("1Mib"))
     s3_object_key = file.name
     presigned_url = await simcore_s3_api.create_single_presigned_upload_link(
         bucket=with_s3_bucket,
@@ -775,7 +774,7 @@ async def test_create_single_presigned_upload_link_with_non_existing_bucket_rais
     create_file_of_size: Callable[[ByteSize], Path],
     default_expiration_time_seconds: int,
 ):
-    file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1Mib"))
+    file = create_file_of_size(ByteSizeAdapter.validate_python("1Mib"))
     s3_object_key = file.name
     with pytest.raises(S3BucketInvalidError):
         await simcore_s3_api.create_single_presigned_upload_link(
@@ -1082,7 +1081,7 @@ async def test_copy_file_invalid_raises(
     create_file_of_size: Callable[[ByteSize], Path],
     faker: Faker,
 ):
-    file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1MiB"))
+    file = create_file_of_size(ByteSizeAdapter.validate_python("1MiB"))
     uploaded_file = await upload_file(file)
     dst_object_key = faker.file_name()
     # NOTE: since aioboto3 13.1.0 this raises S3KeyNotFoundError instead of S3BucketInvalidError
@@ -1107,9 +1106,9 @@ async def test_copy_file_invalid_raises(
     "directory_size, min_file_size, max_file_size",
     [
         (
-            _BYTE_SIZE_ADAPTER.validate_python("1Mib"),
-            _BYTE_SIZE_ADAPTER.validate_python("1B"),
-            _BYTE_SIZE_ADAPTER.validate_python("10Kib"),
+            ByteSizeAdapter.validate_python("1Mib"),
+            ByteSizeAdapter.validate_python("1B"),
+            ByteSizeAdapter.validate_python("10Kib"),
         )
     ],
     ids=byte_size_ids,
@@ -1133,9 +1132,9 @@ async def test_get_directory_metadata(
     "directory_size, min_file_size, max_file_size",
     [
         (
-            _BYTE_SIZE_ADAPTER.validate_python("1Mib"),
-            _BYTE_SIZE_ADAPTER.validate_python("1B"),
-            _BYTE_SIZE_ADAPTER.validate_python("10Kib"),
+            ByteSizeAdapter.validate_python("1Mib"),
+            ByteSizeAdapter.validate_python("1B"),
+            ByteSizeAdapter.validate_python("10Kib"),
         )
     ],
     ids=byte_size_ids,
@@ -1165,9 +1164,9 @@ async def test_get_directory_metadata_raises(
     "directory_size, min_file_size, max_file_size",
     [
         (
-            _BYTE_SIZE_ADAPTER.validate_python("1Mib"),
-            _BYTE_SIZE_ADAPTER.validate_python("1B"),
-            _BYTE_SIZE_ADAPTER.validate_python("10Kib"),
+            ByteSizeAdapter.validate_python("1Mib"),
+            ByteSizeAdapter.validate_python("1B"),
+            ByteSizeAdapter.validate_python("10Kib"),
         )
     ],
     ids=byte_size_ids,
@@ -1201,9 +1200,9 @@ async def test_delete_file_recursively(
     "directory_size, min_file_size, max_file_size",
     [
         (
-            _BYTE_SIZE_ADAPTER.validate_python("1Mib"),
-            _BYTE_SIZE_ADAPTER.validate_python("1B"),
-            _BYTE_SIZE_ADAPTER.validate_python("10Kib"),
+            ByteSizeAdapter.validate_python("1Mib"),
+            ByteSizeAdapter.validate_python("1B"),
+            ByteSizeAdapter.validate_python("10Kib"),
         )
     ],
     ids=byte_size_ids,
@@ -1239,9 +1238,9 @@ async def test_delete_file_recursively_raises(
     "directory_size, min_file_size, max_file_size",
     [
         (
-            _BYTE_SIZE_ADAPTER.validate_python("1Mib"),
-            _BYTE_SIZE_ADAPTER.validate_python("1B"),
-            _BYTE_SIZE_ADAPTER.validate_python("10Kib"),
+            ByteSizeAdapter.validate_python("1Mib"),
+            ByteSizeAdapter.validate_python("1B"),
+            ByteSizeAdapter.validate_python("10Kib"),
         )
     ],
     ids=byte_size_ids,
@@ -1339,14 +1338,14 @@ def run_async_test(*args, **kwargs) -> None:
     "directory_size, min_file_size, max_file_size",
     [
         (
-            _BYTE_SIZE_ADAPTER.validate_python("1Mib"),
-            _BYTE_SIZE_ADAPTER.validate_python("1B"),
-            _BYTE_SIZE_ADAPTER.validate_python("10Kib"),
+            ByteSizeAdapter.validate_python("1Mib"),
+            ByteSizeAdapter.validate_python("1B"),
+            ByteSizeAdapter.validate_python("10Kib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("500Mib"),
-            _BYTE_SIZE_ADAPTER.validate_python("10Mib"),
-            _BYTE_SIZE_ADAPTER.validate_python("50Mib"),
+            ByteSizeAdapter.validate_python("500Mib"),
+            ByteSizeAdapter.validate_python("10Mib"),
+            ByteSizeAdapter.validate_python("50Mib"),
         ),
     ],
     ids=byte_size_ids,

packages/aws-library/tests/test_s3_utils.py

@@ -4,72 +4,69 @@
 # pylint: disable=unused-variable
 
 
-from typing import Final
-
 import pytest
 from aws_library.s3._utils import (
     _MULTIPART_MAX_NUMBER_OF_PARTS,
     _MULTIPART_UPLOADS_TARGET_MAX_PART_SIZE,
     compute_num_file_chunks,
 )
-from pydantic import ByteSize, TypeAdapter
+from common_library.pydantic_type_adapters import ByteSizeAdapter
+from pydantic import ByteSize
 from pytest_simcore.helpers.parametrizations import byte_size_ids
 
-_BYTE_SIZE_ADAPTER: Final[TypeAdapter[ByteSize]] = TypeAdapter(ByteSize)
-
 
 @pytest.mark.parametrize(
     "file_size, expected_num_chunks, expected_chunk_size",
     [
         (
-            _BYTE_SIZE_ADAPTER.validate_python("5Mib"),
+            ByteSizeAdapter.validate_python("5Mib"),
             1,
-            _BYTE_SIZE_ADAPTER.validate_python("10Mib"),
+            ByteSizeAdapter.validate_python("10Mib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("10Mib"),
+            ByteSizeAdapter.validate_python("10Mib"),
             1,
-            _BYTE_SIZE_ADAPTER.validate_python("10Mib"),
+            ByteSizeAdapter.validate_python("10Mib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("20Mib"),
+            ByteSizeAdapter.validate_python("20Mib"),
             2,
-            _BYTE_SIZE_ADAPTER.validate_python("10Mib"),
+            ByteSizeAdapter.validate_python("10Mib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("50Mib"),
+            ByteSizeAdapter.validate_python("50Mib"),
             5,
-            _BYTE_SIZE_ADAPTER.validate_python("10Mib"),
+            ByteSizeAdapter.validate_python("10Mib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("150Mib"),
+            ByteSizeAdapter.validate_python("150Mib"),
             15,
-            _BYTE_SIZE_ADAPTER.validate_python("10Mib"),
+            ByteSizeAdapter.validate_python("10Mib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("550Mib"),
+            ByteSizeAdapter.validate_python("550Mib"),
             55,
-            _BYTE_SIZE_ADAPTER.validate_python("10Mib"),
+            ByteSizeAdapter.validate_python("10Mib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("560Gib"),
+            ByteSizeAdapter.validate_python("560Gib"),
             5735,
-            _BYTE_SIZE_ADAPTER.validate_python("100Mib"),
+            ByteSizeAdapter.validate_python("100Mib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("5Tib"),
+            ByteSizeAdapter.validate_python("5Tib"),
             8739,
-            _BYTE_SIZE_ADAPTER.validate_python("600Mib"),
+            ByteSizeAdapter.validate_python("600Mib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("15Tib"),
+            ByteSizeAdapter.validate_python("15Tib"),
             7680,
-            _BYTE_SIZE_ADAPTER.validate_python("2Gib"),
+            ByteSizeAdapter.validate_python("2Gib"),
         ),
         (
-            _BYTE_SIZE_ADAPTER.validate_python("9431773844"),
+            ByteSizeAdapter.validate_python("9431773844"),
             900,
-            _BYTE_SIZE_ADAPTER.validate_python("10Mib"),
+            ByteSizeAdapter.validate_python("10Mib"),
         ),
     ],
     ids=byte_size_ids,
@@ -83,7 +80,7 @@ def test_compute_num_file_chunks(
 
 
 def test_enormous_file_size_raises_value_error():
-    enormous_file_size = _BYTE_SIZE_ADAPTER.validate_python(
+    enormous_file_size = ByteSizeAdapter.validate_python(
         (
             max(_MULTIPART_UPLOADS_TARGET_MAX_PART_SIZE)
             * _MULTIPART_MAX_NUMBER_OF_PARTS

packages/common-library/src/common_library/pydantic_type_adapters.py

@@ -1,8 +1,12 @@
 from typing import Final
 
 from common_library.pydantic_networks_extension import AnyHttpUrlLegacy, AnyUrlLegacy
-from pydantic import TypeAdapter
+from pydantic import ByteSize, TypeAdapter
 
 AnyUrlLegacyAdapter: Final[TypeAdapter[AnyUrlLegacy]] = TypeAdapter(AnyUrlLegacy)
 
-AnyHttpUrlLegacyAdapter: Final[TypeAdapter[AnyHttpUrlLegacy]] = TypeAdapter(AnyHttpUrlLegacy)
+AnyHttpUrlLegacyAdapter: Final[TypeAdapter[AnyHttpUrlLegacy]] = TypeAdapter(
+    AnyHttpUrlLegacy
+)
+
+ByteSizeAdapter: Final[TypeAdapter[ByteSize]] = TypeAdapter(ByteSize)

packages/common-library/src/common_library/pydantic_validators.py

@@ -0,0 +1,20 @@
+import datetime
+
+from pydantic import field_validator
+
+
+def _try_convert_str_to_float_or_return(
+    v: datetime.timedelta | str | float,
+) -> datetime.timedelta | str | float:
+    if isinstance(v, str):
+        try:
+            return float(v)
+        except ValueError:
+            # returns format like "1:00:00"
+            return v
+    return v
+
+
+def timedelta_try_convert_str_to_float(field: str):
+    """Transforms a float/int number into a valid datetime as it used to work in the past"""
+    return field_validator(field, mode="before")(_try_convert_str_to_float_or_return)

packages/common-library/tests/test_pydantic_settings_validators.py

@@ -0,0 +1,40 @@
+from datetime import timedelta
+
+import pytest
+from common_library.pydantic_validators import timedelta_try_convert_str_to_float
+from faker import Faker
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
+
+
+def test_validate_timedelta_in_legacy_mode(
+    monkeypatch: pytest.MonkeyPatch, faker: Faker
+):
+    class Settings(BaseSettings):
+        APP_NAME: str
+        REQUEST_TIMEOUT: timedelta = Field(default=timedelta(seconds=40))
+
+        _try_convert_request_timeout = timedelta_try_convert_str_to_float(
+            "REQUEST_TIMEOUT"
+        )
+
+        model_config = SettingsConfigDict()
+
+    app_name = faker.pystr()
+    env_vars: dict[str, str | bool] = {"APP_NAME": app_name}
+
+    # without timedelta
+    setenvs_from_dict(monkeypatch, env_vars)
+    settings = Settings()
+    print(settings.model_dump())
+    assert app_name == settings.APP_NAME
+    assert timedelta(seconds=40) == settings.REQUEST_TIMEOUT
+
+    # with timedelta in seconds
+    env_vars["REQUEST_TIMEOUT"] = "5555"
+    setenvs_from_dict(monkeypatch, env_vars)
+    settings = Settings()
+    print(settings.model_dump())
+    assert app_name == settings.APP_NAME
+    assert timedelta(seconds=5555) == settings.REQUEST_TIMEOUT

packages/models-library/src/models_library/aiodocker_api.py

@@ -11,7 +11,7 @@
 
 
 class AioDockerContainerSpec(ContainerSpec):
-    Env: dict[str, str | None] | None = Field(  # type: ignore
+    Env: dict[str, str | None] | None = Field(
         default=None,
         description="aiodocker expects here a dictionary and re-convert it back internally`.\n",
     )