Merge branch 'is4481/upgrade-libs' into is4481/upgrade-services

Author: giancarloromeo

packages/aws-library/src/aws_library/s3/_client.py

@@ -13,6 +13,7 @@
 from boto3.s3.transfer import TransferConfig
 from botocore import exceptions as botocore_exc
 from botocore.client import Config
+from common_library.pydantic_type_adapters import AnyUrlLegacyAdapter
 from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart
 from models_library.basic_types import SHA256Str
 from pydantic import AnyUrl, ByteSize, TypeAdapter
@@ -43,8 +44,8 @@
 _MAX_CONCURRENT_COPY: Final[int] = 4
 _AWS_MAX_ITEMS_PER_PAGE: Final[int] = 1000
 
-_ANY_URL_ADAPTER: Final[TypeAdapter[AnyUrl]] = TypeAdapter(AnyUrl)
-_LIST_ANY_URL_ADAPTER: Final[TypeAdapter[list[AnyUrl]]] = TypeAdapter(list[AnyUrl])
+
+ListAnyUrlTypeAdapter: Final[TypeAdapter[list[AnyUrl]]] = TypeAdapter(list[AnyUrl])
 
 
 class UploadedBytesTransferredCallback(Protocol):
@@ -254,7 +255,7 @@ async def create_single_presigned_download_link(
         bucket: S3BucketName,
         object_key: S3ObjectKey,
         expiration_secs: int,
-    ) -> AnyUrl:
+    ) -> str:
         # NOTE: ensure the bucket/object exists, this will raise if not
         await self._client.head_bucket(Bucket=bucket)
         await self._client.head_object(Bucket=bucket, Key=object_key)
@@ -263,20 +264,20 @@ async def create_single_presigned_download_link(
             Params={"Bucket": bucket, "Key": object_key},
             ExpiresIn=expiration_secs,
         )
-        return _ANY_URL_ADAPTER.validate_python(generated_link)
+        return f"{AnyUrlLegacyAdapter.validate_python(generated_link)}"
 
     @s3_exception_handler(_logger)
     async def create_single_presigned_upload_link(
         self, *, bucket: S3BucketName, object_key: S3ObjectKey, expiration_secs: int
-    ) -> AnyUrl:
+    ) -> str:
         # NOTE: ensure the bucket/object exists, this will raise if not
         await self._client.head_bucket(Bucket=bucket)
         generated_link = await self._client.generate_presigned_url(
             "put_object",
             Params={"Bucket": bucket, "Key": object_key},
             ExpiresIn=expiration_secs,
         )
-        return _ANY_URL_ADAPTER.validate_python(generated_link)
+        return f"{AnyUrlLegacyAdapter.validate_python(generated_link)}"
 
     @s3_exception_handler(_logger)
     async def create_multipart_upload_links(
@@ -299,7 +300,7 @@ async def create_multipart_upload_links(
         # compute the number of links, based on the announced file size
         num_upload_links, chunk_size = compute_num_file_chunks(file_size)
         # now create the links
-        upload_links = _LIST_ANY_URL_ADAPTER.validate_python(
+        upload_links = ListAnyUrlTypeAdapter.validate_python(
             await asyncio.gather(
                 *(
                     self._client.generate_presigned_url(
@@ -473,6 +474,6 @@ def is_multipart(file_size: ByteSize) -> bool:
 
     @staticmethod
     def compute_s3_url(*, bucket: S3BucketName, object_key: S3ObjectKey) -> AnyUrl:
-        return _ANY_URL_ADAPTER.validate_python(
+        return AnyUrlLegacyAdapter.validate_python(
             f"s3://{bucket}/{urllib.parse.quote(object_key)}"
         )

packages/aws-library/tests/test_s3_client.py

@@ -694,11 +694,11 @@ async def test_create_single_presigned_download_link(
         object_key=with_uploaded_file_on_s3.s3_key,
         expiration_secs=default_expiration_time_seconds,
     )
-    assert isinstance(download_url, AnyUrl)
+    assert download_url
 
     dest_file = tmp_path / faker.file_name()
     async with ClientSession() as session:
-        response = await session.get(str(download_url))
+        response = await session.get(download_url)
         response.raise_for_status()
         with dest_file.open("wb") as fp:
             fp.write(await response.read())
@@ -744,7 +744,7 @@ async def test_create_single_presigned_upload_link(
     create_file_of_size: Callable[[ByteSize], Path],
     default_expiration_time_seconds: int,
     upload_to_presigned_link: Callable[
-        [Path, AnyUrl, S3BucketName, S3ObjectKey], Awaitable[None]
+        [Path, str, S3BucketName, S3ObjectKey], Awaitable[None]
     ],
 ):
     file = create_file_of_size(_BYTE_SIZE_ADAPTER.validate_python("1Mib"))

packages/common-library/src/common_library/pydantic_networks_extension.py

@@ -1,15 +1,25 @@
 from typing import Annotated, TypeAlias
 
-from pydantic import AfterValidator, AnyHttpUrl, HttpUrl
+from pydantic import AfterValidator, AnyHttpUrl, AnyUrl, HttpUrl
+from pydantic_core import Url
 
 
-def _strip_last_slash(url: str) -> str:
-    return url.rstrip("/")
+def _strip_last_slash(url: Url) -> str:
+    return f"{url}".rstrip("/")
 
 
+AnyUrlLegacy: TypeAlias = Annotated[
+    AnyUrl,
+    AfterValidator(_strip_last_slash),
+]
+
 AnyHttpUrlLegacy: TypeAlias = Annotated[
-    str, AnyHttpUrl, AfterValidator(_strip_last_slash)
+    AnyHttpUrl,
+    AfterValidator(_strip_last_slash),
 ]
 
 
-HttpUrlLegacy: TypeAlias = Annotated[str, HttpUrl, AfterValidator(_strip_last_slash)]
+HttpUrlLegacy: TypeAlias = Annotated[
+    HttpUrl,
+    AfterValidator(_strip_last_slash),
+]

packages/common-library/src/common_library/pydantic_type_adapters.py

@@ -0,0 +1,8 @@
+from typing import Final
+
+from common_library.pydantic_networks_extension import AnyHttpUrlLegacy, AnyUrlLegacy
+from pydantic import TypeAdapter
+
+AnyUrlLegacyAdapter: Final[TypeAdapter[AnyUrlLegacy]] = TypeAdapter(AnyUrlLegacy)
+
+AnyHttpUrlLegacyAdapter: Final[TypeAdapter[AnyHttpUrlLegacy]] = TypeAdapter(AnyHttpUrlLegacy)

packages/common-library/tests/test_pydantic_fields_extension.py

@@ -1,4 +1,4 @@
-from typing import Literal
+from typing import Any, Callable, Literal
 
 import pytest
 from common_library.pydantic_fields_extension import get_type, is_literal, is_nullable
@@ -68,5 +68,5 @@ class MyModel(BaseModel):
         (is_nullable, False, "e"),
     ],
 )
-def test_field_fn(fn, expected, name):
+def test_field_fn(fn: Callable[[Any], Any], expected: Any, name: str):
     assert expected == fn(MyModel.model_fields[name])

packages/common-library/tests/test_pydantic_networks_extension.py

@@ -1,20 +1,39 @@
+import pytest
 from common_library.pydantic_networks_extension import AnyHttpUrlLegacy
-from pydantic import AnyHttpUrl, TypeAdapter
+from pydantic import AnyHttpUrl, BaseModel, TypeAdapter, ValidationError
 from pydantic_core import Url
 
 
+class A(BaseModel):
+    url: AnyHttpUrlLegacy
+
+
 def test_any_http_url():
     url = TypeAdapter(AnyHttpUrl).validate_python(
         "http://backgroud.testserver.io",
     )
 
     assert isinstance(url, Url)
-    assert f"{url}" == "http://backgroud.testserver.io/"    # NOTE: trailing '/' added in Pydantic v2
+    assert (
+        f"{url}" == "http://backgroud.testserver.io/"
+    )  # trailing slash added (in Pydantic v2)
+
 
 def test_any_http_url_legacy():
     url = TypeAdapter(AnyHttpUrlLegacy).validate_python(
-            "http://backgroud.testserver.io",
+        "http://backgroud.testserver.io",
     )
 
     assert isinstance(url, str)
-    assert url == "http://backgroud.testserver.io"
+    assert url == "http://backgroud.testserver.io"  # no trailing slash was added
+
+
+def test_valid_any_http_url_legacy_field():
+    a = A(url="http://backgroud.testserver.io")  # type: ignore
+
+    assert a.url == "http://backgroud.testserver.io"  # no trailing slash was added
+
+
+def test_not_valid_any_http_url_legacy_field():
+    with pytest.raises(ValidationError):
+        A(url="htttttp://backgroud.testserver.io")  # type: ignore

packages/common-library/tests/test_serialization.py

@@ -1,5 +1,3 @@
-from typing import Final
-
 import pytest
 from common_library.serialization import model_dump_with_secrets
 from pydantic import BaseModel, SecretStr
@@ -10,9 +8,6 @@ class Credentials(BaseModel):
     PASSWORD: SecretStr | None = None
 
 
-ME: Final[Credentials] = Credentials(USERNAME="DeepThought", PASSWORD=SecretStr("42"))
-
-
 @pytest.mark.parametrize(
     "expected,show_secrets",
     [
@@ -27,4 +22,4 @@ class Credentials(BaseModel):
     ],
 )
 def test_model_dump_with_secrets(expected: dict, show_secrets: bool):
-    assert expected == model_dump_with_secrets(ME, show_secrets=show_secrets)
+    assert expected == model_dump_with_secrets(Credentials(USERNAME="DeepThought", PASSWORD=SecretStr("42")), show_secrets=show_secrets)

packages/models-library/src/models_library/projects_nodes_io.py

@@ -7,7 +7,7 @@
 """
 
 from pathlib import Path
-from typing import Annotated, Final, TypeAlias
+from typing import Annotated, TypeAlias
 from uuid import UUID
 
 from models_library.basic_types import ConstrainedStr, KeyIDStr
@@ -45,9 +45,6 @@
 ]
 
 
-_ANY_URL_ADAPTER: Final[TypeAdapter[AnyUrl]] = TypeAdapter(AnyUrl)
-
-
 class SimcoreS3DirectoryID(ConstrainedStr):
     """
     A simcore directory has the following structure:
@@ -126,7 +123,7 @@ class DownloadLink(BaseModel):
     """I/O port type to hold a generic download link to a file (e.g. S3 pre-signed link, etc)"""
 
     download_link: Annotated[
-        str, BeforeValidator(lambda x: str(_ANY_URL_ADAPTER.validate_python(x)))
+        str, BeforeValidator(lambda x: str(TypeAdapter(AnyUrl).validate_python(x)))
     ] = Field(..., alias="downloadLink")
     label: str | None = Field(default=None, description="Display name")
     model_config = ConfigDict(
@@ -177,10 +174,9 @@ def legacy_enforce_str_to_int(cls, v):
         return v
 
     model_config = ConfigDict(
-        populate_by_name=True
+        populate_by_name=True,
     )
 
-
 class SimCoreFileLink(BaseFileLink):
     """I/O port type to hold a link to a file in simcore S3 storage"""
 

packages/models-library/src/models_library/rest_pagination.py

@@ -1,7 +1,7 @@
 from typing import Annotated, Final, Generic, TypeAlias, TypeVar
 
+from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter
 from pydantic import (
-    AnyHttpUrl,
     BaseModel,
     BeforeValidator,
     ConfigDict,
@@ -15,8 +15,6 @@
 
 from .utils.common_validators import none_to_empty_list_pre_validator
 
-_ANY_HTTP_URL_ADAPTER: Final[TypeAdapter[AnyHttpUrl]] = TypeAdapter(AnyHttpUrl)
-
 # Default limit values
 #  - Using same values across all pagination entrypoints simplifies
 #    interconnecting paginated calls
@@ -101,7 +99,7 @@ class PageLinks(
     PageRefs[
         Annotated[
             str,
-            BeforeValidator(lambda x: str(_ANY_HTTP_URL_ADAPTER.validate_python(x))),
+            BeforeValidator(lambda x: str(AnyHttpUrlLegacyAdapter.validate_python(x))),
         ]
     ]
 ):

packages/models-library/src/models_library/rest_pagination_utils.py

@@ -1,7 +1,7 @@
 from math import ceil
 from typing import Any, Protocol, TypedDict, Union, runtime_checkable
 
-from pydantic import AnyHttpUrl, TypeAdapter
+from common_library.pydantic_type_adapters import AnyHttpUrlLegacyAdapter
 
 from .rest_pagination import PageLinks, PageMetaInfoLimitOffset
 
@@ -29,7 +29,6 @@ def replace_query_params(self, **kwargs: Any) -> "_StarletteURL":
 
 
 _URLType = Union[_YarlURL, _StarletteURL]
-_ANY_HTTP_URL_ADAPTER: TypeAdapter = TypeAdapter(AnyHttpUrl)
 
 
 def _replace_query(url: _URLType, query: dict[str, Any]) -> str:
@@ -39,7 +38,9 @@ def _replace_query(url: _URLType, query: dict[str, Any]) -> str:
         new_url = url.update_query(query)
     else:
         new_url = url.replace_query_params(**query)
-    return f"{new_url}"
+
+    new_url_str = f"{new_url}"
+    return f"{AnyHttpUrlLegacyAdapter.validate_python(new_url_str)}"
 
 
 class PageDict(TypedDict):
@@ -72,33 +73,21 @@ def paginate_data(
             total=total, count=len(chunk), limit=limit, offset=offset
         ),
         _links=PageLinks(
-            self=(
-                _ANY_HTTP_URL_ADAPTER.validate_python(
-                    _replace_query(request_url, {"offset": offset, "limit": limit}),
-                )
-            ),
-            first=_ANY_HTTP_URL_ADAPTER.validate_python(
-                _replace_query(request_url, {"offset": 0, "limit": limit})
-            ),
-            prev=_ANY_HTTP_URL_ADAPTER.validate_python(
-                _replace_query(
-                    request_url, {"offset": max(offset - limit, 0), "limit": limit}
-                ),
+            self=_replace_query(request_url, {"offset": offset, "limit": limit}),
+            first=_replace_query(request_url, {"offset": 0, "limit": limit}),
+            prev=_replace_query(
+                request_url, {"offset": max(offset - limit, 0), "limit": limit}
             )
             if offset > 0
             else None,
-            next=_ANY_HTTP_URL_ADAPTER.validate_python(
-                _replace_query(
-                    request_url,
-                    {"offset": min(offset + limit, last_page * limit), "limit": limit},
-                ),
+            next=_replace_query(
+                request_url,
+                {"offset": min(offset + limit, last_page * limit), "limit": limit},
             )
             if offset < (last_page * limit)
             else None,
-            last=_ANY_HTTP_URL_ADAPTER.validate_python(
-                _replace_query(
-                    request_url, {"offset": last_page * limit, "limit": limit}
-                ),
+            last=_replace_query(
+                request_url, {"offset": last_page * limit, "limit": limit}
             ),
         ),
         data=chunk,