⬆️Migration: Storage (#6599)

Author: sanderegg

packages/aws-library/src/aws_library/s3/_client.py

@@ -13,7 +13,6 @@
 from boto3.s3.transfer import TransferConfig
 from botocore import exceptions as botocore_exc
 from botocore.client import Config
-from common_library.pydantic_networks_extension import AnyUrlLegacy
 from models_library.api_schemas_storage import ETag, S3BucketName, UploadedPart
 from models_library.basic_types import SHA256Str
 from pydantic import AnyUrl, ByteSize, TypeAdapter
@@ -255,7 +254,7 @@ async def create_single_presigned_download_link(
         bucket: S3BucketName,
         object_key: S3ObjectKey,
         expiration_secs: int,
-    ) -> str:
+    ) -> AnyUrl:
         # NOTE: ensure the bucket/object exists, this will raise if not
         await self._client.head_bucket(Bucket=bucket)
         await self._client.head_object(Bucket=bucket, Key=object_key)
@@ -264,20 +263,20 @@ async def create_single_presigned_download_link(
             Params={"Bucket": bucket, "Key": object_key},
             ExpiresIn=expiration_secs,
         )
-        return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}"
+        return TypeAdapter(AnyUrl).validate_python(generated_link)
 
     @s3_exception_handler(_logger)
     async def create_single_presigned_upload_link(
         self, *, bucket: S3BucketName, object_key: S3ObjectKey, expiration_secs: int
-    ) -> str:
+    ) -> AnyUrl:
         # NOTE: ensure the bucket/object exists, this will raise if not
         await self._client.head_bucket(Bucket=bucket)
         generated_link = await self._client.generate_presigned_url(
             "put_object",
             Params={"Bucket": bucket, "Key": object_key},
             ExpiresIn=expiration_secs,
         )
-        return f"{TypeAdapter(AnyUrlLegacy).validate_python(generated_link)}"
+        return TypeAdapter(AnyUrl).validate_python(generated_link)
 
     @s3_exception_handler(_logger)
     async def create_multipart_upload_links(
@@ -474,6 +473,6 @@ def is_multipart(file_size: ByteSize) -> bool:
 
     @staticmethod
     def compute_s3_url(*, bucket: S3BucketName, object_key: S3ObjectKey) -> AnyUrl:
-        return TypeAdapter(AnyUrlLegacy).validate_python(
+        return TypeAdapter(AnyUrl).validate_python(
             f"s3://{bucket}/{urllib.parse.quote(object_key)}"
         )

packages/aws-library/tests/test_s3_client.py

@@ -742,7 +742,7 @@ async def test_create_single_presigned_upload_link(
     create_file_of_size: Callable[[ByteSize], Path],
     default_expiration_time_seconds: int,
     upload_to_presigned_link: Callable[
-        [Path, str, S3BucketName, S3ObjectKey], Awaitable[None]
+        [Path, AnyUrl, S3BucketName, S3ObjectKey], Awaitable[None]
     ],
 ):
     file = create_file_of_size(TypeAdapter(ByteSize).validate_python("1Mib"))
@@ -1289,20 +1289,24 @@ def test_is_multipart(file_size: ByteSize, expected_multipart: bool):
         (
             "some-bucket",
             "an/object/separate/by/slashes",
-            "s3://some-bucket/an/object/separate/by/slashes",
+            TypeAdapter(AnyUrl).validate_python(
+                "s3://some-bucket/an/object/separate/by/slashes"
+            ),
         ),
         (
             "some-bucket",
             "an/object/separate/by/slashes-?/3#$",
-            r"s3://some-bucket/an/object/separate/by/slashes-%3F/3%23%24",
+            TypeAdapter(AnyUrl).validate_python(
+                r"s3://some-bucket/an/object/separate/by/slashes-%3F/3%23%24"
+            ),
         ),
     ],
 )
 def test_compute_s3_url(
     bucket: S3BucketName, object_key: S3ObjectKey, expected_s3_url: AnyUrl
 ):
     assert (
-        str(SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key))
+        SimcoreS3API.compute_s3_url(bucket=bucket, object_key=object_key)
         == expected_s3_url
     )
 

packages/common-library/src/common_library/pydantic_networks_extension.py

@@ -1,18 +1,13 @@
 from typing import Annotated, TypeAlias
 
-from pydantic import AfterValidator, AnyHttpUrl, AnyUrl, HttpUrl
+from pydantic import AfterValidator, AnyHttpUrl, HttpUrl
 from pydantic_core import Url
 
 
 def _strip_last_slash(url: Url) -> str:
     return f"{url}".rstrip("/")
 
 
-AnyUrlLegacy: TypeAlias = Annotated[
-    AnyUrl,
-    AfterValidator(_strip_last_slash),
-]
-
 AnyHttpUrlLegacy: TypeAlias = Annotated[
     AnyHttpUrl,
     AfterValidator(_strip_last_slash),

packages/models-library/src/models_library/api_schemas_storage.py

@@ -8,7 +8,7 @@
 
 from datetime import datetime
 from enum import Enum
-from typing import Annotated, Any, TypeAlias
+from typing import Annotated, Any, Literal, Self, TypeAlias
 from uuid import UUID
 
 from pydantic import (
@@ -107,6 +107,10 @@ class DatasetMetaDataGet(BaseModel):
     )
 
 
+UNDEFINED_SIZE_TYPE: TypeAlias = Literal[-1]
+UNDEFINED_SIZE: UNDEFINED_SIZE_TYPE = -1
+
+
 # /locations/{location_id}/files/metadata:
 # /locations/{location_id}/files/{file_id}/metadata:
 class FileMetaDataGet(BaseModel):
@@ -130,8 +134,8 @@ class FileMetaDataGet(BaseModel):
     )
     created_at: datetime
     last_modified: datetime
-    file_size: ByteSize | int = Field(
-        default=-1, description="File size in bytes (-1 means invalid)"
+    file_size: UNDEFINED_SIZE_TYPE | ByteSize = Field(
+        default=UNDEFINED_SIZE, description="File size in bytes (-1 means invalid)"
     )
     entity_tag: ETag | None = Field(
         default=None,
@@ -149,7 +153,7 @@ class FileMetaDataGet(BaseModel):
     )
 
     model_config = ConfigDict(
-        extra="forbid",
+        extra="ignore",
         from_attributes=True,
         json_schema_extra={
             "examples": [
@@ -312,19 +316,18 @@ class FoldersBody(BaseModel):
     nodes_map: dict[NodeID, NodeID] = Field(default_factory=dict)
 
     @model_validator(mode="after")
-    @classmethod
-    def ensure_consistent_entries(cls, values):
-        source_node_keys = (NodeID(n) for n in values["source"].get("workbench", {}))
-        if set(source_node_keys) != set(values["nodes_map"].keys()):
+    def ensure_consistent_entries(self) -> Self:
+        source_node_keys = (NodeID(n) for n in self.source.get("workbench", {}))
+        if set(source_node_keys) != set(self.nodes_map.keys()):
             msg = "source project nodes do not fit with nodes_map entries"
             raise ValueError(msg)
         destination_node_keys = (
-            NodeID(n) for n in values["destination"].get("workbench", {})
+            NodeID(n) for n in self.destination.get("workbench", {})
         )
-        if set(destination_node_keys) != set(values["nodes_map"].values()):
+        if set(destination_node_keys) != set(self.nodes_map.values()):
             msg = "destination project nodes do not fit with nodes_map values"
             raise ValueError(msg)
-        return values
+        return self
 
 
 class SoftCopyBody(BaseModel):

packages/models-library/src/models_library/utils/json_serialization.py

@@ -24,6 +24,7 @@
 
 import orjson
 from pydantic import NameEmail, SecretBytes, SecretStr
+from pydantic_core import Url
 from pydantic_extra_types.color import Color
 
 
@@ -84,6 +85,7 @@ def decimal_encoder(dec_value: Decimal) -> int | float:
     SecretBytes: str,
     SecretStr: str,
     set: list,
+    Url: str,
     UUID: str,
 }
 

packages/settings-library/src/settings_library/node_ports.py

@@ -1,5 +1,5 @@
 from datetime import timedelta
-from typing import Final
+from typing import Final, Self
 
 from pydantic import Field, NonNegativeInt, PositiveInt, SecretStr, model_validator
 
@@ -22,14 +22,13 @@ def auth_required(self) -> bool:
         return self.STORAGE_USERNAME is not None and self.STORAGE_PASSWORD is not None
 
     @model_validator(mode="after")
-    @classmethod
-    def _validate_auth_fields(cls, values):
-        username = values.STORAGE_USERNAME
-        password = values.STORAGE_PASSWORD
+    def _validate_auth_fields(self) -> Self:
+        username = self.STORAGE_USERNAME
+        password = self.STORAGE_PASSWORD
         if (username is None) != (password is None):
             msg = f"Both {username=} and {password=} must be either set or unset!"
             raise ValueError(msg)
-        return values
+        return self
 
 
 class NodePortsSettings(BaseCustomSettings):

packages/settings-library/src/settings_library/utils_cli.py

@@ -2,6 +2,7 @@
 import logging
 import os
 from collections.abc import Callable
+from enum import Enum
 from pprint import pformat
 from typing import Any
 
@@ -64,7 +65,8 @@ def print_as_envfile(
 
         if verbose and field.description:
             typer.echo(f"# {field.description}")
-
+        if isinstance(value, Enum):
+            value = value.value
         typer.echo(f"{name}={value}")
 
 

packages/simcore-sdk/src/simcore_sdk/node_data/data_manager.py

@@ -6,7 +6,7 @@
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID, StorageFileID
 from models_library.users import UserID
-from pydantic import ByteSize, TypeAdapter
+from pydantic import TypeAdapter
 from servicelib.archiving_utils import unarchive_dir
 from servicelib.logging_utils import log_context
 from servicelib.progress_bar import ProgressBarData
@@ -184,15 +184,6 @@ async def _delete_legacy_archive(
     )
 
 
-async def get_remote_size(
-    *, user_id: UserID, project_id: ProjectID, node_uuid: NodeID, source_path: Path
-) -> ByteSize:
-    s3_object = __create_s3_object_key(project_id, node_uuid, source_path)
-    return await filemanager.get_path_size(
-        user_id=user_id, store_id=SIMCORE_LOCATION, s3_object=s3_object
-    )
-
-
 async def push(
     user_id: UserID,
     project_id: ProjectID,

packages/simcore-sdk/src/simcore_sdk/node_ports_common/filemanager.py

@@ -15,7 +15,7 @@
 from models_library.basic_types import IDStr, SHA256Str
 from models_library.projects_nodes_io import LocationID, LocationName, StorageFileID
 from models_library.users import UserID
-from pydantic import AnyUrl, ByteSize, TypeAdapter, parse_obj_as
+from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.file_utils import create_sha256_checksum
 from servicelib.progress_bar import ProgressBarData
 from settings_library.aws_s3_cli import AwsS3CliSettings
@@ -189,7 +189,9 @@ async def download_path_from_s3(
                     aws_s3_cli_settings,
                     progress_bar,
                     local_directory_path=local_path,
-                    download_s3_link=TypeAdapter(AnyUrl).validate_python(f"{download_link}"),
+                    download_s3_link=TypeAdapter(AnyUrl).validate_python(
+                        f"{download_link}"
+                    ),
                 )
             elif r_clone_settings:
                 await r_clone.sync_s3_to_local(
@@ -572,21 +574,6 @@ async def get_file_metadata(
     )
 
 
-async def get_path_size(
-    user_id: UserID,
-    store_id: LocationID,
-    s3_object: StorageFileID,
-    client_session: ClientSession | None = None,
-) -> ByteSize:
-    file_metadata: FileMetaDataGet = await _get_file_meta_data(
-        user_id=user_id,
-        store_id=store_id,
-        s3_object=s3_object,
-        client_session=client_session,
-    )
-    return ByteSize(file_metadata.file_size)
-
-
 async def delete_file(
     user_id: UserID,
     store_id: LocationID,

scripts/release/monitor_release/settings.py

@@ -1,6 +1,6 @@
 import os
 from pathlib import Path
-from typing import Final
+from typing import Final, Self
 
 from dotenv import load_dotenv
 from pydantic import BaseModel, Field, HttpUrl, TypeAdapter, model_validator
@@ -47,7 +47,7 @@ class ReleaseSettings(BaseSettings):
     model_config = SettingsConfigDict(extra="ignore")
 
     @model_validator(mode="after")
-    def deduce_portainer_url(self):
+    def deduce_portainer_url(self) -> Self:
         self.portainer_url = TypeAdapter(HttpUrl).validate_python(
             f"https://{self.PORTAINER_DOMAIN}"
         )