♻️Pydantic V2 migration: various fixes (#6647)

Author: sanderegg

packages/aws-library/tests/test_s3_client.py

@@ -696,7 +696,7 @@ async def test_create_single_presigned_download_link(
 
     dest_file = tmp_path / faker.file_name()
     async with ClientSession() as session:
-        response = await session.get(download_url)
+        response = await session.get(f"{download_url}")
         response.raise_for_status()
         with dest_file.open("wb") as fp:
             fp.write(await response.read())

packages/models-library/src/models_library/docker.py

@@ -75,7 +75,7 @@ def to_simcore_runtime_docker_label_key(key: str) -> DockerLabelKey:
 class StandardSimcoreDockerLabels(BaseModel):
     """
     Represents the standard label on oSparc created containers (not yet services)
-    In order to create this object in code, please use construct() method!
+    In order to create this object in code, please use model_construct() method!
     """
 
     user_id: UserID = Field(..., alias=f"{_SIMCORE_RUNTIME_DOCKER_LABEL_PREFIX}user-id")  # type: ignore[literal-required]

packages/postgres-database/tests/test_utils_projects.py

@@ -12,7 +12,7 @@
 from aiopg.sa.connection import SAConnection
 from aiopg.sa.result import RowProxy
 from faker import Faker
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 from simcore_postgres_database.models.projects import projects
 from simcore_postgres_database.utils_projects import (
     DBProjectNotFoundError,
@@ -69,7 +69,7 @@ async def test_get_project_trashed_at_column_can_be_converted_to_datetime(
 
         row = result.fetchone()
 
-    trashed_at = parse_obj_as(datetime | None, row.trashed_at)
+    trashed_at = TypeAdapter(datetime | None).validate_python(row.trashed_at)
     assert trashed_at == expected
 
 

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/agent/volumes.py

@@ -4,7 +4,7 @@
 
 from models_library.projects_nodes_io import NodeID
 from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace
-from pydantic import NonNegativeInt, parse_obj_as
+from pydantic import NonNegativeInt, TypeAdapter
 from servicelib.logging_utils import log_decorator
 from servicelib.rabbitmq import RabbitMQRPCClient
 
@@ -29,7 +29,9 @@ async def remove_volumes_without_backup_for_service(
                 "swarm_stack_name": swarm_stack_name,
             }
         ),
-        parse_obj_as(RPCMethodName, "remove_volumes_without_backup_for_service"),
+        TypeAdapter(RPCMethodName).validate_python(
+            "remove_volumes_without_backup_for_service"
+        ),
         node_id=node_id,
         timeout_s=_REQUEST_TIMEOUT,
     )
@@ -51,7 +53,9 @@ async def backup_and_remove_volumes_for_all_services(
                 "swarm_stack_name": swarm_stack_name,
             }
         ),
-        parse_obj_as(RPCMethodName, "backup_and_remove_volumes_for_all_services"),
+        TypeAdapter(RPCMethodName).validate_python(
+            "backup_and_remove_volumes_for_all_services"
+        ),
         timeout_s=_REQUEST_TIMEOUT,
     )
     assert result is None  # nosec

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/dynamic_sidecar/disk_usage.py

@@ -1,14 +1,20 @@
 import logging
+from typing import Final
 
 from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage
 from models_library.projects_nodes_io import NodeID
 from models_library.rabbitmq_basic_types import RPCMethodName, RPCNamespace
-from pydantic import parse_obj_as
-from servicelib.logging_utils import log_decorator
-from servicelib.rabbitmq import RabbitMQRPCClient
+from pydantic import TypeAdapter
+
+from ....logging_utils import log_decorator
+from ... import RabbitMQRPCClient
 
 _logger = logging.getLogger(__name__)
 
+_UPDATE_DISK_USAGE: Final[RPCMethodName] = TypeAdapter(RPCMethodName).validate_python(
+    "update_disk_usage"
+)
+
 
 @log_decorator(_logger, level=logging.DEBUG)
 async def update_disk_usage(
@@ -21,6 +27,8 @@ async def update_disk_usage(
         {"service": "dy-sidecar", "node_id": f"{node_id}"}
     )
     result = await rabbitmq_rpc_client.request(
-        rpc_namespace, parse_obj_as(RPCMethodName, "update_disk_usage"), usage=usage
+        rpc_namespace,
+        _UPDATE_DISK_USAGE,
+        usage=usage,
     )
     assert result is None  # nosec

packages/service-library/src/servicelib/rabbitmq/rpc_interfaces/resource_usage_tracker/service_runs.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Final, cast
+from typing import Final
 
 from models_library.api_schemas_resource_usage_tracker import (
     RESOURCE_USAGE_TRACKER_RPC_NAMESPACE,
@@ -115,5 +115,5 @@ async def export_service_runs(
         filters=filters,
         timeout_s=_DEFAULT_TIMEOUT_S,
     )
-    assert cast(AnyUrl, isinstance(result, AnyUrl))  # nosec
+    assert isinstance(result, AnyUrl)  # nosec
     return result

packages/simcore-sdk/src/simcore_sdk/node_ports_common/aws_s3_cli_utils.py

@@ -1,7 +1,7 @@
 import logging
 import re
 
-from pydantic import ByteSize, parse_obj_as
+from pydantic import ByteSize, TypeAdapter
 from servicelib.logging_utils import log_catch
 from servicelib.progress_bar import ProgressBarData
 
@@ -35,5 +35,5 @@ async def __call__(self, logs: str) -> None:
         _logger.debug("received logs: %s", logs)
         with log_catch(_logger, reraise=False):
             if _size := _parse_size(logs):
-                _bytes = parse_obj_as(ByteSize, _size)
+                _bytes = TypeAdapter(ByteSize).validate_python(_size)
                 await self.progress_bar.set_(_bytes)

packages/simcore-sdk/src/simcore_sdk/node_ports_v2/nodeports_v2.py

@@ -1,6 +1,7 @@
 import logging
+import traceback
 from abc import ABC, abstractmethod
-from asyncio import CancelledError
+from asyncio import CancelledError, Task
 from collections.abc import Callable, Coroutine
 from pathlib import Path
 from typing import Any
@@ -12,6 +13,7 @@
 from models_library.services_types import ServicePortKey
 from models_library.users import UserID
 from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from pydantic_core import InitErrorDetails
 from servicelib.progress_bar import ProgressBarData
 from servicelib.utils import logged_gather
 from settings_library.aws_s3_cli import AwsS3CliSettings
@@ -28,6 +30,28 @@
 log = logging.getLogger(__name__)
 
 
+#  -> @GitHK this looks very dangerous, using a lot of protected stuff, just checking the number of ignores shows it's a bad idea...
+def _format_error(task: Task) -> str:
+    # pylint:disable=protected-access
+    assert task._exception  # nosec  # noqa: SLF001
+    error_list = traceback.format_exception(
+        type(task._exception),  # noqa: SLF001
+        task._exception,  # noqa: SLF001
+        task._exception.__traceback__,  # noqa: SLF001
+    )
+    return "\n".join(error_list)
+
+
+def _get_error_details(task: Task, port_key: str) -> InitErrorDetails:
+    # pylint:disable=protected-access
+    return InitErrorDetails(
+        type="value_error",
+        loc=(f"{port_key}",),
+        input=_format_error(task),
+        ctx={"error": task._exception},  # noqa: SLF001
+    )
+
+
 class OutputsCallbacks(ABC):
     @abstractmethod
     async def aborted(self, key: ServicePortKey) -> None:
@@ -72,9 +96,9 @@ def __init__(self, **data: Any):
 
         # let's pass ourselves down
         for input_key in self.internal_inputs:
-            self.internal_inputs[input_key]._node_ports = self
+            self.internal_inputs[input_key]._node_ports = self  # noqa: SLF001
         for output_key in self.internal_outputs:
-            self.internal_outputs[output_key]._node_ports = self
+            self.internal_outputs[output_key]._node_ports = self  # noqa: SLF001
 
     @property
     async def inputs(self) -> InputsList:
@@ -132,10 +156,11 @@ async def set(
 
     async def set_file_by_keymap(self, item_value: Path) -> None:
         for output in (await self.outputs).values():
-            if is_file_type(output.property_type) and output.file_to_key_map:
-                if item_value.name in output.file_to_key_map:
-                    await output.set(item_value)
-                    return
+            if (is_file_type(output.property_type) and output.file_to_key_map) and (
+                item_value.name in output.file_to_key_map
+            ):
+                await output.set(item_value)
+                return
         raise PortNotFound(msg=f"output port for item {item_value} not found")
 
     async def _node_ports_creator_cb(self, node_uuid: NodeIDStr) -> type["Nodeports"]:
@@ -152,9 +177,9 @@ async def _auto_update_from_db(self) -> None:
         # let's pass ourselves down
         # pylint: disable=protected-access
         for input_key in self.internal_inputs:
-            self.internal_inputs[input_key]._node_ports = self
+            self.internal_inputs[input_key]._node_ports = self  # noqa: SLF001
         for output_key in self.internal_outputs:
-            self.internal_outputs[output_key]._node_ports = self
+            self.internal_outputs[output_key]._node_ports = self  # noqa: SLF001
 
     async def set_multiple(
         self,

packages/simcore-sdk/src/simcore_sdk/node_ports_v2/port_utils.py

@@ -8,8 +8,7 @@
 from models_library.basic_types import IDStr, SHA256Str
 from models_library.services_types import FileName, ServicePortKey
 from models_library.users import UserID
-from pydantic import AnyUrl, ByteSize
-from pydantic.tools import parse_obj_as
+from pydantic import AnyUrl, ByteSize, TypeAdapter
 from servicelib.progress_bar import ProgressBarData
 from settings_library.aws_s3_cli import AwsS3CliSettings
 from settings_library.r_clone import RCloneSettings
@@ -64,6 +63,7 @@ async def get_value_from_link(
         file_name = other_value.name
         # move the file to the right final location
         # if a file alias is present use it
+
         if file_to_key_map:
             file_name = next(iter(file_to_key_map))
 
@@ -101,7 +101,7 @@ async def get_download_link_from_storage(
 
     # could raise ValidationError but will never do it since
     assert isinstance(link, URL)  # nosec
-    url: AnyUrl = parse_obj_as(AnyUrl, f"{link}")
+    url: AnyUrl = TypeAdapter(AnyUrl).validate_python(f"{link}")
     return url
 
 
@@ -123,7 +123,7 @@ async def get_download_link_from_storage_overload(
         s3_object=s3_object,
         link_type=link_type,
     )
-    url: AnyUrl = parse_obj_as(AnyUrl, f"{link}")
+    url: AnyUrl = TypeAdapter(AnyUrl).validate_python(f"{link}")
     return url
 
 

packages/simcore-sdk/tests/integration/conftest.py

@@ -17,7 +17,7 @@
 from models_library.generics import Envelope
 from models_library.projects_nodes_io import LocationID, NodeIDStr, SimcoreS3FileID
 from models_library.users import UserID
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 from pytest_simcore.helpers.faker_factories import random_project, random_user
 from settings_library.aws_s3_cli import AwsS3CliSettings
 from settings_library.r_clone import RCloneSettings, S3Provider
@@ -94,7 +94,7 @@ def create_valid_file_uuid(
 ) -> Callable[[str, Path], SimcoreS3FileID]:
     def _create(key: str, file_path: Path) -> SimcoreS3FileID:
         clean_path = Path(f"{project_id}/{node_uuid}/{key}/{file_path.name}")
-        return parse_obj_as(SimcoreS3FileID, f"{clean_path}")
+        return TypeAdapter(SimcoreS3FileID).validate_python(f"{clean_path}")
 
     return _create