⬆️ Upgrade Director v2 service (Pydantic v2)

api/specs/director/schemas/scripts/create_node-meta-schema.py

@@ -15,7 +15,7 @@
 
 if __name__ == "__main__":
     with Path.open(CURRENT_DIR.parent / "node-meta-v0.0.1-pydantic.json", "w") as f:
-        schema = ServiceMetaDataPublished.schema_json()
+        schema = ServiceMetaDataPublished.model_json_schema()
         schema_without_ref = jsonref.loads(schema)
 
         json.dump(schema_without_ref, f, indent=2)

packages/common-library/src/common_library/serialization.py

@@ -1,11 +1,9 @@
 from datetime import timedelta
 from typing import Any
 
-from pydantic import BaseModel, SecretStr
+from pydantic import BaseModel, SecretStr, TypeAdapter
 from pydantic_core import Url
 
-from .pydantic_fields_extension import get_type
-
 
 def model_dump_with_secrets(
     settings_obj: BaseModel, *, show_secrets: bool, **pydantic_export_options
@@ -22,19 +20,18 @@ def model_dump_with_secrets(
             data[field_name] = field_data.total_seconds()
 
         elif isinstance(field_data, SecretStr):
-            if show_secrets:
-                data[field_name] = field_data.get_secret_value()
-            else:
-                data[field_name] = str(field_data)
+            data[field_name] = (
+                field_data.get_secret_value() if show_secrets else str(field_data)
+            )
 
         elif isinstance(field_data, Url):
             data[field_name] = str(field_data)
 
         elif isinstance(field_data, dict):
-            field_type = get_type(settings_obj.model_fields[field_name])
-            if issubclass(field_type, BaseModel):
+            field_type = settings_obj.model_fields[field_name].annotation
+            if field_type:
                 data[field_name] = model_dump_with_secrets(
-                    field_type.model_validate(field_data),
+                    TypeAdapter(field_type).validate_python(field_data),
                     show_secrets=show_secrets,
                     **pydantic_export_options,
                 )

packages/common-library/tests/test_serialization.py

@@ -4,22 +4,31 @@
 
 
 class Credentials(BaseModel):
-    USERNAME: str | None = None
-    PASSWORD: SecretStr | None = None
+    username: str
+    password: SecretStr
+
+
+class Access(BaseModel):
+    credentials: Credentials
 
 
 @pytest.mark.parametrize(
     "expected,show_secrets",
     [
         (
-            {"USERNAME": "DeepThought", "PASSWORD": "42"},
+            {"credentials": {"username": "DeepThought", "password": "42"}},
             True,
         ),
         (
-            {"USERNAME": "DeepThought", "PASSWORD": "**********"},
+            {"credentials": {"username": "DeepThought", "password": "**********"}},
             False,  # hide secrets
         ),
     ],
 )
 def test_model_dump_with_secrets(expected: dict, show_secrets: bool):
-    assert expected == model_dump_with_secrets(Credentials(USERNAME="DeepThought", PASSWORD=SecretStr("42")), show_secrets=show_secrets)
+    assert expected == model_dump_with_secrets(
+        Access(
+            credentials=Credentials(username="DeepThought", password=SecretStr("42"))
+        ),
+        show_secrets=show_secrets,
+    )

packages/models-library/src/models_library/aiodocker_api.py

@@ -11,12 +11,13 @@
 
 
 class AioDockerContainerSpec(ContainerSpec):
-    Env: dict[str, str | None] | None = Field(
+    env: dict[str, str | None] | None = Field(  # type: ignore[assignment]
         default=None,
-        description="aiodocker expects here a dictionary and re-convert it back internally`.\n",
+        alias="Env",
+        description="aiodocker expects here a dictionary and re-convert it back internally",
     )
 
-    @field_validator("Env", mode="before")
+    @field_validator("env", mode="before")
     @classmethod
     def convert_list_to_dict(cls, v):
         if v is not None and isinstance(v, list):
@@ -33,25 +34,22 @@ def convert_list_to_dict(cls, v):
 class AioDockerResources1(Resources1):
     # NOTE: The Docker REST API documentation is wrong!!!
     # Do not set that back to singular Reservation.
-    Reservation: ResourceObject | None = Field(
+    reservation: ResourceObject | None = Field(
         None, description="Define resources reservation.", alias="Reservations"
     )
 
     model_config = ConfigDict(populate_by_name=True)
 
 
 class AioDockerTaskSpec(TaskSpec):
-    ContainerSpec: AioDockerContainerSpec | None = Field(
-        None,
+    container_spec: AioDockerContainerSpec | None = Field(
+        default=None, alias="ContainerSpec"
     )
 
-    Resources: AioDockerResources1 | None = Field(
-        None,
-        description="Resource requirements which apply to each individual container created\nas part of the service.\n",
-    )
+    resources: AioDockerResources1 | None = Field(default=None, alias="Resources")
 
 
 class AioDockerServiceSpec(ServiceSpec):
-    TaskTemplate: AioDockerTaskSpec | None = None
+    task_template: AioDockerTaskSpec | None = Field(default=None, alias="TaskTemplate")
 
     model_config = ConfigDict(populate_by_name=True, alias_generator=camel_to_snake)

packages/models-library/src/models_library/api_schemas_directorv2/clusters.py

@@ -1,4 +1,4 @@
-from typing import TypeAlias
+from typing import Any, TypeAlias
 
 from pydantic import (
     AnyHttpUrl,
@@ -48,13 +48,12 @@ class WorkerMetrics(BaseModel):
 class UsedResources(DictModel[str, NonNegativeFloat]):
     @model_validator(mode="before")
     @classmethod
-    def ensure_negative_value_is_zero(cls, values):
+    def ensure_negative_value_is_zero(cls, values: dict[str, Any]):
         # dasks adds/remove resource values and sometimes
         # they end up being negative instead of 0
-        if v := values.get("__root__", {}):
-            for res_key, res_value in v.items():
-                if res_value < 0:
-                    v[res_key] = 0
+        for res_key, res_value in values.items():
+            if res_value < 0:
+                values[res_key] = 0
         return values
 
 
@@ -126,6 +125,7 @@ class ClusterCreate(BaseCluster):
                     "name": "My awesome cluster",
                     "type": ClusterTypeInModel.ON_PREMISE,
                     "endpoint": "https://registry.osparc-development.fake.dev",
+                    "owner": None,
                     "authentication": {
                         "type": "simple",
                         "username": "someuser",
@@ -168,15 +168,16 @@ def set_default_thumbnail_if_empty(cls, v, info: ValidationInfo):
 
 
 class ClusterPatch(BaseCluster):
-    name: str | None = None  # type: ignore[assignment]
-    description: str | None = None
-    type: ClusterTypeInModel | None = None  # type: ignore[assignment]
-    owner: GroupID | None = None  # type: ignore[assignment]
-    thumbnail: HttpUrl | None = None
-    endpoint: AnyUrl | None = None  # type: ignore[assignment]
-    authentication: ExternalClusterAuthentication | None = None  # type: ignore[assignment]
+    name: str | None = Field(default=None)  # type: ignore[assignment]
+    description: str | None = Field(default=None)
+    type: ClusterTypeInModel | None = Field(default=None)  # type: ignore[assignment]
+    owner: GroupID | None = Field(default=None)  # type: ignore[assignment]
+    thumbnail: HttpUrl | None = Field(default=None)
+    endpoint: AnyUrl | None = Field(default=None)  # type: ignore[assignment]
+    authentication: ExternalClusterAuthentication | None = Field(default=None)  # type: ignore[assignment]
     access_rights: dict[GroupID, ClusterAccessRights] | None = Field(  # type: ignore[assignment]
-        default=None, alias="accessRights"
+        default=None,
+        alias="accessRights"
     )
 
     model_config = ConfigDict(

packages/models-library/src/models_library/api_schemas_directorv2/comp_tasks.py

@@ -1,7 +1,15 @@
 from typing import Any, TypeAlias
 
 from models_library.basic_types import IDStr
-from pydantic import AnyHttpUrl, AnyUrl, BaseModel, ConfigDict, Field, field_validator
+from pydantic import (
+    AnyHttpUrl,
+    AnyUrl,
+    BaseModel,
+    ConfigDict,
+    Field,
+    ValidationInfo,
+    field_validator,
+)
 
 from ..clusters import ClusterID
 from ..projects import ProjectID
@@ -63,16 +71,18 @@ class ComputationCreate(BaseModel):
 
     @field_validator("product_name")
     @classmethod
-    def ensure_product_name_defined_if_computation_starts(cls, v, values):
-        if "start_pipeline" in values and values["start_pipeline"] and v is None:
+    def _ensure_product_name_defined_if_computation_starts(
+        cls, v, info: ValidationInfo
+    ):
+        if info.data.get("start_pipeline") and v is None:
             msg = "product_name must be set if computation shall start!"
             raise ValueError(msg)
         return v
 
     @field_validator("use_on_demand_clusters")
     @classmethod
-    def ensure_expected_options(cls, v, values):
-        if v is True and ("cluster_id" in values and values["cluster_id"] is not None):
+    def _ensure_expected_options(cls, v, info: ValidationInfo):
+        if v and info.data.get("cluster_id") is not None:
             msg = "cluster_id cannot be set if use_on_demand_clusters is set"
             raise ValueError(msg)
         return v

packages/models-library/src/models_library/clusters.py

@@ -221,14 +221,14 @@ class Cluster(BaseCluster):
         },
     )
 
-    @model_validator(mode="before")
+    @model_validator(mode="after")
     @classmethod
-    def check_owner_has_access_rights(cls, values):
-        is_default_cluster = bool(values["id"] == DEFAULT_CLUSTER_ID)
-        owner_gid = values["owner"]
+    def _check_owner_has_access_rights(cls, values):
+        is_default_cluster = bool(values.id == DEFAULT_CLUSTER_ID)
+        owner_gid = values.owner
 
         # check owner is in the access rights, if not add it
-        access_rights = values.get("access_rights", values.get("accessRights", {}))
+        access_rights = values.access_rights or {}
         if owner_gid not in access_rights:
             access_rights[owner_gid] = (
                 CLUSTER_USER_RIGHTS if is_default_cluster else CLUSTER_ADMIN_RIGHTS
@@ -239,5 +239,5 @@ def check_owner_has_access_rights(cls, values):
         ):
             msg = f"the cluster owner access rights are incorrectly set: {access_rights[owner_gid]}"
             raise ValueError(msg)
-        values["access_rights"] = access_rights
+        values.access_rights = access_rights
         return values

packages/models-library/src/models_library/docker.py

@@ -37,13 +37,7 @@ def from_key(cls, key: str) -> "DockerLabelKey":
     str, StringConstraints(pattern=DOCKER_GENERIC_TAG_KEY_RE)
 ]
 
-
-class DockerPlacementConstraint(ConstrainedStr):
-    strip_whitespace = True
-    regex = re.compile(
-        r"^(?!-)(?![.])(?!.*--)(?!.*[.][.])[a-zA-Z0-9.-]*(?<!-)(?<![.])(!=|==)[a-zA-Z0-9_. -]*$"
-    )
-
+DockerPlacementConstraint: TypeAlias = Annotated[str, StringConstraints(strip_whitespace = True, pattern = re.compile(r"^(?!-)(?![.])(?!.*--)(?!.*[.][.])[a-zA-Z0-9.-]*(?<!-)(?<![.])(!=|==)[a-zA-Z0-9_. -]*$"))]
 
 _SIMCORE_RUNTIME_DOCKER_LABEL_PREFIX: Final[str] = "io.simcore.runtime."
 _BACKWARDS_COMPATIBILITY_SIMCORE_RUNTIME_DOCKER_LABELS_MAP: Final[dict[str, str]] = {

packages/models-library/src/models_library/errors.py

@@ -1,4 +1,6 @@
-from typing import Any, TypedDict
+from typing import Any
+from typing_extensions import TypedDict
+
 
 Loc = tuple[int | str, ...]