pydantic2 making ported services green (#6701)

Co-authored-by: Andrei Neagu <[email protected]>

Author: GitHK

packages/common-library/src/common_library/serialization.py

@@ -1,11 +1,10 @@
+import contextlib
 from datetime import timedelta
 from typing import Any
 
-from pydantic import BaseModel, SecretStr
+from pydantic import BaseModel, SecretStr, TypeAdapter, ValidationError
 from pydantic_core import Url
 
-from .pydantic_fields_extension import get_type
-
 
 def model_dump_with_secrets(
     settings_obj: BaseModel, *, show_secrets: bool, **pydantic_export_options
@@ -31,10 +30,11 @@ def model_dump_with_secrets(
             data[field_name] = str(field_data)
 
         elif isinstance(field_data, dict):
-            field_type = get_type(settings_obj.model_fields[field_name])
-            if issubclass(field_type, BaseModel):
+            possible_pydantic_model = settings_obj.model_fields[field_name].annotation
+            # NOTE: data could be a dict which does not represent a pydantic model or a union of models
+            with contextlib.suppress(AttributeError, ValidationError):
                 data[field_name] = model_dump_with_secrets(
-                    field_type.model_validate(field_data),
+                    TypeAdapter(possible_pydantic_model).validate_python(field_data),
                     show_secrets=show_secrets,
                     **pydantic_export_options,
                 )

packages/models-library/src/models_library/api_schemas_clusters_keeper/clusters.py

@@ -1,7 +1,7 @@
 import datetime
 from enum import auto
 
-from pydantic import AnyUrl, BaseModel
+from pydantic import AnyUrl, BaseModel, Field
 
 from ..clusters import ClusterAuthentication
 from ..users import UserID
@@ -17,7 +17,7 @@ class ClusterState(StrAutoEnum):
 
 class OnDemandCluster(BaseModel):
     endpoint: AnyUrl
-    authentication: ClusterAuthentication
+    authentication: ClusterAuthentication = Field(discriminator="type")
     state: ClusterState
     user_id: UserID
     wallet_id: WalletID | None

packages/models-library/src/models_library/api_schemas_directorv2/clusters.py

@@ -114,7 +114,7 @@ class ClusterDetailsGet(ClusterDetails):
 
 class ClusterCreate(BaseCluster):
     owner: GroupID | None = None  # type: ignore[assignment]
-    authentication: ExternalClusterAuthentication
+    authentication: ExternalClusterAuthentication = Field(discriminator="type")
     access_rights: dict[GroupID, ClusterAccessRights] = Field(
         alias="accessRights", default_factory=dict
     )
@@ -144,9 +144,9 @@ class ClusterCreate(BaseCluster):
                         "password": "somepassword",
                     },
                     "accessRights": {
-                        154: CLUSTER_ADMIN_RIGHTS,  # type: ignore[dict-item]
-                        12: CLUSTER_MANAGER_RIGHTS,  # type: ignore[dict-item]
-                        7899: CLUSTER_USER_RIGHTS,  # type: ignore[dict-item]
+                        154: CLUSTER_ADMIN_RIGHTS.model_dump(),  # type:ignore[dict-item]
+                        12: CLUSTER_MANAGER_RIGHTS.model_dump(),  # type:ignore[dict-item]
+                        7899: CLUSTER_USER_RIGHTS.model_dump(),  # type:ignore[dict-item]
                     },
                 },
             ]
@@ -174,7 +174,7 @@ class ClusterPatch(BaseCluster):
     owner: GroupID | None = None  # type: ignore[assignment]
     thumbnail: HttpUrl | None = None
     endpoint: AnyUrl | None = None  # type: ignore[assignment]
-    authentication: ExternalClusterAuthentication | None = None  # type: ignore[assignment]
+    authentication: ExternalClusterAuthentication | None = Field(None, discriminator="type")  # type: ignore[assignment]
     access_rights: dict[GroupID, ClusterAccessRights] | None = Field(  # type: ignore[assignment]
         default=None, alias="accessRights"
     )
@@ -190,9 +190,9 @@ class ClusterPatch(BaseCluster):
                 },
                 {
                     "accessRights": {
-                        154: CLUSTER_ADMIN_RIGHTS,  # type: ignore[dict-item]
-                        12: CLUSTER_MANAGER_RIGHTS,  # type: ignore[dict-item]
-                        7899: CLUSTER_USER_RIGHTS,  # type: ignore[dict-item]
+                        154: CLUSTER_ADMIN_RIGHTS.model_dump(),  # type:ignore[dict-item]
+                        12: CLUSTER_MANAGER_RIGHTS.model_dump(),  # type:ignore[dict-item]
+                        7899: CLUSTER_USER_RIGHTS.model_dump(),  # type:ignore[dict-item]
                     },
                 },
             ]
@@ -203,5 +203,7 @@ class ClusterPatch(BaseCluster):
 class ClusterPing(BaseModel):
     endpoint: AnyHttpUrl
     authentication: ClusterAuthentication = Field(
-        ..., description="Dask gateway authentication"
+        ...,
+        description="Dask gateway authentication",
+        discriminator="type",
     )

packages/models-library/src/models_library/clusters.py

@@ -140,7 +140,7 @@ class BaseCluster(BaseModel):
     )
     endpoint: AnyUrl
     authentication: ClusterAuthentication = Field(
-        ..., description="Dask gateway authentication"
+        ..., description="Dask gateway authentication", discriminator="type"
     )
     access_rights: dict[GroupID, ClusterAccessRights] = Field(default_factory=dict)
 

packages/models-library/src/models_library/projects.py

@@ -4,7 +4,7 @@
 
 from datetime import datetime
 from enum import Enum
-from typing import Any, Final, TypeAlias
+from typing import Annotated, Any, Final, TypeAlias
 from uuid import UUID
 
 from models_library.basic_types import ConstrainedStr
@@ -77,7 +77,7 @@ class BaseProjectModel(BaseModel):
     last_change_date: datetime = Field(...)
 
     # Pipeline of nodes (SEE projects_nodes.py)
-    workbench: NodesDict = Field(..., description="Project's pipeline")
+    workbench: Annotated[NodesDict, Field(..., description="Project's pipeline")]
 
     # validators
     _empty_thumbnail_is_none = field_validator("thumbnail", mode="before")(

packages/models-library/src/models_library/projects_state.py

@@ -3,6 +3,7 @@
 """
 
 from enum import Enum, unique
+from typing import Annotated
 
 from pydantic import (
     BaseModel,
@@ -126,7 +127,7 @@ class ProjectRunningState(BaseModel):
 
 
 class ProjectState(BaseModel):
-    locked: ProjectLocked = Field(..., description="The project lock state")
+    locked: Annotated[ProjectLocked, Field(..., description="The project lock state")]
     state: ProjectRunningState = Field(..., description="The project running state")
 
     model_config = ConfigDict(extra="forbid")

packages/pytest-simcore/src/pytest_simcore/helpers/httpx_calls_capture_parameters.py

@@ -1,6 +1,6 @@
-from typing import Literal
+from typing import Annotated, Literal
 
-from pydantic import field_validator, model_validator, ConfigDict, BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
 from .httpx_calls_capture_errors import OpenApiSpecError
 
@@ -94,7 +94,7 @@ class CapturedParameter(BaseModel):
     in_: Literal["path", "header", "query"] = Field(..., alias="in")
     name: str
     required: bool
-    schema_: CapturedParameterSchema = Field(..., alias="schema")
+    schema_: Annotated[CapturedParameterSchema, Field(..., alias="schema")]
     response_value: str | None = (
         None  # attribute for storing the params value in a concrete response
     )

packages/service-library/src/servicelib/aiohttp/tracing.py

@@ -31,9 +31,7 @@
 except ImportError:
     HAS_BOTOCORE = False
 try:
-    from opentelemetry.instrumentation.aiopg import (  # type: ignore[import-not-found]
-        AiopgInstrumentor,
-    )
+    from opentelemetry.instrumentation.aiopg import AiopgInstrumentor
 
     HAS_AIOPG = True
 except ImportError:

packages/service-library/src/servicelib/fastapi/tracing.py

@@ -28,9 +28,7 @@
     HAS_ASYNCPG = False
 
 try:
-    from opentelemetry.instrumentation.aiopg import (  # type: ignore[import-not-found]
-        AiopgInstrumentor,
-    )
+    from opentelemetry.instrumentation.aiopg import AiopgInstrumentor
 
     HAS_AIOPG = True
 except ImportError:

services/api-server/src/simcore_service_api_server/core/settings.py

@@ -1,4 +1,5 @@
 from functools import cached_property
+from typing import Annotated
 
 from models_library.basic_types import BootModeEnum, LogLevel
 from pydantic import (
@@ -87,9 +88,10 @@ class ApplicationSettings(BasicSettings):
     # DOCKER BOOT
     SC_BOOT_MODE: BootModeEnum | None = None
 
-    API_SERVER_POSTGRES: PostgresSettings | None = Field(
-        json_schema_extra={"auto_default_from_env": True}
-    )
+    API_SERVER_POSTGRES: Annotated[
+        PostgresSettings | None,
+        Field(json_schema_extra={"auto_default_from_env": True}),
+    ]
 
     API_SERVER_RABBITMQ: RabbitSettings | None = Field(
         json_schema_extra={"auto_default_from_env": True},