Merge branch 'pydantic_v2_migration_do_not_squash_updates' into is4481/upgrade-director-v2-service

Author: giancarloromeo

packages/common-library/src/common_library/pydantic_validators.py

@@ -1,10 +1,35 @@
 import datetime
+import re
 import warnings
 from datetime import timedelta
 
 from pydantic import TypeAdapter, field_validator
 
 
+def _validate_legacy_timedelta_str(time_str: str | timedelta) -> str | timedelta:
+    if not isinstance(time_str, str):
+        return time_str
+
+    # Match the format [-][DD ][HH:MM]SS[.ffffff]
+    match = re.match(
+        r"^(?P<sign>-)?(?:(?P<days>\d+)\s)?(?:(?P<hours>\d+):)?(?:(?P<minutes>\d+):)?(?P<seconds>\d+)(?P<fraction>\.\d+)?$",
+        time_str,
+    )
+    if not match:
+        return time_str
+
+    # Extract components with defaults if not present
+    sign = match.group("sign") or ""
+    days = match.group("days") or "0"
+    hours = match.group("hours") or "0"
+    minutes = match.group("minutes") or "0"
+    seconds = match.group("seconds")
+    fraction = match.group("fraction") or ""
+
+    # Convert to the format [-][DD]D[,][HH:MM:]SS[.ffffff]
+    return f"{sign}{int(days)}D,{int(hours):02}:{int(minutes):02}:{seconds}{fraction}"
+
+
 def validate_numeric_string_as_timedelta(field: str):
     """Transforms a float/int number into a valid datetime as it used to work in the past"""
 
@@ -29,7 +54,7 @@ def _numeric_string_as_timedelta(
                 return converted_value
             except ValueError:
                 # returns format like "1:00:00"
-                return v
+                return _validate_legacy_timedelta_str(v)
         return v
 
     return field_validator(field, mode="before")(_numeric_string_as_timedelta)

packages/common-library/tests/test_pydantic_validators.py

@@ -1,13 +1,45 @@
 from datetime import timedelta
+from typing import Annotated
 
 import pytest
-from common_library.pydantic_validators import validate_numeric_string_as_timedelta
+from common_library.pydantic_validators import (
+    validate_legacy_timedelta_str,
+    validate_numeric_string_as_timedelta,
+)
 from faker import Faker
-from pydantic import Field
+from pydantic import BeforeValidator, Field
 from pydantic_settings import BaseSettings, SettingsConfigDict
 from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict
 
 
+def test_validate_legacy_timedelta(monkeypatch: pytest.MonkeyPatch, faker: Faker):
+    class Settings(BaseSettings):
+        APP_NAME: str
+        REQUEST_TIMEOUT: Annotated[
+            timedelta, BeforeValidator(validate_legacy_timedelta_str)
+        ] = Field(default=timedelta(hours=1))
+
+        model_config = SettingsConfigDict()
+
+    app_name = faker.pystr()
+    env_vars: dict[str, str | bool] = {"APP_NAME": app_name}
+
+    # without timedelta
+    setenvs_from_dict(monkeypatch, env_vars)
+    settings = Settings()
+    print(settings.model_dump())
+    assert app_name == settings.APP_NAME
+    assert timedelta(hours=1) == settings.REQUEST_TIMEOUT
+
+    # with timedelta in seconds
+    env_vars["REQUEST_TIMEOUT"] = "2 1:10:00"
+    setenvs_from_dict(monkeypatch, env_vars)
+    settings = Settings()
+    print(settings.model_dump())
+    assert app_name == settings.APP_NAME
+    assert timedelta(days=2, hours=1, minutes=10) == settings.REQUEST_TIMEOUT
+
+
 def test_validate_timedelta_in_legacy_mode(
     monkeypatch: pytest.MonkeyPatch, faker: Faker
 ):

packages/models-library/src/models_library/api_schemas_directorv2/clusters.py

@@ -96,7 +96,7 @@ class ClusterGet(Cluster):
         alias="accessRights", default_factory=dict
     )
 
-    model_config = ConfigDict(populate_by_name=True)
+    model_config = ConfigDict(extra="allow", populate_by_name=True)
 
     @model_validator(mode="before")
     @classmethod
@@ -112,7 +112,7 @@ class ClusterDetailsGet(ClusterDetails):
 
 
 class ClusterCreate(BaseCluster):
-    owner: GroupID | None  # type: ignore[assignment]
+    owner: GroupID | None
     authentication: ExternalClusterAuthentication
     access_rights: dict[GroupID, ClusterAccessRights] = Field(
         alias="accessRights", default_factory=dict

packages/models-library/src/models_library/api_schemas_webserver/auth.py

@@ -83,9 +83,10 @@ class ApiKeyGet(BaseModel):
     api_secret: str
 
     model_config = ConfigDict(
+        from_attributes=True,
         json_schema_extra={
             "examples": [
                 {"display_name": "myapi", "api_key": "key", "api_secret": "secret"},
             ]
-        }
+        },
     )

packages/models-library/src/models_library/api_schemas_webserver/clusters.py

@@ -8,7 +8,7 @@
 class ClusterPathParams(BaseModel):
     cluster_id: ClusterID
     model_config = ConfigDict(
-        populate_by_name=True, 
+        populate_by_name=True,
         extra="forbid",
     )
 

packages/models-library/src/models_library/api_schemas_webserver/folders_v2.py

@@ -45,7 +45,7 @@ class CreateFolderBodyParams(InputSchema):
 
 class PutFolderBodyParams(InputSchema):
     name: IDStr
-    parent_folder_id: FolderID | None
+    parent_folder_id: FolderID | None = None
     model_config = ConfigDict(extra="forbid")
 
     _null_or_none_str_to_none_validator = field_validator(

packages/models-library/src/models_library/api_schemas_webserver/groups.py

@@ -1,6 +1,7 @@
 from contextlib import suppress
 
 from pydantic import (
+    AnyHttpUrl,
     AnyUrl,
     BaseModel,
     ConfigDict,
@@ -90,7 +91,7 @@ def _sanitize_legacy_data(cls, v):
         if v:
             # Enforces null if thumbnail is not valid URL or empty
             with suppress(ValidationError):
-                return TypeAdapter(AnyUrl).validate_python(v)
+                return TypeAdapter(AnyHttpUrl).validate_python(v)
         return None
 
 
@@ -143,12 +144,14 @@ class AllUsersGroups(BaseModel):
 
 
 class GroupUserGet(BaseModel):
-    id: str | None = Field(None, description="the user id")
+    id: str | None = Field(None, description="the user id", coerce_numbers_to_str=True)
     login: LowerCaseEmailStr | None = Field(None, description="the user login email")
     first_name: str | None = Field(None, description="the user first name")
     last_name: str | None = Field(None, description="the user last name")
     gravatar_id: str | None = Field(None, description="the user gravatar id hash")
-    gid: str | None = Field(None, description="the user primary gid")
+    gid: str | None = Field(
+        None, description="the user primary gid", coerce_numbers_to_str=True
+    )
     access_rights: GroupAccessRights = Field(..., alias="accessRights")
 
     model_config = ConfigDict(

packages/models-library/src/models_library/api_schemas_webserver/projects.py

@@ -6,9 +6,11 @@
 """
 
 from datetime import datetime
-from typing import Any, Literal, TypeAlias
+from typing import Annotated, Any, Literal, TypeAlias
 
-from pydantic import ConfigDict, Field, HttpUrl, field_validator
+from models_library.folders import FolderID
+from models_library.workspaces import WorkspaceID
+from pydantic import BeforeValidator, ConfigDict, Field, HttpUrl, field_validator
 
 from ..api_schemas_long_running_tasks.tasks import TaskGet
 from ..basic_types import LongTruncatedStr, ShortTruncatedStr
@@ -105,7 +107,7 @@ class ProjectReplace(InputSchema):
     uuid: ProjectID
     name: ShortTruncatedStr
     description: LongTruncatedStr
-    thumbnail: HttpUrl | None
+    thumbnail: Annotated[HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator)] = Field(default=None)
     creation_date: DateTimeStr
     last_change_date: DateTimeStr
     workbench: NodesDict
@@ -121,25 +123,17 @@ class ProjectReplace(InputSchema):
         default_factory=dict, json_schema_extra={"default": {}}
     )
 
-    _empty_is_none = field_validator("thumbnail", mode="before")(
-        empty_str_to_none_pre_validator
-    )
-
 
 class ProjectPatch(InputSchema):
     name: ShortTruncatedStr | None = Field(default=None)
     description: LongTruncatedStr | None = Field(default=None)
-    thumbnail: HttpUrl | None = Field(default=None)
+    thumbnail: Annotated[HttpUrl | None, BeforeValidator(empty_str_to_none_pre_validator)] = Field(default=None)
     access_rights: dict[GroupIDStr, AccessRights] | None = Field(default=None)
     classifiers: list[ClassifierID] | None = Field(default=None)
     dev: dict | None = Field(default=None)
     ui: StudyUI | None = Field(default=None)
     quality: dict[str, Any] | None = Field(default=None)
-
-    _empty_is_none = field_validator("thumbnail", mode="before")(
-        empty_str_to_none_pre_validator
-    )
-
+    
 
 __all__: tuple[str, ...] = (
     "EmptyModel",

packages/models-library/src/models_library/api_schemas_webserver/resource_usage.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 from decimal import Decimal
 
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field
 
 from ..projects import ProjectID
 from ..projects_nodes_io import NodeID
@@ -48,7 +48,7 @@ class ServiceRunGet(
 class PricingUnitGet(OutputSchema):
     pricing_unit_id: PricingUnitId
     unit_name: str
-    unit_extra_info: dict
+    unit_extra_info: UnitExtraInfo
     current_cost_per_unit: Decimal
     default: bool
 
@@ -131,7 +131,7 @@ class UpdatePricingUnitBodyParams(InputSchema):
     unit_extra_info: UnitExtraInfo
     default: bool
     specific_info: SpecificInfo
-    pricing_unit_cost_update: PricingUnitCostUpdate | None
+    pricing_unit_cost_update: PricingUnitCostUpdate | None = Field(default=None)
 
     model_config = ConfigDict(
         str_strip_whitespace=True,

packages/models-library/src/models_library/api_schemas_webserver/wallets.py

@@ -2,7 +2,7 @@
 from decimal import Decimal
 from typing import Annotated, Literal, TypeAlias
 
-from pydantic import ConfigDict, Field, HttpUrl, PlainSerializer, field_validator
+from pydantic import ConfigDict, Field, HttpUrl, PlainSerializer, ValidationInfo, field_validator
 
 from ..basic_types import AmountDecimal, IDStr, NonNegativeDecimal
 from ..users import GroupID
@@ -20,7 +20,10 @@ class WalletGet(OutputSchema):
     created: datetime
     modified: datetime
 
-    model_config = ConfigDict(frozen=False)
+    model_config = ConfigDict(
+        from_attributes=True,
+        frozen=False
+    )
 
 
 class WalletGetWithAvailableCredits(WalletGet):
@@ -139,6 +142,7 @@ class PaymentMethodGet(OutputSchema):
     )
 
     model_config = ConfigDict(
+        frozen=False,
         json_schema_extra={
             "examples": [
                 {
@@ -200,8 +204,8 @@ class ReplaceWalletAutoRecharge(InputSchema):
 
     @field_validator("monthly_limit_in_usd")
     @classmethod
-    def _monthly_limit_greater_than_top_up(cls, v, values):
-        top_up = values["top_up_amount_in_usd"]
+    def _monthly_limit_greater_than_top_up(cls, v, info: ValidationInfo):
+        top_up = info.data["top_up_amount_in_usd"]
         if v is not None and v < top_up:
             msg = "Monthly limit ({v} USD) should be greater than top up amount ({top_up} USD)"
             raise ValueError(msg)