WIP: Upgrade director-v2 service (Pydantic v2)

packages/models-library/src/models_library/errors.py

@@ -1,4 +1,6 @@
-from typing import Any, TypedDict
+from typing import Any
+
+from typing_extensions import TypedDict
 
 Loc = tuple[int | str, ...]
 

services/director-v2/requirements/_test.txt

@@ -16,7 +16,7 @@ aiohttp==3.9.5
     #   -c requirements/_base.txt
     #   aiobotocore
     #   dask-gateway-server
-aioitertools==0.12.0
+aioitertools==0.11.0
     # via aiobotocore
 aiormq==6.8.0
     # via
@@ -43,7 +43,7 @@ attrs==23.2.0
     #   -c requirements/_base.txt
     #   aiohttp
     #   pytest-docker
-bokeh==3.5.2
+bokeh==3.5.1
     # via dask
 boto3==1.34.131
     # via aiobotocore
@@ -59,9 +59,9 @@ certifi==2024.2.2
     #   httpcore
     #   httpx
     #   requests
-cffi==1.17.1
+cffi==1.17.0
     # via cryptography
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via
     #   -c requirements/_base.txt
     #   requests
@@ -77,11 +77,11 @@ cloudpickle==3.0.0
     #   distributed
 colorlog==6.8.2
     # via dask-gateway-server
-contourpy==1.3.0
+contourpy==1.2.1
     # via bokeh
 coverage==7.6.1
     # via pytest-cov
-cryptography==43.0.1
+cryptography==43.0.0
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   dask-gateway-server
@@ -100,7 +100,7 @@ docker==7.1.0
     # via -r requirements/_test.in
 execnet==2.1.1
     # via pytest-xdist
-faker==29.0.0
+faker==27.0.0
     # via -r requirements/_test.in
 flaky==3.8.1
     # via -r requirements/_test.in
@@ -181,7 +181,7 @@ multidict==6.0.5
     #   aiohttp
     #   async-asgi-testclient
     #   yarl
-mypy==1.12.0
+mypy==1.11.1
     # via sqlalchemy
 mypy-extensions==1.0.0
     # via mypy
@@ -203,7 +203,7 @@ pamqp==3.3.0
     # via
     #   -c requirements/_base.txt
     #   aiormq
-pandas==2.2.3
+pandas==2.2.2
     # via bokeh
 partd==1.4.2
     # via
@@ -221,7 +221,7 @@ psutil==6.0.0
     #   distributed
 pycparser==2.22
     # via cffi
-pytest==8.3.3
+pytest==8.3.2
     # via
     #   -r requirements/_test.in
     #   pytest-asyncio
@@ -252,7 +252,7 @@ python-dateutil==2.9.0.post0
     #   botocore
     #   faker
     #   pandas
-pytz==2024.2
+pytz==2024.1
     # via pandas
 pyyaml==6.0.1
     # via
@@ -310,19 +310,19 @@ tornado==6.4
     #   distributed
 traitlets==5.14.3
     # via dask-gateway-server
-types-networkx==3.2.1.20240918
+types-networkx==3.2.1.20240820
     # via -r requirements/_test.in
 types-psycopg2==2.9.21.20240819
     # via -r requirements/_test.in
-types-pyyaml==6.0.12.20240917
+types-pyyaml==6.0.12.20240808
     # via -r requirements/_test.in
 typing-extensions==4.11.0
     # via
     #   -c requirements/_base.txt
     #   alembic
     #   mypy
     #   sqlalchemy2-stubs
-tzdata==2024.2
+tzdata==2024.1
     # via pandas
 urllib3==2.2.1
     # via
@@ -336,7 +336,7 @@ wrapt==1.16.0
     # via
     #   -c requirements/_base.txt
     #   aiobotocore
-xyzservices==2024.9.0
+xyzservices==2024.6.0
     # via bokeh
 yarl==1.9.4
     # via

services/director-v2/requirements/_tools.txt

@@ -1,8 +1,8 @@
-astroid==3.3.4
+astroid==3.2.4
     # via pylint
 black==24.8.0
     # via -r requirements/../../../requirements/devenv.txt
-build==1.2.2
+build==1.2.1
     # via pip-tools
 bump2version==1.0.1
     # via -r requirements/../../../requirements/devenv.txt
@@ -18,17 +18,17 @@ dill==0.3.8
     # via pylint
 distlib==0.3.8
     # via virtualenv
-filelock==3.16.1
+filelock==3.15.4
     # via virtualenv
-identify==2.6.1
+identify==2.6.0
     # via pre-commit
 isort==5.13.2
     # via
     #   -r requirements/../../../requirements/devenv.txt
     #   pylint
 mccabe==0.7.0
     # via pylint
-mypy==1.12.0
+mypy==1.11.1
     # via
     #   -c requirements/_test.txt
     #   -r requirements/../../../requirements/devenv.txt
@@ -51,14 +51,14 @@ pip==24.2
     # via pip-tools
 pip-tools==7.4.1
     # via -r requirements/../../../requirements/devenv.txt
-platformdirs==4.3.6
+platformdirs==4.2.2
     # via
     #   black
     #   pylint
     #   virtualenv
 pre-commit==3.8.0
     # via -r requirements/../../../requirements/devenv.txt
-pylint==3.3.0
+pylint==3.2.6
     # via -r requirements/../../../requirements/devenv.txt
 pyproject-hooks==1.1.0
     # via
@@ -71,9 +71,9 @@ pyyaml==6.0.1
     #   -c requirements/_test.txt
     #   pre-commit
     #   watchdog
-ruff==0.6.7
+ruff==0.6.1
     # via -r requirements/../../../requirements/devenv.txt
-setuptools==74.0.0
+setuptools==75.2.0
     # via
     #   -c requirements/_base.txt
     #   pip-tools
@@ -84,9 +84,9 @@ typing-extensions==4.11.0
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   mypy
-virtualenv==20.26.5
+virtualenv==20.26.3
     # via pre-commit
-watchdog==5.0.2
+watchdog==4.0.2
     # via -r requirements/_tools.in
 wheel==0.44.0
     # via pip-tools

services/director-v2/requirements/ci.txt

@@ -10,7 +10,6 @@
 # installs base + tests requirements
 --requirement _base.txt
 --requirement _test.txt
---requirement _tools.txt
 
 # installs this repo's packages
 simcore-common-library @ ../../packages/common-library/

services/director-v2/src/simcore_service_director_v2/core/dynamic_services_settings/sidecar.py

@@ -9,7 +9,7 @@
     ensure_unique_dict_values_validator,
     ensure_unique_list_values_validator,
 )
-from pydantic import Field, PositiveInt, validator
+from pydantic import AliasChoices, ValidationInfo, field_validator, Field, PositiveInt
 from settings_library.aws_s3_cli import AwsS3CliSettings
 from settings_library.base import BaseCustomSettings
 from settings_library.efs import AwsEfsSettings
@@ -45,10 +45,10 @@ class RCloneSettings(SettingsLibraryRCloneSettings):
         description="VFS operation mode, defines how and when the disk cache is synced",
     )
 
-    @validator("R_CLONE_POLL_INTERVAL_SECONDS")
+    @field_validator("R_CLONE_POLL_INTERVAL_SECONDS")
     @classmethod
-    def enforce_r_clone_requirement(cls, v: int, values) -> PositiveInt:
-        dir_cache_time = values["R_CLONE_DIR_CACHE_TIME_SECONDS"]
+    def _enforce_r_clone_requirement(cls, v: int, info: ValidationInfo) -> PositiveInt:
+        dir_cache_time = info.data["R_CLONE_DIR_CACHE_TIME_SECONDS"]
         if v >= dir_cache_time:
             msg = f"R_CLONE_POLL_INTERVAL_SECONDS={v} must be lower than R_CLONE_DIR_CACHE_TIME_SECONDS={dir_cache_time}"
             raise ValueError(msg)
@@ -60,7 +60,7 @@ class PlacementSettings(BaseCustomSettings):
     # https://docs.docker.com/engine/swarm/services/#control-service-placement.
     DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: list[DockerPlacementConstraint] = Field(
         default_factory=list,
-        example='["node.labels.region==east", "one!=yes"]',
+        examples=['["node.labels.region==east", "one!=yes"]'],
     )
 
     DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: dict[
@@ -72,22 +72,20 @@ class PlacementSettings(BaseCustomSettings):
             "see https://github.com/ITISFoundation/osparc-simcore/issues/5250 "
             "When `None` (default), uses generic resources"
         ),
-        example='{"AIRAM": "node.labels.custom==true"}',
+        examples=['{"AIRAM": "node.labels.custom==true"}'],
     )
 
-    _unique_custom_constraints = validator(
+    _unique_custom_constraints = field_validator(
         "DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS",
-        allow_reuse=True,
     )(ensure_unique_list_values_validator)
 
-    _unique_resource_placement_constraints_substitutions = validator(
+    _unique_resource_placement_constraints_substitutions = field_validator(
         "DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS",
-        allow_reuse=True,
     )(ensure_unique_dict_values_validator)
 
-    @validator("DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS")
+    @field_validator("DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS")
     @classmethod
-    def warn_if_any_values_provided(cls, value: dict) -> dict:
+    def _warn_if_any_values_provided(cls, value: dict) -> dict:
         if len(value) > 0:
             warnings.warn(  # noqa: B028
                 "Generic resources will be replaced by the following "
@@ -101,40 +99,40 @@ def warn_if_any_values_provided(cls, value: dict) -> dict:
 class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings):
     DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED: bool = Field(  # doc: https://docs.docker.com/engine/swarm/networking/#configure-service-discovery
         default=False,
-        env=["DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED"],
+        validation_alias=AliasChoices("DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED"),
         description="dynamic-sidecar's service 'endpoint_spec' with {'Mode': 'dnsrr'}",
     )
     DYNAMIC_SIDECAR_SC_BOOT_MODE: BootModeEnum = Field(
         ...,
         description="Boot mode used for the dynamic-sidecar services"
         "By defaults, it uses the same boot mode set for the director-v2",
-        env=["DYNAMIC_SIDECAR_SC_BOOT_MODE", "SC_BOOT_MODE"],
+        validation_alias=AliasChoices("DYNAMIC_SIDECAR_SC_BOOT_MODE", "SC_BOOT_MODE"),
     )
 
     DYNAMIC_SIDECAR_LOG_LEVEL: str = Field(
         "WARNING",
         description="log level of the dynamic sidecar"
         "If defined, it captures global env vars LOG_LEVEL and LOGLEVEL from the director-v2 service",
-        env=["DYNAMIC_SIDECAR_LOG_LEVEL", "LOG_LEVEL", "LOGLEVEL"],
+        validation_alias=AliasChoices("DYNAMIC_SIDECAR_LOG_LEVEL", "LOG_LEVEL", "LOGLEVEL"),
     )
 
     DYNAMIC_SIDECAR_IMAGE: str = Field(
         ...,
-        regex=DYNAMIC_SIDECAR_DOCKER_IMAGE_RE,
+        pattern=DYNAMIC_SIDECAR_DOCKER_IMAGE_RE,
         description="used by the director to start a specific version of the dynamic-sidecar",
     )
 
-    DYNAMIC_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(auto_default_from_env=True)
+    DYNAMIC_SIDECAR_R_CLONE_SETTINGS: RCloneSettings = Field(json_schema_extra={"auto_default_from_env": True})
 
     DYNAMIC_SIDECAR_AWS_S3_CLI_SETTINGS: AwsS3CliSettings | None = Field(
-        auto_default_from_env=True
+        json_schema_extra={"auto_default_from_env": True}
     )
     DYNAMIC_SIDECAR_EFS_SETTINGS: AwsEfsSettings | None = Field(
-        auto_default_from_env=True
+        json_schema_extra={"auto_default_from_env": True}
     )
 
     DYNAMIC_SIDECAR_PLACEMENT_SETTINGS: PlacementSettings = Field(
-        auto_default_from_env=True
+        json_schema_extra={"auto_default_from_env": True}
     )
 
     #
@@ -144,7 +142,7 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings):
     DYNAMIC_SIDECAR_MOUNT_PATH_DEV: Path | None = Field(
         None,
         description="Host path to the dynamic-sidecar project. Used as source path to mount to the dynamic-sidecar [DEVELOPMENT ONLY]",
-        example="osparc-simcore/services/dynamic-sidecar",
+        examples=["osparc-simcore/services/dynamic-sidecar"],
     )
 
     DYNAMIC_SIDECAR_PORT: PortInt = Field(
@@ -157,35 +155,36 @@ class DynamicSidecarSettings(BaseCustomSettings, MixinLoggingSettings):
         description="Publishes the service on localhost for debuging and testing [DEVELOPMENT ONLY]"
         "Can be used to access swagger doc from the host as http://127.0.0.1:30023/dev/doc "
         "where 30023 is the host published port",
+        validate_default=True
     )
 
-    @validator("DYNAMIC_SIDECAR_MOUNT_PATH_DEV", pre=True)
+    @field_validator("DYNAMIC_SIDECAR_MOUNT_PATH_DEV", mode="before")
     @classmethod
-    def auto_disable_if_production(cls, v, values):
-        if v and values.get("DYNAMIC_SIDECAR_SC_BOOT_MODE") == BootModeEnum.PRODUCTION:
+    def _auto_disable_if_production(cls, v, info: ValidationInfo):
+        if v and info.data.get("DYNAMIC_SIDECAR_SC_BOOT_MODE") == BootModeEnum.PRODUCTION:
             _logger.warning(
                 "In production DYNAMIC_SIDECAR_MOUNT_PATH_DEV cannot be set to %s, enforcing None",
                 v,
             )
             return None
         return v
 
-    @validator("DYNAMIC_SIDECAR_EXPOSE_PORT", pre=True, always=True)
+    @field_validator("DYNAMIC_SIDECAR_EXPOSE_PORT", mode="before")
     @classmethod
-    def auto_enable_if_development(cls, v, values):
+    def _auto_enable_if_development(cls, v, info: ValidationInfo):
         if (
-            boot_mode := values.get("DYNAMIC_SIDECAR_SC_BOOT_MODE")
+            boot_mode := info.data.get("DYNAMIC_SIDECAR_SC_BOOT_MODE")
         ) and boot_mode.is_devel_mode():
             # Can be used to access swagger doc from the host as http://127.0.0.1:30023/dev/doc
             return True
         return v
 
-    @validator("DYNAMIC_SIDECAR_IMAGE", pre=True)
+    @field_validator("DYNAMIC_SIDECAR_IMAGE", mode="before")
     @classmethod
-    def strip_leading_slashes(cls, v: str) -> str:
+    def _strip_leading_slashes(cls, v: str) -> str:
         return v.lstrip("/")
 
-    @validator("DYNAMIC_SIDECAR_LOG_LEVEL")
+    @field_validator("DYNAMIC_SIDECAR_LOG_LEVEL")
     @classmethod
     def _validate_log_level(cls, value) -> str:
         log_level: str = cls.validate_log_level(value)