ported dynamic-scheduler

Author: Andrei Neagu

packages/common-library/src/common_library/pydantic_validators.py

@@ -0,0 +1,20 @@
+import datetime
+
+from pydantic import field_validator
+
+
+def _try_convert_str_to_float_or_return(
+    v: datetime.timedelta | str | float,
+) -> datetime.timedelta | str | float:
+    if isinstance(v, str):
+        try:
+            return float(v)
+        except ValueError:
+            # returns format like "1:00:00"
+            return v
+    return v
+
+
+def timedelta_try_convert_str_to_float(field: str):
+    """Transforms a float/int number into a valid datetime as it used to work in the past"""
+    return field_validator(field, mode="before")(_try_convert_str_to_float_or_return)

packages/models-library/src/models_library/api_schemas_directorv2/dynamic_services.py

@@ -1,6 +1,6 @@
-from typing import TypeAlias
+from typing import Final, TypeAlias
 
-from pydantic import BaseModel, ByteSize, ConfigDict, Field
+from pydantic import BaseModel, ByteSize, ConfigDict, Field, TypeAdapter
 
 from ..resource_tracker import HardwareInfo, PricingInfo
 from ..services import ServicePortKey
@@ -75,6 +75,9 @@ class DynamicServiceCreate(ServiceDetails):
 
 
 DynamicServiceGet: TypeAlias = RunningDynamicServiceDetails
+DynamicServiceGetAdapter: Final[TypeAdapter[DynamicServiceGet]] = TypeAdapter(
+    DynamicServiceGet
+)
 
 
 class GetProjectInactivityResponse(BaseModel):

packages/models-library/src/models_library/api_schemas_dynamic_scheduler/dynamic_services.py

@@ -1,11 +1,13 @@
+from typing import Final
+
 from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceCreate
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID
 from models_library.resource_tracker import HardwareInfo, PricingInfo
 from models_library.services_resources import ServiceResourcesDictHelpers
 from models_library.users import UserID
 from models_library.wallets import WalletInfo
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, TypeAdapter
 
 
 class DynamicServiceStart(DynamicServiceCreate):
@@ -35,6 +37,11 @@ class DynamicServiceStart(DynamicServiceCreate):
     )
 
 
+DynamicServiceStartAdapter: Final[TypeAdapter[DynamicServiceStart]] = TypeAdapter(
+    DynamicServiceStart
+)
+
+
 class DynamicServiceStop(BaseModel):
     user_id: UserID
     project_id: ProjectID
@@ -53,3 +60,8 @@ class DynamicServiceStop(BaseModel):
             }
         }
     )
+
+
+DynamicServiceStopAdapter: Final[TypeAdapter[DynamicServiceStop]] = TypeAdapter(
+    DynamicServiceStop
+)

packages/models-library/src/models_library/api_schemas_webserver/projects_nodes.py

@@ -1,7 +1,7 @@
 # mypy: disable-error-code=truthy-function
-from typing import Any, Literal, TypeAlias
+from typing import Any, Final, Literal, TypeAlias
 
-from pydantic import ConfigDict, Field
+from pydantic import ConfigDict, Field, TypeAdapter
 
 from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut
 from ..basic_types import PortInt
@@ -106,7 +106,7 @@ class NodeGet(OutputSchema):
                     "service_basepath": "/x/E1O2E-LAH",
                     "service_state": "pending",
                     "service_message": "no suitable node (insufficient resources on 1 node)",
-                    "user_id": 123,
+                    "user_id": "123",
                 },
                 # dynamic
                 {
@@ -120,13 +120,16 @@ class NodeGet(OutputSchema):
                     "service_basepath": "/x/E1O2E-LAH",
                     "service_state": "pending",
                     "service_message": "no suitable node (insufficient resources on 1 node)",
-                    "user_id": 123,
+                    "user_id": "123",
                 },
             ]
         }
     )
 
 
+NodeGetAdapter: Final[TypeAdapter[NodeGet]] = TypeAdapter(NodeGet)
+
+
 class NodeGetIdle(OutputSchema):
     service_state: Literal["idle"]
     service_uuid: NodeID
@@ -145,6 +148,9 @@ def from_node_id(cls, node_id: NodeID) -> "NodeGetIdle":
     )
 
 
+NodeGetIdleAdapter: Final[TypeAdapter[NodeGetIdle]] = TypeAdapter(NodeGetIdle)
+
+
 class NodeGetUnknown(OutputSchema):
     service_state: Literal["unknown"]
     service_uuid: NodeID

packages/settings-library/src/settings_library/basic_types.py

@@ -4,16 +4,17 @@
 #       an extra dependency to a larger models_library (intra-repo library)
 
 from enum import Enum
-from typing import Annotated, TypeAlias
+from typing import Annotated, Final, TypeAlias
 
-from pydantic import Field, StringConstraints
+from pydantic import Field, StringConstraints, TypeAdapter
 
 # port number range
 PortInt: TypeAlias = Annotated[int, Field(gt=0, lt=65535)]
 
 
 # e.g. 'v5'
 VersionTag: TypeAlias = Annotated[str, StringConstraints(pattern=r"^v\d$")]
+VersionTagAdapter: Final[TypeAdapter[VersionTag]] = TypeAdapter(VersionTag)
 
 
 class LogLevel(str, Enum):

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py

@@ -1,9 +1,10 @@
 import datetime
 from functools import cached_property
 
-from pydantic import Field, parse_obj_as, validator
+from common_library.pydantic_validators import timedelta_try_convert_str_to_float
+from pydantic import AliasChoices, ConfigDict, Field, field_validator
 from settings_library.application import BaseApplicationSettings
-from settings_library.basic_types import LogLevel, VersionTag
+from settings_library.basic_types import LogLevel, VersionTag, VersionTagAdapter
 from settings_library.director_v2 import DirectorV2Settings
 from settings_library.rabbit import RabbitSettings
 from settings_library.redis import RedisSettings
@@ -19,20 +20,22 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings):
     # CODE STATICS ---------------------------------------------------------
     API_VERSION: str = API_VERSION
     APP_NAME: str = PROJECT_NAME
-    API_VTAG: VersionTag = parse_obj_as(VersionTag, API_VTAG)
+    API_VTAG: VersionTag = VersionTagAdapter.validate_python(API_VTAG)
 
     # RUNTIME  -----------------------------------------------------------
 
     DYNAMIC_SCHEDULER__LOGLEVEL: LogLevel = Field(
         default=LogLevel.INFO,
-        env=["DYNAMIC_SCHEDULER__LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"],
+        validation_alias=AliasChoices(
+            "DYNAMIC_SCHEDULER__LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"
+        ),
     )
     DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field(
         default=False,
-        env=[
+        validation_alias=AliasChoices(
             "DYNAMIC_SCHEDULER__LOG_FORMAT_LOCAL_DEV_ENABLED",
             "LOG_FORMAT_LOCAL_DEV_ENABLED",
-        ],
+        ),
         description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!",
     )
 
@@ -48,11 +51,17 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings):
     def LOG_LEVEL(self):  # noqa: N802
         return self.DYNAMIC_SCHEDULER__LOGLEVEL
 
-    @validator("DYNAMIC_SCHEDULER__LOGLEVEL")
+    _try_convert_dynamic_scheduler_stop_service_timeout = (
+        timedelta_try_convert_str_to_float("DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT")
+    )
+
+    @field_validator("DYNAMIC_SCHEDULER__LOGLEVEL")
     @classmethod
     def valid_log_level(cls, value: str) -> str:
         return cls.validate_log_level(value)
 
+    model_config = ConfigDict(extra="allow")
+
 
 class ApplicationSettings(_BaseApplicationSettings):
     """Web app's environment variables
@@ -61,24 +70,28 @@ class ApplicationSettings(_BaseApplicationSettings):
     """
 
     DYNAMIC_SCHEDULER_RABBITMQ: RabbitSettings = Field(
-        auto_default_from_env=True, description="settings for service/rabbitmq"
+        json_schema_extra={"auto_default_from_env": True},
+        description="settings for service/rabbitmq",
     )
 
     DYNAMIC_SCHEDULER_REDIS: RedisSettings = Field(
-        auto_default_from_env=True, description="settings for service/redis"
+        json_schema_extra={"auto_default_from_env": True},
+        description="settings for service/redis",
     )
 
     DYNAMIC_SCHEDULER_SWAGGER_API_DOC_ENABLED: bool = Field(
         default=True, description="If true, it displays swagger doc at /doc"
     )
 
     DYNAMIC_SCHEDULER_DIRECTOR_V2_SETTINGS: DirectorV2Settings = Field(
-        auto_default_from_env=True, description="settings for director-v2 service"
+        json_schema_extra={"auto_default_from_env": True},
+        description="settings for director-v2 service",
     )
 
     DYNAMIC_SCHEDULER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True
 
     DYNAMIC_SCHEDULER_PROFILING: bool = False
     DYNAMIC_SCHEDULER_TRACING: TracingSettings | None = Field(
-        auto_default_from_env=True, description="settings for opentelemetry tracing"
+        json_schema_extra={"auto_default_from_env": True},
+        description="settings for opentelemetry tracing",
     )

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/models/schemas/meta.py

@@ -1,17 +1,17 @@
-from typing import Any, ClassVar
-
 from models_library.api_schemas__common.meta import BaseMeta
-from pydantic import HttpUrl
+from pydantic import ConfigDict, HttpUrl
 
 
 class Meta(BaseMeta):
     docs_url: HttpUrl
-
-    class Config:
-        schema_extra: ClassVar[dict[str, Any]] = {
-            "example": {
-                "name": "simcore_service_dynamic_scheduler",
-                "version": "2.4.45",
-                "docs_url": "https://foo.io/doc",
-            }
+    model_config = ConfigDict(
+        json_schema_extra={
+            "examples": [
+                {
+                    "name": "simcore_service_dynamic_scheduler",
+                    "version": "2.4.45",
+                    "docs_url": "https://foo.io/doc",
+                }
+            ]
         }
+    )

services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/director_v2/_public_client.py

@@ -2,11 +2,18 @@
 from typing import Any
 
 from fastapi import FastAPI, status
-from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet
+from models_library.api_schemas_directorv2.dynamic_services import (
+    DynamicServiceGet,
+    DynamicServiceGetAdapter,
+)
 from models_library.api_schemas_dynamic_scheduler.dynamic_services import (
     DynamicServiceStart,
 )
-from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle
+from models_library.api_schemas_webserver.projects_nodes import (
+    NodeGet,
+    NodeGetAdapter,
+    NodeGetIdle,
+)
 from models_library.projects_nodes_io import NodeID
 from servicelib.fastapi.app_state import SingletonInAppStateMixin
 from servicelib.fastapi.http_client import AttachLifespanMixin, HasClientSetupInterface
@@ -43,9 +50,9 @@ async def get_status(
             # in case of legacy version
             # we need to transfer the correct format!
             if "data" in dict_response:
-                return NodeGet.parse_obj(dict_response["data"])
+                return NodeGetAdapter.validate_python(dict_response["data"])
 
-            return DynamicServiceGet.parse_obj(dict_response)
+            return DynamicServiceGetAdapter.validate_python(dict_response)
         except UnexpectedStatusError as e:
             if (
                 e.response.status_code  # type: ignore[attr-defined] # pylint:disable=no-member
@@ -62,9 +69,9 @@ async def run_dynamic_service(
 
         # legacy services
         if "data" in dict_response:
-            return NodeGet.parse_obj(dict_response["data"])
+            return NodeGetAdapter.validate_python(dict_response["data"])
 
-        return DynamicServiceGet.parse_obj(dict_response)
+        return DynamicServiceGetAdapter.validate_python(dict_response)
 
     async def stop_dynamic_service(
         self,

services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py

@@ -9,12 +9,20 @@
 from faker import Faker
 from fastapi import FastAPI, status
 from fastapi.encoders import jsonable_encoder
-from models_library.api_schemas_directorv2.dynamic_services import DynamicServiceGet
+from models_library.api_schemas_directorv2.dynamic_services import (
+    DynamicServiceGet,
+    DynamicServiceGetAdapter,
+)
 from models_library.api_schemas_dynamic_scheduler.dynamic_services import (
     DynamicServiceStart,
+    DynamicServiceStartAdapter,
     DynamicServiceStop,
 )
-from models_library.api_schemas_webserver.projects_nodes import NodeGet, NodeGetIdle
+from models_library.api_schemas_webserver.projects_nodes import (
+    NodeGet,
+    NodeGetAdapter,
+    NodeGetIdle,
+)
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID
 from models_library.users import UserID
@@ -52,14 +60,16 @@ def node_not_found(faker: Faker) -> NodeID:
 
 @pytest.fixture
 def service_status_new_style() -> DynamicServiceGet:
-    return DynamicServiceGet.parse_obj(
-        DynamicServiceGet.Config.schema_extra["examples"][1]
+    return DynamicServiceGetAdapter.validate_python(
+        DynamicServiceGet.model_config["json_schema_extra"]["examples"][1]
     )
 
 
 @pytest.fixture
 def service_status_legacy() -> NodeGet:
-    return NodeGet.parse_obj(NodeGet.Config.schema_extra["examples"][1])
+    return NodeGetAdapter.validate_python(
+        NodeGet.model_config["json_schema_extra"]["examples"][1]
+    )
 
 
 @pytest.fixture
@@ -173,8 +183,8 @@ async def test_get_state(
 @pytest.fixture
 def dynamic_service_start() -> DynamicServiceStart:
     # one for legacy and one for new style?
-    return DynamicServiceStart.parse_obj(
-        DynamicServiceStart.Config.schema_extra["example"]
+    return DynamicServiceStartAdapter.validate_python(
+        DynamicServiceStart.model_config["json_schema_extra"]["example"]
     )
 
 

services/dynamic-scheduler/tests/unit/conftest.py

@@ -4,26 +4,32 @@
 import pytest
 from models_library.api_schemas_dynamic_scheduler.dynamic_services import (
     DynamicServiceStart,
+    DynamicServiceStartAdapter,
     DynamicServiceStop,
+    DynamicServiceStopAdapter,
 )
 from models_library.projects_nodes_io import NodeID
 
 
 @pytest.fixture
 def get_dynamic_service_start() -> Callable[[NodeID], DynamicServiceStart]:
     def _(node_id: NodeID) -> DynamicServiceStart:
-        dict_data = deepcopy(DynamicServiceStart.Config.schema_extra["example"])
+        dict_data = deepcopy(
+            DynamicServiceStart.model_config["json_schema_extra"]["example"]
+        )
         dict_data["service_uuid"] = f"{node_id}"
-        return DynamicServiceStart.parse_obj(dict_data)
+        return DynamicServiceStartAdapter.validate_python(dict_data)
 
     return _
 
 
 @pytest.fixture
 def get_dynamic_service_stop() -> Callable[[NodeID], DynamicServiceStop]:
     def _(node_id: NodeID) -> DynamicServiceStop:
-        dict_data = deepcopy(DynamicServiceStop.Config.schema_extra["example"])
+        dict_data = deepcopy(
+            DynamicServiceStop.model_config["json_schema_extra"]["example"]
+        )
         dict_data["node_id"] = f"{node_id}"
-        return DynamicServiceStop.parse_obj(dict_data)
+        return DynamicServiceStopAdapter.validate_python(dict_data)
 
     return _