diff --git a/services/clusters-keeper/requirements/_base.txt b/services/clusters-keeper/requirements/_base.txt index 5c2cff64acf4..2b7f7c6b2341 100644 --- a/services/clusters-keeper/requirements/_base.txt +++ b/services/clusters-keeper/requirements/_base.txt @@ -43,6 +43,8 @@ aiormq==6.8.0 # via aio-pika aiosignal==1.3.1 # via aiohttp +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -54,13 +56,10 @@ arrow==1.3.0 # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -async-timeout==4.0.3 - # via - # aiohttp - # redis attrs==23.2.0 # via # aiohttp @@ -116,24 +115,10 @@ dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via pydantic -exceptiongroup==1.2.1 - # via anyio fast-depends==2.4.2 # via faststream -fastapi==0.99.1 +fastapi==0.115.0 # via - # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator @@ -269,7 +254,7 @@ psutil==5.9.8 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -pydantic==1.10.15 +pydantic==2.9.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -280,7 +265,6 @@ pydantic==1.10.15 # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -297,6 +281,26 @@ pydantic==1.10.15 # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.18.0 # via rich pyinstrument==4.6.2 @@ -307,6 +311,8 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore +python-dotenv==1.0.1 + # via pydantic-settings pyyaml==6.0.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -375,7 +381,7 @@ sortedcontainers==2.4.0 # via # -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt # distributed -starlette==0.27.0 +starlette==0.38.6 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -437,16 +443,15 @@ typing-extensions==4.11.0 # via # aiodebug # aiodocker - # anyio # fastapi # faststream # pydantic + # pydantic-core # typer # types-aiobotocore # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm - # uvicorn urllib3==2.2.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt diff --git a/services/clusters-keeper/requirements/_test.txt b/services/clusters-keeper/requirements/_test.txt index 3539d24ebada..491a8027fd1a 100644 --- a/services/clusters-keeper/requirements/_test.txt +++ b/services/clusters-keeper/requirements/_test.txt @@ -11,6 +11,10 @@ aiosignal==1.3.1 # via # -c requirements/_base.txt # aiohttp +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic antlr4-python3-runtime==4.13.2 # via moto anyio==4.3.0 @@ -19,11 +23,6 @@ anyio==4.3.0 # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -async-timeout==4.0.3 - # via - # -c requirements/_base.txt - # aiohttp - # redis attrs==23.2.0 # via # -c requirements/_base.txt @@ -82,11 +81,6 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -exceptiongroup==1.2.1 - # via - # -c requirements/_base.txt - # anyio - # pytest faker==27.0.0 # via -r requirements/_test.in fakeredis==2.23.5 @@ -209,11 +203,15 @@ py-partiql-parser==0.5.5 # via moto pycparser==2.22 # via cffi -pydantic==1.10.15 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.2 # via moto pytest==8.3.2 @@ -239,7 +237,9 @@ python-dateutil==2.9.0.post0 # faker # moto python-dotenv==1.0.1 - # via -r requirements/_test.in + # via + # -c requirements/_base.txt + # -r requirements/_test.in pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt @@ -302,21 +302,16 @@ sortedcontainers==2.4.0 # fakeredis sympy==1.13.2 # via cfn-lint -tomli==2.0.1 - # via - # coverage - # pytest types-pyyaml==6.0.12.20240808 # via -r requirements/_test.in typing-extensions==4.11.0 # via # -c requirements/_base.txt # aiodocker - # anyio # aws-sam-translator # cfn-lint - # fakeredis # pydantic + # pydantic-core urllib3==2.2.1 # via # -c requirements/../../../requirements/constraints.txt diff --git a/services/clusters-keeper/requirements/_tools.txt b/services/clusters-keeper/requirements/_tools.txt index db86636a373b..7a2c3f9d91b9 100644 --- a/services/clusters-keeper/requirements/_tools.txt +++ b/services/clusters-keeper/requirements/_tools.txt @@ -74,22 +74,12 @@ setuptools==73.0.1 # via # -c requirements/_test.txt # pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.11.0 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/_meta.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/_meta.py index 828216222aa9..58d79f3b9ba7 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/_meta.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/_meta.py @@ -9,17 +9,21 @@ from models_library.basic_types import VersionStr, VersionTag from packaging.version import Version -from pydantic import parse_obj_as +from pydantic import TypeAdapter _current_distribution = distribution("simcore-service-clusters-keeper") __version__: str = version("simcore-service-clusters-keeper") APP_NAME: Final[str] = _current_distribution.metadata["Name"] -API_VERSION: Final[VersionStr] = parse_obj_as(VersionStr, __version__) +API_VERSION: Final[VersionStr] = TypeAdapter(VersionStr).validate_python(__version__) VERSION: Final[Version] = Version(__version__) -API_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, f"v{VERSION.major}") -RPC_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, f"v{VERSION.major}") +API_VTAG: Final[VersionTag] = TypeAdapter(VersionTag).validate_python( + f"v{VERSION.major}" +) +RPC_VTAG: Final[VersionTag] = TypeAdapter(VersionTag).validate_python( + f"v{VERSION.major}" +) def get_summary() -> str: diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py index 4683725e52fc..2e528e1b30b8 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/application.py @@ -25,7 +25,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: - logger.info("app settings: %s", settings.json(indent=1)) + logger.info("app settings: %s", settings.model_dump_json(indent=1)) app = FastAPI( debug=settings.CLUSTERS_KEEPER_DEBUG, diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/errors.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/errors.py index 068a13f702e6..02824102d435 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/errors.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/errors.py @@ -1,7 +1,7 @@ -from pydantic.errors import PydanticErrorMixin +from common_library.errors_classes import OsparcErrorMixin -class ClustersKeeperRuntimeError(PydanticErrorMixin, RuntimeError): +class ClustersKeeperRuntimeError(OsparcErrorMixin, RuntimeError): msg_template: str = "clusters-keeper unexpected error" diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py index 3c1e69d85a21..106ca40b5a7e 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/core/settings.py @@ -1,6 +1,6 @@ import datetime from functools import cached_property -from typing import Any, ClassVar, Final, Literal, cast +from typing import Annotated, Final, Literal, cast from aws_library.ec2 import EC2InstanceBootSpecific, EC2Tags from fastapi import FastAPI @@ -12,14 +12,18 @@ ) from models_library.clusters import InternalClusterAuthentication from pydantic import ( + AliasChoices, + BeforeValidator, Field, NonNegativeFloat, NonNegativeInt, PositiveInt, SecretStr, - parse_obj_as, - validator, + TypeAdapter, + WrapValidator, + field_validator, ) +from pydantic_settings import SettingsConfigDict from settings_library.base import BaseCustomSettings from settings_library.docker_registry import RegistrySettings from settings_library.ec2 import EC2Settings @@ -34,10 +38,9 @@ class ClustersKeeperEC2Settings(EC2Settings): - class Config(EC2Settings.Config): - env_prefix = CLUSTERS_KEEPER_ENV_PREFIX - - schema_extra: ClassVar[dict[str, Any]] = { # type: ignore[misc] + model_config = SettingsConfigDict( + env_prefix=CLUSTERS_KEEPER_ENV_PREFIX, + json_schema_extra={ "examples": [ { f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_ACCESS_KEY_ID": "my_access_key_id", @@ -46,7 +49,8 @@ class Config(EC2Settings.Config): f"{CLUSTERS_KEEPER_ENV_PREFIX}EC2_SECRET_ACCESS_KEY": "my_secret_access_key", } ], - } + }, + ) class WorkersEC2InstancesSettings(BaseCustomSettings): @@ -77,7 +81,7 @@ class WorkersEC2InstancesSettings(BaseCustomSettings): # NAME PREFIX is not exposed since we override it anyway WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field( ..., - min_items=1, + min_length=1, description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic" " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), " " this is required to start a new EC2 instance", @@ -108,14 +112,14 @@ class WorkersEC2InstancesSettings(BaseCustomSettings): "a tag must have a key and an optional value. see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html]", ) - @validator("WORKERS_EC2_INSTANCES_ALLOWED_TYPES") + @field_validator("WORKERS_EC2_INSTANCES_ALLOWED_TYPES") @classmethod def check_valid_instance_names( cls, value: dict[str, EC2InstanceBootSpecific] ) -> dict[str, EC2InstanceBootSpecific]: # NOTE: needed because of a flaw in BaseCustomSettings # issubclass raises TypeError if used on Aliases - parse_obj_as(list[InstanceTypeType], list(value)) + TypeAdapter(list[InstanceTypeType]).validate_python(list(value)) return value @@ -130,7 +134,7 @@ class PrimaryEC2InstancesSettings(BaseCustomSettings): ) PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS: list[str] = Field( ..., - min_items=1, + min_length=1, description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic" " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), " " this is required to start a new EC2 instance", @@ -182,17 +186,17 @@ class PrimaryEC2InstancesSettings(BaseCustomSettings): "that take longer than this time will be terminated as sometimes it happens that EC2 machine fail on start.", ) - @validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES") + @field_validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES") @classmethod def check_valid_instance_names( cls, value: dict[str, EC2InstanceBootSpecific] ) -> dict[str, EC2InstanceBootSpecific]: # NOTE: needed because of a flaw in BaseCustomSettings # issubclass raises TypeError if used on Aliases - parse_obj_as(list[InstanceTypeType], list(value)) + TypeAdapter(list[InstanceTypeType]).validate_python(list(value)) return value - @validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES") + @field_validator("PRIMARY_EC2_INSTANCES_ALLOWED_TYPES") @classmethod def check_only_one_value( cls, value: dict[str, EC2InstanceBootSpecific] @@ -231,30 +235,35 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): # RUNTIME ----------------------------------------------------------- CLUSTERS_KEEPER_DEBUG: bool = Field( - default=False, description="Debug mode", env=["CLUSTERS_KEEPER_DEBUG", "DEBUG"] + default=False, + description="Debug mode", + validation_alias=AliasChoices("CLUSTERS_KEEPER_DEBUG", "DEBUG"), ) CLUSTERS_KEEPER_LOGLEVEL: LogLevel = Field( - LogLevel.INFO, env=["CLUSTERS_KEEPER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"] + LogLevel.INFO, + validation_alias=AliasChoices( + "CLUSTERS_KEEPER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "CLUSTERS_KEEPER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) CLUSTERS_KEEPER_EC2_ACCESS: ClustersKeeperEC2Settings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES: PrimaryEC2InstancesSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES: WorkersEC2InstancesSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX: str = Field( @@ -262,14 +271,18 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): description="set a prefix to all machines created (useful for testing)", ) - CLUSTERS_KEEPER_RABBITMQ: RabbitSettings | None = Field(auto_default_from_env=True) + CLUSTERS_KEEPER_RABBITMQ: RabbitSettings | None = Field( + json_schema_extra={"auto_default_from_env": True} + ) CLUSTERS_KEEPER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True - CLUSTERS_KEEPER_REDIS: RedisSettings = Field(auto_default_from_env=True) + CLUSTERS_KEEPER_REDIS: RedisSettings = Field( + json_schema_extra={"auto_default_from_env": True} + ) CLUSTERS_KEEPER_REGISTRY: RegistrySettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env": True} ) CLUSTERS_KEEPER_TASK_INTERVAL: datetime.timedelta = Field( @@ -320,10 +333,18 @@ class ApplicationSettings(BaseCustomSettings, MixinLoggingSettings): def LOG_LEVEL(self) -> LogLevel: # noqa: N802 return self.CLUSTERS_KEEPER_LOGLEVEL - @validator("CLUSTERS_KEEPER_LOGLEVEL") + @field_validator("CLUSTERS_KEEPER_LOGLEVEL") @classmethod - def valid_log_level(cls, value: str) -> str: + def _valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) + + + @field_validator("CLUSTERS_KEEPER_TASK_INTERVAL", "SERVICE_TRACKING_HEARTBEAT", mode="before") + @classmethod + def _validate_interval(cls, value: str | datetime.timedelta) -> int | datetime.timedelta: + if isinstance(value, str): + return int(value) + return value def get_application_settings(app: FastAPI) -> ApplicationSettings: diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py index a7c23143a0b7..799b724cdcf9 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/clusters_management_core.py @@ -8,7 +8,7 @@ from fastapi import FastAPI from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.logging_utils import log_catch from ..core.settings import get_application_settings @@ -35,8 +35,10 @@ def _get_instance_last_heartbeat(instance: EC2InstanceData) -> datetime.datetime return None -_USER_ID_TAG_KEY: Final[AWSTagKey] = parse_obj_as(AWSTagKey, "user_id") -_WALLET_ID_TAG_KEY: Final[AWSTagKey] = parse_obj_as(AWSTagKey, "wallet_id") +_USER_ID_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python("user_id") +_WALLET_ID_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python( + "wallet_id" +) async def _get_all_associated_worker_instances( diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/dask.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/dask.py index de585dc654fe..af1d0df0e663 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/dask.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/modules/dask.py @@ -34,7 +34,7 @@ async def ping_scheduler( require_encryption=True, ) async with distributed.Client( - url, asynchronous=True, timeout=_CONNECTION_TIMEOUT, security=security + f"{url}", asynchronous=True, timeout=_CONNECTION_TIMEOUT, security=security ): ... return True @@ -59,7 +59,7 @@ async def is_scheduler_busy( require_encryption=True, ) async with distributed.Client( - url, asynchronous=True, timeout=_CONNECTION_TIMEOUT, security=security + f"{url}", asynchronous=True, timeout=_CONNECTION_TIMEOUT, security=security ) as client: datasets_on_scheduler = await _wrap_client_async_routine(client.list_datasets()) _logger.info("cluster currently has %s datasets", len(datasets_on_scheduler)) diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py index 48eb4dee380b..a2c037075cbb 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py @@ -94,7 +94,7 @@ def _convert_to_env_dict(entries: dict[str, Any]) -> str: f"EC2_INSTANCES_NAME_PREFIX={cluster_machines_name_prefix}", f"LOG_LEVEL={app_settings.LOG_LEVEL}", f"WORKERS_EC2_INSTANCES_ALLOWED_TYPES={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_ALLOWED_TYPES)}", - f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags)}", # type: ignore[arg-type] + f"WORKERS_EC2_INSTANCES_CUSTOM_TAGS={_convert_to_env_dict(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_CUSTOM_TAGS | additional_custom_tags)}", f"WORKERS_EC2_INSTANCES_KEY_NAME={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_KEY_NAME}", f"WORKERS_EC2_INSTANCES_MAX_INSTANCES={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_MAX_INSTANCES}", f"WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS={_convert_to_env_list(app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS)}", diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py index 957644f63463..266557358b7f 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/dask.py @@ -1,13 +1,15 @@ from aws_library.ec2 import EC2InstanceData from fastapi import FastAPI from models_library.clusters import InternalClusterAuthentication -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from ..core.settings import get_application_settings def get_scheduler_url(ec2_instance: EC2InstanceData) -> AnyUrl: - url: AnyUrl = parse_obj_as(AnyUrl, f"tls://{ec2_instance.aws_private_dns}:8786") + url: AnyUrl = TypeAdapter(AnyUrl).validate_python( + f"tls://{ec2_instance.aws_private_dns}:8786" + ) return url diff --git a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py index c74bbc554d9a..a19dac17d322 100644 --- a/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py +++ b/services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py @@ -4,17 +4,19 @@ from aws_library.ec2 import AWSTagKey, AWSTagValue, EC2Tags from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import parse_obj_as +from pydantic import TypeAdapter from .._meta import VERSION from ..core.settings import ApplicationSettings _APPLICATION_TAG_KEY: Final[str] = "io.simcore.clusters-keeper" -_APPLICATION_VERSION_TAG: Final[EC2Tags] = parse_obj_as( - EC2Tags, {f"{_APPLICATION_TAG_KEY}.version": f"{VERSION}"} +_APPLICATION_VERSION_TAG: Final[EC2Tags] = TypeAdapter(EC2Tags).validate_python( + {f"{_APPLICATION_TAG_KEY}.version": f"{VERSION}"} ) -HEARTBEAT_TAG_KEY: Final[AWSTagKey] = parse_obj_as(AWSTagKey, "last_heartbeat") +HEARTBEAT_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python( + "last_heartbeat" +) CLUSTER_NAME_PREFIX: Final[str] = "osparc-computational-cluster-" diff --git a/services/clusters-keeper/tests/unit/conftest.py b/services/clusters-keeper/tests/unit/conftest.py index a8f4913d4bb5..3f5a72cca06d 100644 --- a/services/clusters-keeper/tests/unit/conftest.py +++ b/services/clusters-keeper/tests/unit/conftest.py @@ -81,7 +81,7 @@ def mocked_ec2_server_envs( # NOTE: overrides the EC2Settings with what clusters-keeper expects changed_envs: EnvVarsDict = { f"{CLUSTERS_KEEPER_ENV_PREFIX}{k}": v - for k, v in mocked_ec2_server_settings.dict().items() + for k, v in mocked_ec2_server_settings.model_dump().items() } return setenvs_from_dict(monkeypatch, changed_envs) @@ -119,7 +119,9 @@ def app_environment( { random.choice( # noqa: S311 ec2_instances - ): EC2InstanceBootSpecific.Config.schema_extra["examples"][ + ): EC2InstanceBootSpecific.model_config["json_schema_extra"][ + "examples" + ][ 1 ] # NOTE: we use example with custom script } @@ -137,7 +139,9 @@ def app_environment( "WORKERS_EC2_INSTANCES_ALLOWED_TYPES": json.dumps( { ec2_type_name: random.choice( # noqa: S311 - EC2InstanceBootSpecific.Config.schema_extra["examples"] + EC2InstanceBootSpecific.model_config["json_schema_extra"][ + "examples" + ] ) for ec2_type_name in ec2_instances } diff --git a/services/clusters-keeper/tests/unit/test_api_health.py b/services/clusters-keeper/tests/unit/test_api_health.py index 734620afa1b2..a2a14bcf72c0 100644 --- a/services/clusters-keeper/tests/unit/test_api_health.py +++ b/services/clusters-keeper/tests/unit/test_api_health.py @@ -40,7 +40,7 @@ async def test_status_no_rabbit( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is False @@ -60,7 +60,7 @@ async def test_status( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is True @@ -75,7 +75,7 @@ async def test_status( response = await async_client.get("/status") response.raise_for_status() assert response.status_code == status.HTTP_200_OK - status_response = _StatusGet.parse_obj(response.json()) + status_response = _StatusGet.model_validate(response.json()) assert status_response assert status_response.rabbitmq.is_enabled is True diff --git a/services/clusters-keeper/tests/unit/test_core_settings.py b/services/clusters-keeper/tests/unit/test_core_settings.py index 0e467dc1e67b..d734bf32cffa 100644 --- a/services/clusters-keeper/tests/unit/test_core_settings.py +++ b/services/clusters-keeper/tests/unit/test_core_settings.py @@ -45,7 +45,9 @@ def test_multiple_primary_ec2_instances_raises( "PRIMARY_EC2_INSTANCES_ALLOWED_TYPES": json.dumps( { ec2_type_name: random.choice( # noqa: S311 - EC2InstanceBootSpecific.Config.schema_extra["examples"] + EC2InstanceBootSpecific.model_config["json_schema_extra"][ + "examples" + ] ) for ec2_type_name in ec2_instances } diff --git a/services/clusters-keeper/tests/unit/test_modules_dask.py b/services/clusters-keeper/tests/unit/test_modules_dask.py index db1833ffd91f..7f0408d7057c 100644 --- a/services/clusters-keeper/tests/unit/test_modules_dask.py +++ b/services/clusters-keeper/tests/unit/test_modules_dask.py @@ -12,7 +12,7 @@ NoAuthentication, TLSAuthentication, ) -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from simcore_service_clusters_keeper.modules.dask import ( is_scheduler_busy, ping_scheduler, @@ -24,7 +24,9 @@ _authentication_types = [ NoAuthentication(), - TLSAuthentication.construct(**TLSAuthentication.Config.schema_extra["examples"][0]), + TLSAuthentication.model_construct( + **TLSAuthentication.model_config["json_schema_extra"]["examples"][0] + ), ] @@ -36,7 +38,7 @@ async def test_ping_scheduler_non_existing_scheduler( ): assert ( await ping_scheduler( - parse_obj_as(AnyUrl, f"tcp://{faker.ipv4()}:{faker.port_number()}"), + TypeAdapter(AnyUrl).validate_python(f"tcp://{faker.ipv4()}:{faker.port_number()}"), authentication, ) is False @@ -46,7 +48,7 @@ async def test_ping_scheduler_non_existing_scheduler( async def test_ping_scheduler(dask_spec_local_cluster: SpecCluster): assert ( await ping_scheduler( - parse_obj_as(AnyUrl, dask_spec_local_cluster.scheduler_address), + TypeAdapter(AnyUrl).validate_python(dask_spec_local_cluster.scheduler_address), NoAuthentication(), ) is True @@ -69,7 +71,7 @@ async def test_is_scheduler_busy( dask_spec_cluster_client: distributed.Client, ): # nothing runs right now - scheduler_address = parse_obj_as(AnyUrl, dask_spec_local_cluster.scheduler_address) + scheduler_address = TypeAdapter(AnyUrl).validate_python(dask_spec_local_cluster.scheduler_address) assert await is_scheduler_busy(scheduler_address, NoAuthentication()) is False _SLEEP_TIME = 5 @@ -84,5 +86,5 @@ def _some_long_running_fct(sleep_time: int) -> str: busy=True, ) - result = await future.result(timeout=2 * _SLEEP_TIME) # type: ignore + result = await future.result(timeout=2 * _SLEEP_TIME) assert "seconds" in result diff --git a/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py b/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py index a2c23ac06028..110d4fe48271 100644 --- a/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py +++ b/services/clusters-keeper/tests/unit/test_modules_rabbitmq.py @@ -116,7 +116,7 @@ async def test_post_message( f"--> checking for message in rabbit exchange {rabbit_message.channel_name}, {attempt.retry_state.retry_object.statistics}" ) mocked_message_handler.assert_called_once_with( - rabbit_message.json().encode() + rabbit_message.model_dump_json().encode() ) print("... message received") diff --git a/services/clusters-keeper/tests/unit/test_utils_clusters.py b/services/clusters-keeper/tests/unit/test_utils_clusters.py index a6592ed1fa40..7191b1a23f98 100644 --- a/services/clusters-keeper/tests/unit/test_utils_clusters.py +++ b/services/clusters-keeper/tests/unit/test_utils_clusters.py @@ -23,7 +23,7 @@ TLSAuthentication, ) from models_library.utils.json_serialization import json_dumps -from pydantic import ByteSize, parse_obj_as +from pydantic import ByteSize, TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from simcore_service_clusters_keeper.core.settings import ApplicationSettings from simcore_service_clusters_keeper.utils.clusters import ( @@ -60,7 +60,7 @@ def app_environment( monkeypatch, { "CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH": json_dumps( - TLSAuthentication.Config.schema_extra["examples"][0] + TLSAuthentication.model_config["json_schema_extra"]["examples"][0] ) }, ) @@ -178,7 +178,7 @@ def test_create_startup_script_script_size_below_16kb( script_size_in_bytes = len(startup_script.encode("utf-8")) print( - f"current script size is {parse_obj_as(ByteSize, script_size_in_bytes).human_readable()}" + f"current script size is {TypeAdapter(ByteSize).validate_python(script_size_in_bytes).human_readable()}" ) # NOTE: EC2 user data cannot be above 16KB, we keep some margin here assert script_size_in_bytes < 15 * 1024 @@ -240,7 +240,9 @@ def test_startup_script_defines_all_envs_for_docker_compose( "authentication", [ NoAuthentication(), - TLSAuthentication(**TLSAuthentication.Config.schema_extra["examples"][0]), + TLSAuthentication( + **TLSAuthentication.model_config["json_schema_extra"]["examples"][0] + ), ], ) def test_create_cluster_from_ec2_instance(