Merge remote-tracking branch 'upstream/is4481/upgrade-services' into upgrade-dynamic-sceduler

Author: Andrei Neagu

packages/aws-library/src/aws_library/ec2/__init__.py

@@ -1,6 +1,10 @@
 from ._client import SimcoreEC2API
 from ._errors import EC2AccessError, EC2NotConnectedError, EC2RuntimeError
 from ._models import (
+    AWS_TAG_KEY_MAX_LENGTH,
+    AWS_TAG_KEY_MIN_LENGTH,
+    AWS_TAG_VALUE_MAX_LENGTH,
+    AWS_TAG_VALUE_MIN_LENGTH,
     AWSTagKey,
     AWSTagValue,
     EC2InstanceBootSpecific,
@@ -14,6 +18,10 @@
 __all__: tuple[str, ...] = (
     "AWSTagKey",
     "AWSTagValue",
+    "AWS_TAG_KEY_MIN_LENGTH",
+    "AWS_TAG_KEY_MAX_LENGTH",
+    "AWS_TAG_VALUE_MIN_LENGTH",
+    "AWS_TAG_VALUE_MAX_LENGTH",
     "EC2AccessError",
     "EC2InstanceBootSpecific",
     "EC2InstanceConfig",

packages/aws-library/src/aws_library/ec2/_models.py

@@ -2,7 +2,7 @@
 import re
 import tempfile
 from dataclasses import dataclass
-from typing import Annotated, TypeAlias
+from typing import Annotated, Final, TypeAlias
 
 import sh  # type: ignore[import-untyped]
 from models_library.docker import DockerGenericTag
@@ -68,17 +68,21 @@ class EC2InstanceType:
 InstancePrivateDNSName: TypeAlias = str
 
 
+AWS_TAG_KEY_MIN_LENGTH: Final[int] = 1
+AWS_TAG_KEY_MAX_LENGTH: Final[int] = 128
 AWSTagKey: TypeAlias = Annotated[
     # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
     str,
     StringConstraints(
-        min_length=1,
-        max_length=128,
+        min_length=AWS_TAG_KEY_MIN_LENGTH,
+        max_length=AWS_TAG_KEY_MAX_LENGTH,
         pattern=re.compile(r"^(?!(_index|\.{1,2})$)[a-zA-Z0-9\+\-=\._:@]+$"),
     ),
 ]
 
 
+AWS_TAG_VALUE_MIN_LENGTH: Final[int] = 0
+AWS_TAG_VALUE_MAX_LENGTH: Final[int] = 256
 AWSTagValue: TypeAlias = Annotated[
     # see [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html#tag-restrictions]
     # quotes []{} were added as it allows to json encode. it seems to be accepted as a value

packages/models-library/src/models_library/errors_classes.py

@@ -10,15 +10,16 @@ def __missing__(self, key):
 
 class OsparcErrorMixin(PydanticErrorMixin):
     msg_template: str
+    code: str   # type: ignore[assignment]
 
     def __new__(cls, *_args, **_kwargs):
         if not hasattr(cls, "code"):
-            cls.code = cls._get_full_class_name()  # type: ignore[assignment]
+            cls.code = cls._get_full_class_name()
         return super().__new__(cls)
 
     def __init__(self, **ctx: Any) -> None:
         self.__dict__ = ctx
-        super().__init__(message=self._build_message(), code=self.code)
+        super().__init__(message=self._build_message(), code=self.code) # type: ignore[arg-type]
 
     def __str__(self) -> str:
         return self._build_message()

services/autoscaling/src/simcore_service_autoscaling/_meta.py

@@ -1,5 +1,7 @@
 from typing import Final
 
+from pydantic import TypeAdapter
+
 from models_library.basic_types import VersionStr, VersionTag
 from packaging.version import Version
 from servicelib.utils_meta import PackageInfo
@@ -10,7 +12,7 @@
 APP_NAME: Final[str] = info.project_name
 API_VERSION: Final[VersionStr] = info.__version__
 VERSION: Final[Version] = info.version
-API_VTAG: Final[VersionTag] = VersionTag(info.api_prefix_path_tag)
+API_VTAG: Final[VersionTag] = TypeAdapter(VersionTag).validate_python(info.api_prefix_path_tag)
 SUMMARY: Final[str] = info.get_summary()
 
 

services/autoscaling/src/simcore_service_autoscaling/constants.py

@@ -2,32 +2,34 @@
 from typing import Final
 
 from aws_library.ec2._models import AWSTagKey, AWSTagValue, EC2Tags
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 
-BUFFER_MACHINE_PULLING_EC2_TAG_KEY: Final[AWSTagKey] = parse_obj_as(
-    AWSTagKey, "pulling"
+BUFFER_MACHINE_PULLING_EC2_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python(
+    "pulling"
 )
-BUFFER_MACHINE_PULLING_COMMAND_ID_EC2_TAG_KEY: Final[AWSTagKey] = parse_obj_as(
-    AWSTagKey, "ssm-command-id"
+BUFFER_MACHINE_PULLING_COMMAND_ID_EC2_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python(
+    "ssm-command-id"
 )
 PREPULL_COMMAND_NAME: Final[str] = "docker images pulling"
 
 DOCKER_PULL_COMMAND: Final[
     str
 ] = "docker compose -f /docker-pull.compose.yml -p buffering pull"
 
-PRE_PULLED_IMAGES_EC2_TAG_KEY: Final[AWSTagKey] = parse_obj_as(
-    AWSTagKey, "io.simcore.autoscaling.pre_pulled_images"
+PRE_PULLED_IMAGES_EC2_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python(
+    "io.simcore.autoscaling.pre_pulled_images"
 )
 
-BUFFER_MACHINE_TAG_KEY: Final[AWSTagKey] = parse_obj_as(
-    AWSTagKey, "io.simcore.autoscaling.buffer_machine"
+BUFFER_MACHINE_TAG_KEY: Final[AWSTagKey] = TypeAdapter(AWSTagKey).validate_python(
+    "io.simcore.autoscaling.buffer_machine"
 )
 DEACTIVATED_BUFFER_MACHINE_EC2_TAGS: Final[EC2Tags] = {
-    BUFFER_MACHINE_TAG_KEY: parse_obj_as(AWSTagValue, "true")
+    BUFFER_MACHINE_TAG_KEY: TypeAdapter(AWSTagValue).validate_python(
+        "true"
+    )
 }
 ACTIVATED_BUFFER_MACHINE_EC2_TAGS: Final[EC2Tags] = {
-    BUFFER_MACHINE_TAG_KEY: parse_obj_as(AWSTagValue, "false")
+    BUFFER_MACHINE_TAG_KEY: TypeAdapter(AWSTagValue).validate_python("false")
 }
 PRE_PULLED_IMAGES_RE: Final[re.Pattern] = re.compile(
     rf"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_\((\d+)\)"

services/autoscaling/src/simcore_service_autoscaling/core/errors.py

@@ -1,12 +1,7 @@
-from typing import Any
-
 from models_library.errors_classes import OsparcErrorMixin
 
 
 class AutoscalingRuntimeError(OsparcErrorMixin, RuntimeError):
-    def __init__(self, **ctx: Any) -> None:
-        super().__init__(**ctx)
-
     msg_template: str = "Autoscaling unexpected error"
 
 

services/autoscaling/src/simcore_service_autoscaling/core/settings.py

@@ -1,6 +1,6 @@
 import datetime
 from functools import cached_property
-from typing import Final, cast
+from typing import Annotated, Final, cast
 
 from aws_library.ec2 import EC2InstanceBootSpecific, EC2Tags
 from fastapi import FastAPI
@@ -183,7 +183,7 @@ class NodesMonitoringSettings(BaseCustomSettings):
 
 
 class DaskMonitoringSettings(BaseCustomSettings):
-    DASK_MONITORING_URL: AnyUrl = Field(
+    DASK_MONITORING_URL: Annotated[str, AnyUrl] = Field(
         ..., description="the url to the osparc-dask-scheduler"
     )
     DASK_SCHEDULER_AUTH: InternalClusterAuthentication = Field(

services/autoscaling/src/simcore_service_autoscaling/utils/buffer_machines_pool_core.py

@@ -1,11 +1,11 @@
 from collections.abc import Iterable
 from operator import itemgetter
 
-from aws_library.ec2 import AWSTagKey, AWSTagValue, EC2Tags
+from aws_library.ec2 import AWS_TAG_VALUE_MAX_LENGTH, AWSTagKey, AWSTagValue, EC2Tags
 from fastapi import FastAPI
 from models_library.docker import DockerGenericTag
 from models_library.utils.json_serialization import json_dumps
-from pydantic import parse_obj_as, parse_raw_as
+from pydantic import TypeAdapter
 
 from ..constants import (
     ACTIVATED_BUFFER_MACHINE_EC2_TAGS,
@@ -29,8 +29,10 @@ def get_deactivated_buffer_ec2_tags(
     base_ec2_tags = (
         auto_scaling_mode.get_ec2_tags(app) | DEACTIVATED_BUFFER_MACHINE_EC2_TAGS
     )
-    base_ec2_tags[AWSTagKey("Name")] = AWSTagValue(
-        f"{base_ec2_tags[AWSTagKey('Name')]}-buffer"
+    base_ec2_tags[TypeAdapter(AWSTagKey).validate_python("Name")] = TypeAdapter(
+        AWSTagValue
+    ).validate_python(
+        f"{base_ec2_tags[TypeAdapter(AWSTagKey).validate_python('Name')]}-buffer"
     )
     return base_ec2_tags
 
@@ -43,28 +45,36 @@ def dump_pre_pulled_images_as_tags(images: Iterable[DockerGenericTag]) -> EC2Tag
     # AWS Tag Values are limited to 256 characaters so we chunk the images
     # into smaller chunks
     jsonized_images = json_dumps(images)
-    assert AWSTagValue.max_length  # nosec
-    if len(jsonized_images) > AWSTagValue.max_length:
+    assert AWS_TAG_VALUE_MAX_LENGTH  # nosec
+    if len(jsonized_images) > AWS_TAG_VALUE_MAX_LENGTH:
         # let's chunk the string
-        chunk_size = AWSTagValue.max_length
+        chunk_size = AWS_TAG_VALUE_MAX_LENGTH
         chunks = [
             jsonized_images[i : i + chunk_size]
             for i in range(0, len(jsonized_images), chunk_size)
         ]
         return {
-            AWSTagKey(f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_({i})"): AWSTagValue(c)
+            TypeAdapter(AWSTagKey)
+            .validate_python(f"{PRE_PULLED_IMAGES_EC2_TAG_KEY}_({i})"): TypeAdapter(
+                AWSTagValue
+            )
+            .validate_python(c)
             for i, c in enumerate(chunks)
         }
     return {
-        PRE_PULLED_IMAGES_EC2_TAG_KEY: parse_obj_as(AWSTagValue, json_dumps(images))
+        PRE_PULLED_IMAGES_EC2_TAG_KEY: TypeAdapter(AWSTagValue).validate_python(
+            json_dumps(images)
+        )
     }
 
 
 def load_pre_pulled_images_from_tags(tags: EC2Tags) -> list[DockerGenericTag]:
     # AWS Tag values are limited to 256 characters so we chunk the images
     if PRE_PULLED_IMAGES_EC2_TAG_KEY in tags:
         # read directly
-        return parse_raw_as(list[DockerGenericTag], tags[PRE_PULLED_IMAGES_EC2_TAG_KEY])
+        return TypeAdapter(list[DockerGenericTag]).validate_json(
+            tags[PRE_PULLED_IMAGES_EC2_TAG_KEY]
+        )
 
     assembled_json = "".join(
         map(
@@ -80,5 +90,5 @@ def load_pre_pulled_images_from_tags(tags: EC2Tags) -> list[DockerGenericTag]:
         )
     )
     if assembled_json:
-        return parse_raw_as(list[DockerGenericTag], assembled_json)
+        return TypeAdapter(list[DockerGenericTag]).validate_json(assembled_json)
     return []

services/autoscaling/tests/unit/test_utils_rabbitmq.py

@@ -18,7 +18,7 @@
     ProgressRabbitMessageNode,
     ProgressType,
 )
-from pydantic import parse_obj_as
+from pydantic import TypeAdapter
 from pytest_mock.plugin import MockerFixture
 from servicelib.rabbitmq import BIND_TO_ALL_TOPICS, RabbitMQClient
 from settings_library.rabbit import RabbitSettings
@@ -78,8 +78,7 @@ async def test_post_task_log_message(
         "running",
     )
     assert service_with_labels.Spec
-    service_tasks = parse_obj_as(
-        list[Task],
+    service_tasks = TypeAdapter(list[Task]).validate_python(
         await async_docker_client.tasks.list(
             filters={"service": service_with_labels.Spec.Name}
         ),
@@ -103,7 +102,7 @@ async def test_post_task_log_message(
                     messages=[f"[cluster] {log_message}"],
                     log_level=0,
                 )
-                .json()
+                .model_dump_json()
                 .encode()
             )
             print("... message received")
@@ -125,8 +124,7 @@ async def test_post_task_log_message_does_not_raise_if_service_has_no_labels(
 ):
     service_without_labels = await create_service(task_template, {}, "running")
     assert service_without_labels.Spec
-    service_tasks = parse_obj_as(
-        list[Task],
+    service_tasks = TypeAdapter(list[Task]).validate_python(
         await async_docker_client.tasks.list(
             filters={"service": service_without_labels.Spec.Name}
         ),
@@ -170,8 +168,7 @@ async def test_post_task_progress_message(
         "running",
     )
     assert service_with_labels.Spec
-    service_tasks = parse_obj_as(
-        list[Task],
+    service_tasks = TypeAdapter(list[Task]).validate_python(
         await async_docker_client.tasks.list(
             filters={"service": service_with_labels.Spec.Name}
         ),
@@ -195,7 +192,7 @@ async def test_post_task_progress_message(
                     progress_type=ProgressType.CLUSTER_UP_SCALING,
                     report=ProgressReport(actual_value=progress_value, total=1),
                 )
-                .json()
+                .model_dump_json()
                 .encode()
             )
             print("... message received")
@@ -217,8 +214,7 @@ async def test_post_task_progress_does_not_raise_if_service_has_no_labels(
 ):
     service_without_labels = await create_service(task_template, {}, "running")
     assert service_without_labels.Spec
-    service_tasks = parse_obj_as(
-        list[Task],
+    service_tasks = TypeAdapter(list[Task]).validate_python(
         await async_docker_client.tasks.list(
             filters={"service": service_without_labels.Spec.Name}
         ),