test for pre-pulling on hot buffers

Author: sanderegg

packages/pytest-simcore/src/pytest_simcore/helpers/aws_ec2.py

@@ -1,7 +1,7 @@
 import base64
 from collections.abc import Sequence
 
-from common_library.json_serialization import json_dumps
+from common_library.json_serialization import json_loads
 from models_library.docker import DockerGenericTag
 from types_aiobotocore_ec2 import EC2Client
 from types_aiobotocore_ec2.literals import InstanceStateNameType, InstanceTypeType
@@ -46,6 +46,7 @@ async def assert_autoscaled_dynamic_ec2_instances(
     expected_instance_type: InstanceTypeType,
     expected_instance_state: InstanceStateNameType,
     expected_additional_tag_keys: list[str],
+    expected_pre_pulled_images: list[DockerGenericTag] | None = None,
     instance_filters: Sequence[FilterTypeDef] | None,
     expected_user_data: list[str] | None = None,
     check_reservation_index: int | None = None,
@@ -64,6 +65,7 @@ async def assert_autoscaled_dynamic_ec2_instances(
             *expected_additional_tag_keys,
         ],
         expected_user_data=expected_user_data,
+        expected_pre_pulled_images=expected_pre_pulled_images,
         instance_filters=instance_filters,
         check_reservation_index=check_reservation_index,
     )
@@ -143,8 +145,8 @@ def _by_pre_pull_image(ec2_tag: TagTypeDef) -> bool:
             )
             assert "Value" in instance_pre_pulled_images_aws_tag
             assert (
-                instance_pre_pulled_images_aws_tag["Value"]
-                == f"{json_dumps(expected_pre_pulled_images)}"
+                json_loads(instance_pre_pulled_images_aws_tag["Value"]).sort()
+                == expected_pre_pulled_images.sort()
             )
 
         assert "PrivateDnsName" in instance

services/autoscaling/src/simcore_service_autoscaling/core/application.py

@@ -1,5 +1,6 @@
 import logging
 
+from common_library.json_serialization import json_dumps
 from fastapi import FastAPI
 from servicelib.fastapi.tracing import (
     initialize_fastapi_app_tracing,
@@ -32,7 +33,7 @@
 from ..modules.ssm import setup as setup_ssm
 from .settings import ApplicationSettings
 
-logger = logging.getLogger(__name__)
+_logger = logging.getLogger(__name__)
 
 
 def create_app(settings: ApplicationSettings, tracing_config: TracingConfig) -> FastAPI:
@@ -49,6 +50,10 @@ def create_app(settings: ApplicationSettings, tracing_config: TracingConfig) ->
     app.state.settings = settings
     app.state.tracing_config = tracing_config
     assert app.state.settings.API_VERSION == API_VERSION  # nosec
+    _logger.info(
+        "Application settings: %s",
+        json_dumps(settings, indent=2, sort_keys=True),
+    )
 
     # PLUGINS SETUP
     if tracing_config.tracing_enabled:

services/autoscaling/tests/unit/test_modules_cluster_scaling_dynamic.py

@@ -54,7 +54,12 @@
 )
 from pytest_simcore.helpers.logging_tools import log_context
 from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict
-from simcore_service_autoscaling.constants import BUFFER_MACHINE_TAG_KEY
+from simcore_service_autoscaling.constants import (
+    BUFFER_MACHINE_TAG_KEY,
+    MACHINE_PULLING_COMMAND_ID_EC2_TAG_KEY,
+    MACHINE_PULLING_EC2_TAG_KEY,
+    PRE_PULLED_IMAGES_EC2_TAG_KEY,
+)
 from simcore_service_autoscaling.core.settings import ApplicationSettings
 from simcore_service_autoscaling.models import AssociatedInstance, Cluster
 from simcore_service_autoscaling.modules.cluster_scaling._auto_scaling_core import (
@@ -365,6 +370,10 @@ async def test_cluster_scaling_with_no_services_and_machine_buffer_starts_expect
     ec2_instance_custom_tags: dict[str, str],
     instance_type_filters: Sequence[FilterTypeDef],
 ):
+    hot_buffer_instance_type = cast(
+        InstanceTypeType,
+        next(iter(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES)),
+    )
     assert app_settings.AUTOSCALING_EC2_INSTANCES
     await auto_scale_cluster(
         app=initialized_app, auto_scaling_mode=DynamicAutoscalingProvider()
@@ -373,12 +382,7 @@ async def test_cluster_scaling_with_no_services_and_machine_buffer_starts_expect
         ec2_client,
         expected_num_reservations=1,
         expected_num_instances=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MACHINES_BUFFER,
-        expected_instance_type=cast(
-            InstanceTypeType,
-            next(
-                iter(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES)
-            ),
-        ),
+        expected_instance_type=hot_buffer_instance_type,
         expected_instance_state="running",
         expected_additional_tag_keys=list(ec2_instance_custom_tags),
         instance_filters=instance_type_filters,
@@ -391,21 +395,36 @@ async def test_cluster_scaling_with_no_services_and_machine_buffer_starts_expect
     )
     mock_rabbitmq_post_message.reset_mock()
     # calling again should attach the new nodes to the reserve, but nothing should start
+    # it will also trigger pre-pulling of images if there is pre-pulling
+    hot_buffer_has_pre_pull = bool(
+        app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[
+            hot_buffer_instance_type
+        ].pre_pull_images
+    )
+    expected_pre_pull_tag_keys = (
+        [
+            MACHINE_PULLING_COMMAND_ID_EC2_TAG_KEY,
+            MACHINE_PULLING_EC2_TAG_KEY,
+            PRE_PULLED_IMAGES_EC2_TAG_KEY,
+        ]
+        if hot_buffer_has_pre_pull
+        else []
+    )
     await auto_scale_cluster(
         app=initialized_app, auto_scaling_mode=DynamicAutoscalingProvider()
     )
     await assert_autoscaled_dynamic_ec2_instances(
         ec2_client,
         expected_num_reservations=1,
         expected_num_instances=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MACHINES_BUFFER,
-        expected_instance_type=cast(
-            InstanceTypeType,
-            next(
-                iter(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES)
-            ),
-        ),
+        expected_instance_type=hot_buffer_instance_type,
         expected_instance_state="running",
-        expected_additional_tag_keys=list(ec2_instance_custom_tags),
+        expected_additional_tag_keys=list(
+            ec2_instance_custom_tags.keys() | expected_pre_pull_tag_keys
+        ),
+        expected_pre_pulled_images=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[
+            hot_buffer_instance_type
+        ].pre_pull_images,
         instance_filters=instance_type_filters,
     )
     assert fake_node.description
@@ -428,7 +447,10 @@ async def test_cluster_scaling_with_no_services_and_machine_buffer_starts_expect
         },
     )
 
-    # calling it again should not create anything new
+    # calling it again should not create anything new, pre-pulling should be done
+    expected_pre_pull_tag_keys = (
+        [PRE_PULLED_IMAGES_EC2_TAG_KEY] if hot_buffer_has_pre_pull else []
+    )
     for _ in range(10):
         await auto_scale_cluster(
             app=initialized_app, auto_scaling_mode=DynamicAutoscalingProvider()
@@ -437,14 +459,15 @@ async def test_cluster_scaling_with_no_services_and_machine_buffer_starts_expect
         ec2_client,
         expected_num_reservations=1,
         expected_num_instances=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MACHINES_BUFFER,
-        expected_instance_type=cast(
-            InstanceTypeType,
-            next(
-                iter(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES)
-            ),
-        ),
+        expected_instance_type=hot_buffer_instance_type,
         expected_instance_state="running",
-        expected_additional_tag_keys=list(ec2_instance_custom_tags),
+        expected_additional_tag_keys=list(
+            ec2_instance_custom_tags.keys() | expected_pre_pull_tag_keys
+        ),
+        expected_pre_pulled_images=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[
+            hot_buffer_instance_type
+        ].pre_pull_images
+        or None,
         instance_filters=instance_type_filters,
     )