test passes

Author: sanderegg

packages/pytest-simcore/src/pytest_simcore/helpers/aws_ec2.py

@@ -42,7 +42,10 @@ async def assert_autoscaled_dynamic_ec2_instances(
     expected_instance_state: InstanceStateNameType,
     expected_additional_tag_keys: list[str],
     instance_filters: Sequence[FilterTypeDef] | None,
+    expected_user_data: list[str] | None = None,
 ) -> list[InstanceTypeDef]:
+    if expected_user_data is None:
+        expected_user_data = ["docker swarm join"]
     return await assert_ec2_instances(
         ec2_client,
         expected_num_reservations=expected_num_reservations,
@@ -54,7 +57,7 @@ async def assert_autoscaled_dynamic_ec2_instances(
             "io.simcore.autoscaling.monitored_services_labels",
             *expected_additional_tag_keys,
         ],
-        expected_user_data=["docker swarm join"],
+        expected_user_data=expected_user_data,
         instance_filters=instance_filters,
     )
 

services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py

@@ -424,17 +424,34 @@ async def _start_warm_buffer_instances(
     """starts warm buffer if there are assigned tasks, or if a hot buffer of the same type is needed"""
 
     app_settings = get_application_settings(app)
-    needed_hot_buffers = (
-        app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MACHINES_BUFFER
-    )
-    hot_buffer_instance_type = next(
-        iter(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES)
-    )
-    current_hot_buffers = cluster.buffer_drained_nodes
+    assert app_settings.AUTOSCALING_EC2_INSTANCES  # nosec
 
     instances_to_start = [
         i.ec2_instance for i in cluster.buffer_ec2s if i.assigned_tasks
     ]
+
+    if (
+        len(cluster.buffer_drained_nodes)
+        < app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MACHINES_BUFFER
+    ):
+        # check if we can migrate warm buffers to hot buffers
+        hot_buffer_instance_type = cast(
+            InstanceTypeType,
+            next(
+                iter(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES)
+            ),
+        )
+        free_starteable_warm_buffers_to_replace_hot_buffers = [
+            warm_buffer.ec2_instance
+            for warm_buffer in cluster.buffer_ec2s
+            if (warm_buffer.ec2_instance.type == hot_buffer_instance_type)
+            and not warm_buffer.assigned_tasks
+        ]
+        instances_to_start += free_starteable_warm_buffers_to_replace_hot_buffers[
+            : app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MACHINES_BUFFER
+            - len(cluster.buffer_drained_nodes)
+        ]
+
     if not instances_to_start:
         return cluster
 

services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py

@@ -50,6 +50,7 @@
 )
 from pytest_simcore.helpers.logging_tools import log_context
 from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict
+from simcore_service_autoscaling.constants import BUFFER_MACHINE_TAG_KEY
 from simcore_service_autoscaling.core.settings import ApplicationSettings
 from simcore_service_autoscaling.models import AssociatedInstance, Cluster
 from simcore_service_autoscaling.modules.auto_scaling_core import (
@@ -1809,6 +1810,7 @@ async def test_warm_buffers_are_started_to_replace_missing_hot_buffers(
     spied_cluster_analysis: MockType,
     instance_type_filters: Sequence[FilterTypeDef],
     mock_find_node_with_name_returns_fake_node: mock.Mock,
+    mock_compute_node_used_resources: mock.Mock,
     mock_docker_tag_node: mock.Mock,
 ):
     # pre-requisites
@@ -1874,8 +1876,12 @@ async def test_warm_buffers_are_started_to_replace_missing_hot_buffers(
             ),
         ),
         expected_instance_state="running",
-        expected_additional_tag_keys=list(ec2_instance_custom_tags),
+        expected_additional_tag_keys=[
+            *list(ec2_instance_custom_tags),
+            BUFFER_MACHINE_TAG_KEY,
+        ],
         instance_filters=instance_type_filters,
+        expected_user_data=[],
     )
 
     # let's autoscale again, to check the cluster analysis and tag the nodes
@@ -1904,6 +1910,6 @@ async def test_warm_buffers_are_started_to_replace_missing_hot_buffers(
         f"found {len(analyzed_cluster.buffer_ec2s)}"
     )
     assert (
-        len(analyzed_cluster.buffer_drained_nodes)
+        len(analyzed_cluster.pending_ec2s)
         == app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MACHINES_BUFFER
     )