fix

Author: sanderegg

services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py

@@ -449,9 +449,19 @@ async def _start_warm_buffer_instances(
             if (warm_buffer.ec2_instance.type == hot_buffer_instance_type)
             and not warm_buffer.assigned_tasks
         ]
+        # check there are no empty pending ec2s/nodes that are not assigned to any task
+        unnassigned_pending_ec2s = [
+            i.ec2_instance for i in cluster.pending_ec2s if not i.assigned_tasks
+        ]
+        unnassigned_pending_nodes = [
+            i.ec2_instance for i in cluster.pending_nodes if not i.assigned_tasks
+        ]
+
         instances_to_start += free_startable_warm_buffers_to_replace_hot_buffers[
             : app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MACHINES_BUFFER
             - len(cluster.buffer_drained_nodes)
+            - len(unnassigned_pending_ec2s)
+            - len(unnassigned_pending_nodes)
         ]
 
     if not instances_to_start:

services/autoscaling/tests/unit/test_modules_auto_scaling_dynamic.py

@@ -2317,11 +2317,23 @@ async def test_warm_buffers_only_replace_hot_buffer_if_service_is_started_issue7
     assert len(spied_cluster.pending_ec2s) == 1
 
     # running it again shall do nothing
-    await auto_scale_cluster(app=initialized_app, auto_scaling_mode=auto_scaling_mode)
-    spied_cluster = assert_cluster_state(
-        spied_cluster_analysis, expected_calls=1, expected_num_machines=6
+    @tenacity.retry(
+        retry=tenacity.retry_always,
+        reraise=True,
+        wait=tenacity.wait_fixed(0.1),
+        stop=tenacity.stop_after_attempt(10),
     )
-    assert len(spied_cluster.buffer_drained_nodes) == num_hot_buffer - 1
-    assert len(spied_cluster.buffer_ec2s) == buffer_count - 1
-    assert len(spied_cluster.active_nodes) == 1
-    assert len(spied_cluster.pending_ec2s) == 1
+    async def _check_autoscaling_is_stable() -> None:
+        await auto_scale_cluster(
+            app=initialized_app, auto_scaling_mode=auto_scaling_mode
+        )
+        spied_cluster = assert_cluster_state(
+            spied_cluster_analysis, expected_calls=1, expected_num_machines=6
+        )
+        assert len(spied_cluster.buffer_drained_nodes) == num_hot_buffer - 1
+        assert len(spied_cluster.buffer_ec2s) == buffer_count - 1
+        assert len(spied_cluster.active_nodes) == 1
+        assert len(spied_cluster.pending_ec2s) == 1
+
+    with pytest.raises(tenacity.RetryError):
+        await _check_autoscaling_is_stable()