added function to list all images to pre-pull

Author: sanderegg

services/autoscaling/src/simcore_service_autoscaling/utils/utils_docker.py

@@ -13,6 +13,7 @@
 import arrow
 import yaml
 from aws_library.ec2 import EC2InstanceData, Resources
+from aws_library.ec2._models import EC2InstanceBootSpecific
 from models_library.docker import (
     DockerGenericTag,
     DockerLabelKey,
@@ -703,3 +704,13 @@ async def attach_node(
 def is_node_ready(node: Node) -> bool:
     assert node.status  # nosec
     return bool(node.status.state is NodeState.ready)
+
+
+def compute_full_list_of_pre_pulled_images(
+    ec2_boot_specific: EC2InstanceBootSpecific, app_settings: ApplicationSettings
+) -> list[DockerGenericTag]:
+    assert app_settings.AUTOSCALING_EC2_INSTANCES  # nosec
+    common_images = (
+        app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_COLD_START_DOCKER_IMAGES_PRE_PULLING
+    )
+    return sorted(set(common_images) | set(ec2_boot_specific.pre_pull_images))

services/autoscaling/tests/unit/conftest.py

@@ -1243,3 +1243,24 @@ async def _do(
         return instance_ids
 
     return _do
+
+
+@pytest.fixture
+def with_ec2_instances_cold_start_docker_images_pre_pulling(
+    app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker
+) -> EnvVarsDict:
+    images = TypeAdapter(list[DockerGenericTag]).validate_python(
+        [
+            "nginx:latest",
+            "itisfoundation/my-very-nice-service-in-common:latest",
+            "simcore/services/dynamic/another-nice-one:2.4.5161",
+            "asd",
+        ]
+    )
+    envs = setenvs_from_dict(
+        monkeypatch,
+        {
+            "EC2_INSTANCES_COLD_START_DOCKER_IMAGES_PRE_PULLING": json.dumps(images),
+        },
+    )
+    return app_environment | envs

services/autoscaling/tests/unit/test_utils_cluster_scaling.py

@@ -195,27 +195,6 @@ def ec2_instances_boot_ami_pre_pull(
     return app_environment | envs
 
 
-@pytest.fixture
-def ec2_instances_cold_start_docker_images_pre_pulling(
-    app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker
-) -> EnvVarsDict:
-    images = TypeAdapter(list[DockerGenericTag]).validate_python(
-        [
-            "nginx:latest",
-            "itisfoundation/my-very-nice-service:latest",
-            "simcore/services/dynamic/another-nice-one:2.4.5",
-            "asd",
-        ]
-    )
-    envs = setenvs_from_dict(
-        monkeypatch,
-        {
-            "EC2_INSTANCES_COLD_START_DOCKER_IMAGES_PRE_PULLING": json.dumps(images),
-        },
-    )
-    return app_environment | envs
-
-
 @pytest.fixture
 def disabled_registry(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.delenv("REGISTRY_AUTH")
@@ -224,7 +203,7 @@ def disabled_registry(monkeypatch: pytest.MonkeyPatch) -> None:
 async def test_ec2_startup_script_with_pre_pulling(
     minimal_configuration: None,
     ec2_instances_boot_ami_pre_pull: EnvVarsDict,
-    ec2_instances_cold_start_docker_images_pre_pulling: EnvVarsDict,
+    with_ec2_instances_cold_start_docker_images_pre_pulling: EnvVarsDict,
     app_settings: ApplicationSettings,
 ):
     assert app_settings.AUTOSCALING_EC2_INSTANCES

services/autoscaling/tests/unit/test_utils_docker.py

@@ -50,6 +50,7 @@
     attach_node,
     compute_cluster_total_resources,
     compute_cluster_used_resources,
+    compute_full_list_of_pre_pulled_images,
     compute_node_used_resources,
     compute_tasks_needed_resources,
     find_node_with_name,
@@ -1284,3 +1285,37 @@ async def test_attach_node(
     assert is_node_ready_and_available(host_node, availability=Availability.active)
     # but not osparc ready
     assert not is_node_osparc_ready(updated_node)
+
+
+def test_compute_full_list_of_pre_pulled_images(
+    disabled_rabbitmq: None,
+    disabled_ec2: None,
+    disabled_ssm: None,
+    mocked_redis_server: None,
+    enabled_dynamic_mode: EnvVarsDict,
+    disable_autoscaling_background_task: None,
+    with_ec2_instances_cold_start_docker_images_pre_pulling: EnvVarsDict,
+    app_settings: ApplicationSettings,
+):
+    assert app_settings.AUTOSCALING_EC2_INSTANCES
+    assert (
+        app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_COLD_START_DOCKER_IMAGES_PRE_PULLING
+    ), "this test requires some common docker images"
+
+    for (
+        instance_type,
+        instance_boot_specific,
+    ) in app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES.items():
+        returned_list = compute_full_list_of_pre_pulled_images(
+            instance_boot_specific, app_settings
+        )
+        assert (
+            sorted(returned_list) == returned_list
+        ), f"the list for {instance_type} should be sorted"
+        assert len(returned_list) == len(
+            set(returned_list)
+        ), f"the list for {instance_type} should not have duplicates"
+        assert all(
+            i in returned_list
+            for i in app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_COLD_START_DOCKER_IMAGES_PRE_PULLING
+        )