🐛Clusters-keeper/autoscaling: reduce UserData size

packages/aws-library/src/aws_library/ec2/utils.py

@@ -1,10 +1,10 @@
 from textwrap import dedent
 
 
-def compose_user_data(docker_join_bash_command: str) -> str:
+def compose_user_data(bash_command: str) -> str:
     return dedent(
         f"""\
 #!/bin/bash
-{docker_join_bash_command}
+{bash_command}
 """
     )

services/autoscaling/src/simcore_service_autoscaling/utils/utils_ec2.py

@@ -6,7 +6,6 @@
 import logging
 from collections import OrderedDict
 from collections.abc import Callable
-from textwrap import dedent
 
 from aws_library.ec2.models import (
     AWSTagKey,
@@ -60,15 +59,6 @@ def get_ec2_tags_computational(app_settings: ApplicationSettings) -> EC2Tags:
     } | app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_CUSTOM_TAGS
 
 
-def compose_user_data(docker_join_bash_command: str) -> str:
-    return dedent(
-        f"""\
-#!/bin/bash
-{docker_join_bash_command}
-"""
-    )
-
-
 def closest_instance_policy(
     ec2_instance: EC2InstanceType,
     resources: Resources,

services/autoscaling/tests/unit/test_utils_ec2.py

@@ -5,15 +5,13 @@
 
 import pytest
 from aws_library.ec2.models import EC2InstanceType, Resources
-from faker import Faker
 from pydantic import ByteSize
 from simcore_service_autoscaling.core.errors import (
     ConfigurationError,
     Ec2InstanceNotFoundError,
 )
 from simcore_service_autoscaling.utils.utils_ec2 import (
     closest_instance_policy,
-    compose_user_data,
     find_best_fitting_ec2_instance,
 )
 
@@ -63,10 +61,3 @@ async def test_find_best_fitting_ec2_instance_closest_instance_policy(
     )
 
     assert found_instance.resources == expected_ec2_instance.resources
-
-
-def test_compose_user_data(faker: Faker):
-    command = faker.text()
-    user_data = compose_user_data(command)
-    assert user_data.startswith("#!/bin/bash")
-    assert command in user_data

services/clusters-keeper/src/simcore_service_clusters_keeper/data/docker-compose.yml

@@ -95,7 +95,7 @@ services:
       DASK_SCHEDULER_AUTH: '{"type":"tls","tls_ca_file":"${DASK_TLS_CA_FILE}","tls_client_cert":"${DASK_TLS_CERT}","tls_client_key":"${DASK_TLS_KEY}"}'
       EC2_INSTANCES_ALLOWED_TYPES: ${WORKERS_EC2_INSTANCES_ALLOWED_TYPES}
       EC2_INSTANCES_CUSTOM_TAGS: ${WORKERS_EC2_INSTANCES_CUSTOM_TAGS}
-      EC2_INSTANCES_ATTACHED_IAM_PROFILE: ""
+      EC2_INSTANCES_ATTACHED_IAM_PROFILE: ${EC2_INSTANCES_ATTACHED_IAM_PROFILE}
       EC2_INSTANCES_KEY_NAME: ${WORKERS_EC2_INSTANCES_KEY_NAME}
       EC2_INSTANCES_MACHINES_BUFFER: 0
       EC2_INSTANCES_MAX_INSTANCES: ${WORKERS_EC2_INSTANCES_MAX_INSTANCES}
@@ -188,6 +188,7 @@ services:
 networks:
   cluster:
 
+
 configs:
   prometheus-config:
     file: ./prometheus.yml
@@ -200,6 +201,7 @@ volumes:
   redis-data:
   prometheus-data:
 
+
 secrets:
   dask_tls_ca:
     file: ${DASK_TLS_CA_FILE}

services/clusters-keeper/src/simcore_service_clusters_keeper/utils/clusters.py

@@ -101,6 +101,7 @@ def _convert_to_env_dict(entries: dict[str, Any]) -> str:
         f"WORKERS_EC2_INSTANCES_SUBNET_ID={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_SUBNET_ID}",
         f"WORKERS_EC2_INSTANCES_TIME_BEFORE_DRAINING={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_DRAINING}",
         f"WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION={app_settings.CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES.WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION}",
+        f"EC2_INSTANCES_ATTACHED_IAM_PROFILE={app_settings.CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES.PRIMARY_EC2_INSTANCES_ATTACHED_IAM_PROFILE}",
     ]
 
 

services/clusters-keeper/src/simcore_service_clusters_keeper/utils/ec2.py

@@ -1,4 +1,3 @@
-from textwrap import dedent
 from typing import Final
 
 from aws_library.ec2.models import AWSTagKey, AWSTagValue, EC2Tags
@@ -70,14 +69,3 @@ def ec2_instances_for_user_wallet_filter(
         | {AWSTagKey("user_id"): AWSTagValue(f"{user_id}")}
         | {AWSTagKey("wallet_id"): AWSTagValue(f"{wallet_id}")}
     )
-
-
-def compose_user_data(bash_command: str) -> str:
-    return dedent(
-        f"""\
-#!/bin/bash
-echo "started user data bash script"
-{bash_command}
-echo "completed user data bash script"
-"""
-    )

services/clusters-keeper/tests/unit/test_utils_ec2.py

@@ -11,7 +11,6 @@
 from simcore_service_clusters_keeper.utils.ec2 import (
     _APPLICATION_TAG_KEY,
     all_created_ec2_instances_filter,
-    compose_user_data,
     creation_ec2_tags,
     get_cluster_name,
 )
@@ -93,15 +92,3 @@ def test_all_created_ec2_instances_filter(
     assert all(
         tag_key_name in EXPECTED_TAG_KEY_NAMES for tag_key_name in received_tags
     ), f"non expected tag key names in {received_tags.keys()}, expected {EXPECTED_TAG_KEY_NAMES}"
-
-
-@pytest.fixture
-def bash_command(faker: Faker) -> str:
-    return faker.pystr()
-
-
-def test_compose_user_data(bash_command: str):
-    received_user_data = compose_user_data(bash_command)
-    assert received_user_data
-    assert received_user_data.startswith("#!/bin/bash\n")
-    assert bash_command in received_user_data