movinf files around

Author: sanderegg

services/autoscaling/src/simcore_service_autoscaling/core/application.py

@@ -17,7 +17,9 @@
     APP_STARTED_DYNAMIC_BANNER_MSG,
 )
 from ..api.routes import setup_api_routes
-from ..modules.auto_scaling_task import setup as setup_auto_scaler_background_task
+from ..modules.cluster_scaling.auto_scaling_task import (
+    setup as setup_auto_scaler_background_task,
+)
 from ..modules.buffer_machines_pool_task import setup as setup_buffer_machines_pool_task
 from ..modules.docker import setup as setup_docker
 from ..modules.ec2 import setup as setup_ec2

services/autoscaling/src/simcore_service_autoscaling/modules/cluster_scaling/__init__.py

@@ -24,44 +24,44 @@
 from servicelib.utils_formatting import timedelta_as_minute_second
 from types_aiobotocore_ec2.literals import InstanceTypeType
 
-from ..constants import DOCKER_JOIN_COMMAND_EC2_TAG_KEY, DOCKER_JOIN_COMMAND_NAME
-from ..core.errors import (
+from ...constants import DOCKER_JOIN_COMMAND_EC2_TAG_KEY, DOCKER_JOIN_COMMAND_NAME
+from ...core.errors import (
     Ec2InvalidDnsNameError,
     TaskBestFittingInstanceNotFoundError,
     TaskRequirementsAboveRequiredEC2InstanceTypeError,
     TaskRequiresUnauthorizedEC2InstanceTypeError,
 )
-from ..core.settings import ApplicationSettings, get_application_settings
-from ..models import (
+from ...core.settings import ApplicationSettings, get_application_settings
+from ...models import (
     AssignedTasksToInstanceType,
     AssociatedInstance,
     Cluster,
     NonAssociatedInstance,
 )
-from ..utils import utils_docker, utils_ec2
-from ..utils.auto_scaling_core import (
+from ...utils import utils_docker, utils_ec2
+from ...utils.auto_scaling_core import (
     associate_ec2_instances_with_nodes,
     ec2_startup_script,
     find_selected_instance_type_for_task,
     get_machine_buffer_type,
     node_host_name_from_ec2_private_dns,
     sort_drained_nodes,
 )
-from ..utils.buffer_machines_pool_core import (
+from ...utils.buffer_machines_pool_core import (
     get_activated_buffer_ec2_tags,
     get_deactivated_buffer_ec2_tags,
     is_buffer_machine,
 )
-from ..utils.rabbitmq import (
+from ...utils.rabbitmq import (
     post_autoscaling_status_message,
     post_tasks_log_message,
     post_tasks_progress_message,
 )
-from .auto_scaling_mode_base import BaseAutoscaling
-from .docker import get_docker_client
-from .ec2 import get_ec2_client
-from .instrumentation import get_instrumentation, has_instrumentation
-from .ssm import get_ssm_client
+from ..auto_scaling_mode_base import BaseAutoscaling
+from ..docker import get_docker_client
+from ..ec2 import get_ec2_client
+from ..instrumentation import get_instrumentation, has_instrumentation
+from ..ssm import get_ssm_client
 
 _logger = logging.getLogger(__name__)
 

services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_core.py renamed to services/autoscaling/src/simcore_service_autoscaling/modules/cluster_scaling/auto_scaling_core.py

@@ -7,12 +7,12 @@
 from servicelib.background_task import create_periodic_task
 from servicelib.redis import exclusive
 
-from ..core.settings import ApplicationSettings
-from ..utils.redis import create_lock_key_and_value
+from ...core.settings import ApplicationSettings
+from ...utils.redis import create_lock_key_and_value
 from .auto_scaling_core import auto_scale_cluster
-from .auto_scaling_mode_computational import ComputationalAutoscaling
-from .auto_scaling_mode_dynamic import DynamicAutoscaling
-from .redis import get_redis_client
+from ..auto_scaling_mode_computational import ComputationalAutoscaling
+from ..auto_scaling_mode_dynamic import DynamicAutoscaling
+from ..redis import get_redis_client
 
 _TASK_NAME: Final[str] = "Autoscaling EC2 instances"
 

services/autoscaling/src/simcore_service_autoscaling/modules/auto_scaling_task.py renamed to services/autoscaling/src/simcore_service_autoscaling/modules/cluster_scaling/auto_scaling_task.py

@@ -78,7 +78,7 @@
     Cluster,
     DaskTaskResources,
 )
-from simcore_service_autoscaling.modules import auto_scaling_core
+from simcore_service_autoscaling.modules.cluster_scaling import auto_scaling_core
 from simcore_service_autoscaling.modules.auto_scaling_mode_dynamic import (
     DynamicAutoscaling,
 )
@@ -754,9 +754,9 @@ async def _() -> None:
             assert tasks, f"no tasks available for {found_service['Spec']['Name']}"
             assert len(tasks) == 1
             service_task = tasks[0]
-            assert (
-                service_task["Status"]["State"] in expected_states
-            ), f"service {found_service['Spec']['Name']}'s task is {service_task['Status']['State']}"
+            assert service_task["Status"]["State"] in expected_states, (
+                f"service {found_service['Spec']['Name']}'s task is {service_task['Status']['State']}"
+            )
             ctx.logger.info(
                 "%s",
                 f"service {found_service['Spec']['Name']} is now {service_task['Status']['State']} {'.' * number_of_success['count']}",
@@ -983,7 +983,9 @@ def _creator(
         assert (
             datetime.timedelta(seconds=10)
             < app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION
-        ), "this tests relies on the fact that the time before termination is above 10 seconds"
+        ), (
+            "this tests relies on the fact that the time before termination is above 10 seconds"
+        )
         assert app_settings.AUTOSCALING_EC2_INSTANCES
         seconds_delta = (
             -datetime.timedelta(seconds=10)
@@ -1124,12 +1126,12 @@ def ec2_instances_allowed_types_with_only_1_buffered(
             allowed_ec2_types.items(),
         )
     )
-    assert (
-        allowed_ec2_types_with_buffer_defined
-    ), "one type with buffer is needed for the tests!"
-    assert (
-        len(allowed_ec2_types_with_buffer_defined) == 1
-    ), "more than one type with buffer is disallowed in this test!"
+    assert allowed_ec2_types_with_buffer_defined, (
+        "one type with buffer is needed for the tests!"
+    )
+    assert len(allowed_ec2_types_with_buffer_defined) == 1, (
+        "more than one type with buffer is disallowed in this test!"
+    )
     return {
         TypeAdapter(InstanceTypeType).validate_python(k): v
         for k, v in allowed_ec2_types_with_buffer_defined.items()
@@ -1153,9 +1155,9 @@ def _by_buffer_count(
         filter(_by_buffer_count, allowed_ec2_types.items())
     )
     assert allowed_ec2_types_with_buffer_defined, "you need one type with buffer"
-    assert (
-        len(allowed_ec2_types_with_buffer_defined) == 1
-    ), "more than one type with buffer is disallowed in this test!"
+    assert len(allowed_ec2_types_with_buffer_defined) == 1, (
+        "more than one type with buffer is disallowed in this test!"
+    )
     return next(iter(allowed_ec2_types_with_buffer_defined.values())).buffer_count
 
 

services/autoscaling/tests/unit/conftest.py

@@ -41,7 +41,9 @@
 from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict
 from simcore_service_autoscaling.core.settings import ApplicationSettings
 from simcore_service_autoscaling.models import EC2InstanceData
-from simcore_service_autoscaling.modules.auto_scaling_core import auto_scale_cluster
+from simcore_service_autoscaling.modules.cluster_scaling.auto_scaling_core import (
+    auto_scale_cluster,
+)
 from simcore_service_autoscaling.modules.auto_scaling_mode_computational import (
     ComputationalAutoscaling,
 )
@@ -185,9 +187,9 @@ def ec2_instance_custom_tags(
 
 
 @pytest.fixture
-def create_dask_task_resources() -> (
-    Callable[[InstanceTypeType | None, Resources], DaskTaskResources]
-):
+def create_dask_task_resources() -> Callable[
+    [InstanceTypeType | None, Resources], DaskTaskResources
+]:
     def _do(
         ec2_instance_type: InstanceTypeType | None, task_resource: Resources
     ) -> DaskTaskResources:
@@ -1473,7 +1475,9 @@ async def test_cluster_adapts_machines_on_the_fly(
     assert (
         scale_up_params1.num_tasks
         >= app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES
-    ), "this test requires to run a first batch of more services than the maximum number of instances allowed"
+    ), (
+        "this test requires to run a first batch of more services than the maximum number of instances allowed"
+    )
     # we have nothing running now
     all_instances = await ec2_client.describe_instances()
     assert not all_instances["Reservations"]
@@ -1685,7 +1689,9 @@ async def test_cluster_adapts_machines_on_the_fly(
     assert "Instances" in reservation1
     assert len(reservation1["Instances"]) == (
         app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES
-    ), f"expected {app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES} EC2 instances, found {len(reservation1['Instances'])}"
+    ), (
+        f"expected {app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES} EC2 instances, found {len(reservation1['Instances'])}"
+    )
     for instance in reservation1["Instances"]:
         assert "InstanceType" in instance
         assert instance["InstanceType"] == scale_up_params1.expected_instance_type
@@ -1699,9 +1705,9 @@ async def test_cluster_adapts_machines_on_the_fly(
 
     reservation2 = all_instances["Reservations"][1]
     assert "Instances" in reservation2
-    assert (
-        len(reservation2["Instances"]) == 1
-    ), f"expected 1 EC2 instances, found {len(reservation2['Instances'])}"
+    assert len(reservation2["Instances"]) == 1, (
+        f"expected 1 EC2 instances, found {len(reservation2['Instances'])}"
+    )
     for instance in reservation2["Instances"]:
         assert "InstanceType" in instance
         assert instance["InstanceType"] == scale_up_params2.expected_instance_type

services/autoscaling/tests/unit/test_modules_auto_scaling_computational.py

@@ -53,7 +53,7 @@
 from simcore_service_autoscaling.constants import BUFFER_MACHINE_TAG_KEY
 from simcore_service_autoscaling.core.settings import ApplicationSettings
 from simcore_service_autoscaling.models import AssociatedInstance, Cluster
-from simcore_service_autoscaling.modules.auto_scaling_core import (
+from simcore_service_autoscaling.modules.cluster_scaling.auto_scaling_core import (
     _activate_drained_nodes,
     _find_terminateable_instances,
     _try_scale_down_cluster,
@@ -520,9 +520,9 @@ async def _test_cluster_scaling_up_and_down(  # noqa: PLR0915
     all_instances = await ec2_client.describe_instances(Filters=instance_type_filters)
     assert not all_instances["Reservations"]
 
-    assert (
-        scale_up_params.expected_num_instances == 1
-    ), "This test is not made to work with more than 1 expected instance. so please adapt if needed"
+    assert scale_up_params.expected_num_instances == 1, (
+        "This test is not made to work with more than 1 expected instance. so please adapt if needed"
+    )
 
     # create the service(s)
     created_docker_services = await create_services_batch(scale_up_params)
@@ -1283,7 +1283,9 @@ async def test_cluster_adapts_machines_on_the_fly(  # noqa: PLR0915
     assert (
         scale_up_params1.num_services
         >= app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES
-    ), "this test requires to run a first batch of more services than the maximum number of instances allowed"
+    ), (
+        "this test requires to run a first batch of more services than the maximum number of instances allowed"
+    )
     # we have nothing running now
     all_instances = await ec2_client.describe_instances()
     assert not all_instances["Reservations"]
@@ -1500,7 +1502,9 @@ async def test_cluster_adapts_machines_on_the_fly(  # noqa: PLR0915
     assert "Instances" in reservation1
     assert len(reservation1["Instances"]) == (
         app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES
-    ), f"expected {app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES} EC2 instances, found {len(reservation1['Instances'])}"
+    ), (
+        f"expected {app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_MAX_INSTANCES} EC2 instances, found {len(reservation1['Instances'])}"
+    )
     for instance in reservation1["Instances"]:
         assert "InstanceType" in instance
         assert instance["InstanceType"] == scale_up_params1.expected_instance_type
@@ -1514,9 +1518,9 @@ async def test_cluster_adapts_machines_on_the_fly(  # noqa: PLR0915
 
     reservation2 = all_instances["Reservations"][1]
     assert "Instances" in reservation2
-    assert (
-        len(reservation2["Instances"]) == 1
-    ), f"expected 1 EC2 instances, found {len(reservation2['Instances'])}"
+    assert len(reservation2["Instances"]) == 1, (
+        f"expected 1 EC2 instances, found {len(reservation2['Instances'])}"
+    )
     for instance in reservation2["Instances"]:
         assert "InstanceType" in instance
         assert instance["InstanceType"] == scale_up_params2.expected_instance_type
@@ -2245,9 +2249,9 @@ async def test_warm_buffers_only_replace_hot_buffer_if_service_is_started_issue7
     # BUG REPRODUCTION
     #
     # start a service that imposes same type as the hot buffer
-    assert (
-        hot_buffer_instance_type == "t2.xlarge"
-    ), "the test is hard-coded for this type and accordingly resource. If this changed then the resource shall be changed too"
+    assert hot_buffer_instance_type == "t2.xlarge", (
+        "the test is hard-coded for this type and accordingly resource. If this changed then the resource shall be changed too"
+    )
     scale_up_params = _ScaleUpParams(
         imposed_instance_type=hot_buffer_instance_type,
         service_resources=Resources(