diff --git a/scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py b/scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py index 04f368f9f937..3a09e12e3dc3 100644 --- a/scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py +++ b/scripts/maintenance/computational-clusters/autoscaled_monitor/cli.py @@ -209,5 +209,18 @@ def check_database_connection() -> None: asyncio.run(api.check_database_connection(state)) +@app.command() +def terminate_dynamic_instances( + user_id: Annotated[int | None, typer.Option(help="the user ID")] = None, + instance_id: Annotated[str | None, typer.Option(help="the instance ID")] = None, + *, + force: Annotated[bool, typer.Option(help="will not ask for confirmation")] = False, +) -> None: + """this will terminate the instance(s) used for the given user or instance ID.""" + asyncio.run( + api.terminate_dynamic_instances(state, user_id, instance_id, force=force) + ) + + if __name__ == "__main__": app() diff --git a/scripts/maintenance/computational-clusters/autoscaled_monitor/core.py b/scripts/maintenance/computational-clusters/autoscaled_monitor/core.py index baf6b4a73764..9a076d11d7f8 100755 --- a/scripts/maintenance/computational-clusters/autoscaled_monitor/core.py +++ b/scripts/maintenance/computational-clusters/autoscaled_monitor/core.py @@ -491,7 +491,10 @@ async def summary( # get all the running instances assert state.ec2_resource_autoscaling dynamic_instances = await ec2.list_dynamic_instances_from_ec2( - state, user_id, wallet_id + state, + filter_by_user_id=user_id, + filter_by_wallet_id=wallet_id, + filter_by_instance_id=None, ) dynamic_autoscaled_instances = await _parse_dynamic_instances( state, dynamic_instances, state.ssh_key_path, user_id, wallet_id @@ -776,3 +779,48 @@ async def trigger_cluster_termination( async def check_database_connection(state: AppState) -> None: await db.check_db_connection(state) + + +async def terminate_dynamic_instances( + state: AppState, + user_id: int | None, + instance_id: str | None, + *, + force: bool, +) -> None: + if not user_id and not instance_id: + rich.print("either define user_id or instance_id!") + raise typer.Exit(2) + dynamic_instances = await ec2.list_dynamic_instances_from_ec2( + state, + filter_by_user_id=None, + filter_by_wallet_id=None, + filter_by_instance_id=instance_id, + ) + + dynamic_autoscaled_instances = await _parse_dynamic_instances( + state, dynamic_instances, state.ssh_key_path, user_id, None + ) + + if not dynamic_autoscaled_instances: + rich.print("no instances found") + raise typer.Exit(1) + + _print_dynamic_instances( + dynamic_autoscaled_instances, + state.environment, + state.ec2_resource_autoscaling.meta.client.meta.region_name, + output=None, + ) + + for instance in dynamic_autoscaled_instances: + rich.print( + f"terminating instance {instance.ec2_instance.instance_id} with name {utils.get_instance_name(instance.ec2_instance)}" + ) + if force is True or typer.confirm( + f"Are you sure you want to terminate instance {instance.ec2_instance.instance_id}?" + ): + instance.ec2_instance.terminate() + rich.print(f"terminated instance {instance.ec2_instance.instance_id}") + else: + rich.print("not terminating anything") diff --git a/scripts/maintenance/computational-clusters/autoscaled_monitor/ec2.py b/scripts/maintenance/computational-clusters/autoscaled_monitor/ec2.py index 2f1e21423c9d..7dc7c73cf599 100644 --- a/scripts/maintenance/computational-clusters/autoscaled_monitor/ec2.py +++ b/scripts/maintenance/computational-clusters/autoscaled_monitor/ec2.py @@ -18,6 +18,7 @@ def _list_running_ec2_instances( custom_tags: dict[str, str], user_id: int | None, wallet_id: int | None, + instance_id: str | None, ) -> ServiceResourceInstancesCollection: # get all the running instances @@ -37,7 +38,8 @@ def _list_running_ec2_instances( ec2_filters.append({"Name": "tag:user_id", "Values": [f"{user_id}"]}) if wallet_id: ec2_filters.append({"Name": "tag:wallet_id", "Values": [f"{wallet_id}"]}) - + if instance_id: + ec2_filters.append({"Name": "instance-id", "Values": [f"{instance_id}"]}) return ec2_resource.instances.filter(Filters=ec2_filters) @@ -66,13 +68,16 @@ async def list_computational_instances_from_ec2( custom_tags, user_id, wallet_id, + None, ) async def list_dynamic_instances_from_ec2( state: AppState, - user_id: int | None, - wallet_id: int | None, + *, + filter_by_user_id: int | None, + filter_by_wallet_id: int | None, + filter_by_instance_id: str | None, ) -> ServiceResourceInstancesCollection: assert state.environment["EC2_INSTANCES_KEY_NAME"] custom_tags = {} @@ -83,8 +88,9 @@ async def list_dynamic_instances_from_ec2( state.ec2_resource_autoscaling, state.environment["EC2_INSTANCES_KEY_NAME"], custom_tags, - user_id, - wallet_id, + filter_by_user_id, + filter_by_wallet_id, + filter_by_instance_id, ) @@ -101,6 +107,7 @@ async def get_computational_bastion_instance(state: AppState) -> Instance: {}, None, None, + None, ) possible_bastions = list( @@ -120,6 +127,7 @@ async def get_dynamic_bastion_instance(state: AppState) -> Instance: {}, None, None, + None, ) possible_bastions = list( diff --git a/scripts/maintenance/computational-clusters/test.txt b/scripts/maintenance/computational-clusters/test.txt new file mode 100644 index 000000000000..48fdfe03bd1f --- /dev/null +++ b/scripts/maintenance/computational-clusters/test.txt @@ -0,0 +1,136 @@ + dynamic autoscaled instances: us-east-1 +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Instance ┃Running services ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│autoscaling-osparc-production-05-09-2023│n/a │ +│ID: i-0fc848eca40438f07 │ │ +│AMI: ami-079d2dddff7ec5bf2 │ │ +│Type: g4dn.8xlarge │ │ +│Up: 00:21:54 │ │ +│ExtIP: 174.129.68.135 │ │ +│IntIP: 10.0.2.234 │ │ +│/mnt/docker(free): 374.6GiB │ │ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-2-234%22&rangetype=relative&from=4914 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-0913988e3c27955f0 │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││264089│e3348fe4-29a3-11f0-aed8-0242ac174a19│ad4cf922-b8c2-405a-8768-2551b9154b0b│Sim4Life Lite│3.2.179 │00:27:00 │False ││ +│Up: 00:44:26 ││263109│61f6d2a8-29a2-11f0-a827-0242ac174a28│2115738c-8f43-4465-9e62-9d3ad3ea5edd│Sim4Life Lite│3.2.179 │00:39:14 │False ││ +│ExtIP: 54.147.185.53 ││263979│0e813c6c-29a2-11f0-aed8-0242ac174a19│9cadd29d-ffa0-4d64-8529-6e62da8b9cfa│Sim4Life Lite│3.2.179 │00:40:35 │False ││ +│IntIP: 10.0.3.210 ││263315│31818cd0-29a2-11f0-aed8-0242ac174a19│234c8b03-ae6f-44b7-83fd-2e8d50e99c18│Sim4Life Lite│3.2.179 │00:40:39 │False ││ +│/mnt/docker(free): 374.3GiB │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-3-210%22&rangetype=relative&from=6266 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-0acafcf6510669994 │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││263935│e1255fee-29a3-11f0-a827-0242ac174a28│9dc8bc8d-71e5-46ec-be96-1a7ddc279ab9│Sim4Life Lite│3.2.179 │00:20:05 │False ││ +│Up: 00:29:42 ││264097│06d5524e-29a4-11f0-aed8-0242ac174a19│fb765bc1-210d-4264-ab4a-13d3904b1528│Sim4Life Lite│3.2.179 │00:20:11 │False ││ +│ExtIP: 54.227.61.178 ││264093│8095a3c0-2983-11f0-aed8-0242ac174a19│b7c87b97-ddf8-40fd-95f3-040bcf865fa0│Sim4Life Lite│3.2.179 │00:20:11 │False ││ +│IntIP: 10.0.3.38 │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│/mnt/docker(free): 374.3GiB │ │ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-3-38%22&rangetype=relative&from=5382 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-0b5932e8a1bc9031f │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││264074│60b20a22-297d-11f0-aed8-0242ac174a19│6f7013f7-7c28-4f81-808c-bd6b4ed8362f│Sim4Life Lite│3.2.179 │00:37:34 │False ││ +│Up: 11:47:08 ││263112│13f0d568-29a2-11f0-aed8-0242ac174a19│e90dc5a8-1d14-4064-913b-e4331f21716f│Sim4Life Lite│3.2.179 │00:41:21 │False ││ +│ExtIP: 18.208.194.214 ││263127│b337ba8e-29a1-11f0-a827-0242ac174a28│a064c804-88c9-4e2a-8b03-1ca09ed9f47d│s4l-core-lite│3.2.179 │00:43:37 │False ││ +│IntIP: 10.0.2.124 │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│/mnt/docker(free): 374.3GiB │ │ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-2-124%22&rangetype=relative&from=46028 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-04e1f9236667090b3 │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││263122│a2a09aa2-24e6-11f0-aed8-0242ac174a19│c3a03f74-a815-46cf-9752-67e6c7312dab│Sim4Life Lite│3.2.179 │00:21:53 │False ││ +│Up: 08:28:31 ││263070│08d1b382-2984-11f0-a827-0242ac174a28│09a3716c-2c83-4af0-a306-e075681bacb8│Sim4Life Lite│3.2.179 │00:45:52 │False ││ +│ExtIP: 3.90.239.149 ││150025│574acd4c-29a1-11f0-a827-0242ac174a28│527b3c75-a636-40e2-8626-107eeb97bd61│Sim4Life Lite│3.2.179 │00:46:37 │False ││ +│IntIP: 10.0.2.112 │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│/mnt/docker(free): 372.1GiB │ │ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-2-112%22&rangetype=relative&from=34111 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-0e48c0c98a14ef0d5 │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││262998│16ccb350-29a3-11f0-aed8-0242ac174a19│4fdf649d-5e17-4a60-9682-522a818e7325│Sim4Life Lite│3.2.179 │00:33:58 │False ││ +│Up: 00:40:35 ││263110│57691cfa-24e4-11f0-aed8-0242ac174a19│0a195c3f-2936-47a3-9d11-52a684a64d03│Sim4Life Lite│3.2.179 │00:34:03 │False ││ +│ExtIP: 54.196.216.2 ││264024│a0d3215c-29a2-11f0-a827-0242ac174a28│9e72dab1-307e-4b7d-8442-1dd06429f5dc│Sim4Life Lite│3.2.179 │00:36:15 │False ││ +│IntIP: 10.0.2.186 │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│/mnt/docker(free): 372.1GiB │ │ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-2-186%22&rangetype=relative&from=6035 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-052ac050917d3a7f7 │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││264000│6638c0d6-29a4-11f0-a827-0242ac174a28│b3362fc4-7e55-4018-af5c-66d06fae5dbf│Sim4Life Lite│3.2.179 │00:21:46 │False ││ +│Up: 00:37:43 ││264091│70839206-29a3-11f0-aed8-0242ac174a19│41d6e45b-1f65-4fee-9a4c-f8ce0f0e7cde│Sim4Life Lite│3.2.179 │00:31:37 │False ││ +│ExtIP: 34.224.79.128 ││263008│63f0b56e-29a3-11f0-aed8-0242ac174a19│073d4170-2fdf-45a0-bb2b-f237d4419dfb│Sim4Life Lite│3.2.179 │00:31:50 │False ││ +│IntIP: 10.0.2.222 ││262997│4088f7ee-2917-11f0-a827-0242ac174a28│d15d2585-2ca7-4102-9983-a1ed7a3d4822│Sim4Life Lite│3.2.179 │00:32:23 │False ││ +│/mnt/docker(free): 371.9GiB ││264125│376386e8-29a3-11f0-aed8-0242ac174a19│1ac04ca3-1970-4dc5-b36a-8d07420b46b7│Sim4Life Lite│3.2.179 │00:32:43 │False ││ +│ ││263090│4d35053c-29a3-11f0-a827-0242ac174a28│8a7d6ec6-2bf7-4039-9693-f6bbb3e652b9│Sim4Life Lite│3.2.179 │00:32:43 │False ││ +│ │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-2-222%22&rangetype=relative&from=5863 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-009c64beb6bcc8fb4 │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││263009│871ada56-29a3-11f0-aed8-0242ac174a19│1041437f-8a06-4a65-a57c-968288225937│Sim4Life Lite│3.2.179 │00:30:28 │False ││ +│Up: 00:33:34 ││263650│708654c8-29a3-11f0-a827-0242ac174a28│7f4ab81a-2cc1-417c-819b-19a19d0522ce│Sim4Life Lite│3.2.179 │00:30:31 │False ││ +│ExtIP: 54.160.131.70 ││263134│5451c62a-29a3-11f0-a827-0242ac174a28│05988ce7-a665-4213-adb6-d12bc26c6fdd│Sim4Life Lite│3.2.179 │00:30:31 │False ││ +│IntIP: 10.0.2.122 ││263120│79d7cdea-29a3-11f0-a827-0242ac174a28│ed376707-a4e5-49df-b3a8-11ba7630bf9e│Sim4Life Lite│3.2.179 │00:30:32 │False ││ +│/mnt/docker(free): 374.2GiB ││263000│586e46d4-24e4-11f0-a827-0242ac174a28│10eb12d2-af96-4b28-8242-08a34dddfedf│Sim4Life Lite│3.2.179 │00:30:34 │False ││ +│ │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-2-122%22&rangetype=relative&from=5614 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-09d54cc2fc4e7753e │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││263689│8477f4be-29a3-11f0-a827-0242ac174a28│928d1783-0884-420e-8331-fb595fe9c441│Sim4Life Lite│3.2.179 │00:29:31 │False ││ +│Up: 00:31:51 ││264002│879bbeb4-29a3-11f0-a827-0242ac174a28│4ceadc56-6513-4368-baa1-372bb3db814d│Sim4Life Lite│3.2.179 │00:29:35 │False ││ +│ExtIP: 54.197.194.60 ││263293│9d036946-29a3-11f0-aed8-0242ac174a19│66d1bfd5-9aff-437d-9a37-5c7316194bb9│Sim4Life Lite│3.2.179 │00:29:36 │False ││ +│IntIP: 10.0.2.150 ││263146│90e7a168-29a3-11f0-a827-0242ac174a28│77f80839-955e-4b0f-8157-6cfa63147cd8│Sim4Life Lite│3.2.179 │00:29:38 │False ││ +│/mnt/docker(free): 374.2GiB ││264123│8c01b44a-29a3-11f0-aed8-0242ac174a19│1949baad-4e99-4975-bb99-e5b9c4aba266│Sim4Life Lite│3.2.179 │00:29:38 │False ││ +│ │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-2-150%22&rangetype=relative&from=5511 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-0bf06b6fc85aed572 │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││263132│cd5ceb56-24e6-11f0-a827-0242ac174a28│94d97140-26d4-420d-96d6-ad14bae8984c│Sim4Life Lite│3.2.179 │00:28:59 │False ││ +│Up: 00:30:55 ││263124│b7c61756-29a3-11f0-a827-0242ac174a28│4946217a-d381-4c78-a5eb-576254af4efe│Sim4Life Lite│3.2.179 │00:29:00 │False ││ +│ExtIP: 54.172.39.192 ││263129│a8453bfe-29a3-11f0-aed8-0242ac174a19│d3cbda78-2739-4529-9ea5-5b21fb81ec89│Sim4Life Lite│3.2.179 │00:29:07 │False ││ +│IntIP: 10.0.3.144 ││263114│c6d7a926-24e4-11f0-aed8-0242ac174a19│d4987ece-6eac-48b9-a340-63874482a03f│Sim4Life Lite│3.2.179 │00:29:08 │False ││ +│/mnt/docker(free): 374.2GiB ││264121│af8f3568-29a3-11f0-a827-0242ac174a28│cc730efb-b6f4-4734-9d6e-41a83c01964f│Sim4Life Lite│3.2.179 │00:29:12 │False ││ +│ │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-3-144%22&rangetype=relative&from=5455 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-0b870a6100e20c2d3 │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││263126│d4bf67d6-29a3-11f0-aed8-0242ac174a19│c8802f4c-8cae-4857-9901-1c98864b6a62│Sim4Life Lite│3.2.179 │00:27:23 │False ││ +│Up: 00:30:00 ││264120│c64ab82c-29a3-11f0-a827-0242ac174a28│d92e0aa0-2556-496a-9f89-8d44d36ff8fa│Sim4Life Lite│3.2.179 │00:28:03 │False ││ +│ExtIP: 3.80.29.143 ││263626│c0272138-29a3-11f0-aed8-0242ac174a19│67255148-236b-4627-a4ae-ceccaa8d149d│Sim4Life Lite│3.2.179 │00:28:10 │False ││ +│IntIP: 10.0.2.10 ││264124│c1f52622-29a3-11f0-aed8-0242ac174a19│cf621cc5-9738-4bd0-a3ae-183854e94d72│Sim4Life Lite│3.2.179 │00:28:11 │False ││ +│/mnt/docker(free): 374.3GiB │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-2-10%22&rangetype=relative&from=5400 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│autoscaling-osparc-production-05-09-2023│┏━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓│ +│ID: i-0b147bb88d6e4533c │┃UserID┃ProjectID ┃NodeID ┃ServiceName ┃ServiceVersion┃Created Since┃Need intervention┃│ +│AMI: ami-079d2dddff7ec5bf2 │┡━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩│ +│Type: g4dn.8xlarge ││263006│a974cf08-29a3-11f0-a827-0242ac174a28│81768666-a3d3-4d94-9105-27206e05ade5│Sim4Life Lite│3.2.179 │00:18:57 │False ││ +│Up: 00:26:54 ││263087│a6d59188-29a3-11f0-aed8-0242ac174a19│028747fc-dcb1-41ba-bde7-44bb4d637496│Sim4Life Lite│3.2.179 │00:20:27 │False ││ +│ExtIP: 54.91.192.121 ││263960│69c58dec-29a4-11f0-a827-0242ac174a28│976259c4-f376-414b-9151-d81e3890f49c│Sim4Life Lite│3.2.179 │00:20:35 │False ││ +│IntIP: 10.0.2.254 │└──────┴────────────────────────────────────┴────────────────────────────────────┴─────────────┴──────────────┴─────────────┴─────────────────┘│ +│/mnt/docker(free): 374.3GiB │ │ +│Graylog: │https://monitoring.osparc.io/graylog/search?q=source%3A%22ip-10-0-2-254%22&rangetype=relative&from=5214 │ +├────────────────────────────────────────┼───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ │Intervention detection might show false positive if in transient state, be careful and always double-check!! │ +└────────────────────────────────────────┴───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + computational clusters: us-east-1 +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃Instance ┃Computational details ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +└───────────────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘