Skip to content

Commit d30797b

Browse files
joshuarootjoshuaroot
authored
adding asg probe for healthy/unhealthy count as int (#67)
- adding new probe that will return the healthy or unhealthy instance count as an integer value - adding new module `utils.py` - adding new utility that will breakup large iterables into smaller groups to better leverage `describe_*` functions in boto3 Signed-off-by: Joshua Root <[email protected]>
1 parent 893993a commit d30797b

File tree

5 files changed

+250
-20
lines changed

5 files changed

+250
-20
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@
1515
- updating probes ALL list to include describe functions
1616
- adding action to set the desired task count of an ecs service
1717
- updating elbv2 deregister action to include port [#60][60]
18+
- adding probe to asg to report healthy & unhealthy instance counts
19+
- adding utility to breakup large iterables into smaller groups
1820

1921
## [0.12.0][]
2022

‎chaosaws/asg/probes.py

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,12 @@
1010
from chaoslib.exceptions import FailedActivity
1111
from chaoslib.types import Configuration, Secrets
1212
from logzero import logger
13+
from chaosaws.utils import breakup_iterable
1314

1415
__all__ = ["desired_equals_healthy", "desired_equals_healthy_tags",
1516
"wait_desired_equals_healthy", "wait_desired_equals_healthy_tags",
1617
"is_scaling_in_progress", "process_is_suspended", "has_subnets",
17-
"describe_auto_scaling_groups",
18+
"describe_auto_scaling_groups", "instance_count_by_health",
1819
"wait_desired_not_equals_healthy_tags"]
1920

2021

@@ -317,9 +318,74 @@ def has_subnets(subnets: List[str],
317318
return True
318319

319320

321+
def instance_count_by_health(asg_names: List[str] = None,
322+
tags: List[Dict[str, str]] = None,
323+
count_healthy: bool = True,
324+
configuration: Configuration = None,
325+
secrets: Secrets = None) -> int:
326+
"""Reports the number of instances currently in the ASG by their health
327+
status
328+
329+
Params:
330+
OneOf:
331+
- asg_names: a list of asg names to describe
332+
- tags: a list of key/value pairs to collect ASG(s)
333+
334+
- count_healthy: boolean: true for healthy instance count,
335+
false for unhealthy instance count
336+
337+
`tags` are expected as a list of dictionary objects:
338+
[
339+
{'Key': 'TagKey1', 'Value': 'TagValue1'},
340+
{'Key': 'TagKey2', 'Value': 'TagValue2'},
341+
...
342+
]
343+
"""
344+
client = aws_client('autoscaling', configuration, secrets)
345+
asgs = discover_scaling_groups(client, asg_names, tags)
346+
347+
status = 'Healthy'
348+
if not count_healthy:
349+
status = 'Unhealthy'
350+
351+
result = 0
352+
for a in asgs['AutoScalingGroups']:
353+
instances = a['Instances']
354+
for instance in instances:
355+
if instance['HealthStatus'] == status:
356+
result += 1
357+
return result
358+
359+
320360
###############################################################################
321361
# Private functions
322362
###############################################################################
363+
def discover_scaling_groups(client: boto3.client,
364+
asgs: List[str] = None,
365+
tags: List[Dict[str, Any]] = None) -> AWSResponse:
366+
if not any([asgs, tags]):
367+
raise FailedActivity(
368+
'missing one of the required parameters: asg_names or tags')
369+
if not asgs:
370+
asgs = []
371+
372+
if tags:
373+
tag_filter = []
374+
for t in tags:
375+
tag_filter.append({'Name': t['Key'], 'Values': [t['Value']]})
376+
paginator = client.get_paginator('describe_tags')
377+
for p in paginator.paginate(Filters=tag_filter):
378+
asgs.extend([t['ResourceId'] for t in p['Tags'] if t[
379+
'ResourceId'] not in asgs])
380+
381+
results = {'AutoScalingGroups': []}
382+
for group in breakup_iterable(asgs, 50):
383+
response = client.describe_auto_scaling_groups(
384+
AutoScalingGroupNames=group)
385+
results['AutoScalingGroups'].extend(response['AutoScalingGroups'])
386+
return results
387+
388+
323389
def get_asg_by_name(asg_names: List[str],
324390
client: boto3.client) -> AWSResponse:
325391
results = {'AutoScalingGroups': []}

‎chaosaws/utils.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#
2+
def breakup_iterable(values: list, limit: int = 50) -> list:
3+
for i in range(0, len(values), limit):
4+
yield values[i:min(i + limit, len(values))]

‎tests/asg/test_asg_probes.py

Lines changed: 161 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
desired_equals_healthy, desired_equals_healthy_tags, has_subnets,
99
is_scaling_in_progress, wait_desired_equals_healthy, process_is_suspended,
1010
wait_desired_equals_healthy_tags, wait_desired_not_equals_healthy_tags,
11-
describe_auto_scaling_groups)
11+
describe_auto_scaling_groups, instance_count_by_health)
1212
from chaoslib.exceptions import FailedActivity
1313

1414

@@ -137,24 +137,30 @@ def test_desired_equals_healthy_tags_true(aws_client):
137137
client = MagicMock()
138138
aws_client.return_value = client
139139
tags = [{'Key': 'Application', 'Value': 'mychaosapp'}]
140-
client.describe_auto_scaling_groups.return_value = \
141-
{
142-
"AutoScalingGroups": [
143-
{
144-
'AutoScalingGroupName': 'AutoScalingGroup1',
145-
"DesiredCapacity": 1,
146-
"Instances": [{
147-
"HealthStatus": "Healthy",
148-
"LifecycleState": "InService"
149-
}],
150-
'Tags': [{
151-
'ResourceId': 'AutoScalingGroup1',
152-
'Key': 'Application',
153-
'Value': 'mychaosapp'
154-
}]
155-
}
156-
]
157-
}
140+
client.get_paginator.return_value.paginate.return_value = [{
141+
"Tags": [{
142+
"ResourceId": "AutoScalingGroup1",
143+
"ResourceType": "auto-scaling-group",
144+
"Key": "Application",
145+
"Value": "mychaosapp"}]}]
146+
147+
client.describe_auto_scaling_groups.return_value = {
148+
"AutoScalingGroups": [
149+
{
150+
'AutoScalingGroupName': 'AutoScalingGroup1',
151+
"DesiredCapacity": 1,
152+
"Instances": [{
153+
"HealthStatus": "Healthy",
154+
"LifecycleState": "InService"
155+
}],
156+
'Tags': [{
157+
'ResourceId': 'AutoScalingGroup1',
158+
'Key': 'Application',
159+
'Value': 'mychaosapp'
160+
}]
161+
}
162+
]
163+
}
158164
client.get_paginator.return_value.paginate.return_value = [{
159165
"AutoScalingGroups": [
160166
{
@@ -896,3 +902,139 @@ def test_describe_auto_scaling_groups_tags(aws_client):
896902
describe_auto_scaling_groups(tags=tags)
897903
client.describe_auto_scaling_groups.assert_called_with(
898904
AutoScalingGroupNames=["AutoScalingGroup-A", "AutoScalingGroup-B"])
905+
906+
907+
@patch('chaosaws.asg.probes.aws_client', autospec=True)
908+
def test_instance_healthy_count_names(aws_client):
909+
client = MagicMock()
910+
aws_client.return_value = client
911+
asg_names = ['AutoScalingGroup-A']
912+
client.describe_auto_scaling_groups.return_value = {
913+
"AutoScalingGroups": [{
914+
"AutoScalingGroupName": "AutoScalingGroup-A",
915+
"Instances": [
916+
{
917+
"InstanceId": "i-012345678901",
918+
"HealthStatus": "Healthy"
919+
},
920+
{
921+
"InstanceId": "i-012345678902",
922+
"HealthStatus": "Healthy"
923+
},
924+
{
925+
"InstanceId": "i-012345678903",
926+
"HealthStatus": "Unhealthy"
927+
}]}]}
928+
response = instance_count_by_health(asg_names)
929+
client.describe_auto_scaling_groups.assert_called_with(
930+
AutoScalingGroupNames=asg_names)
931+
assert response == 2
932+
933+
934+
@patch('chaosaws.asg.probes.aws_client', autospec=True)
935+
def test_instance_healthy_count_tags(aws_client):
936+
client = MagicMock()
937+
aws_client.return_value = client
938+
tags = [{'Key': 'TestKey', 'Value': 'TestValue'}]
939+
client.get_paginator.return_value.paginate.return_value = [{
940+
"Tags": [{
941+
"ResourceId": "AutoScalingGroup-A",
942+
"ResourceType": "auto-scaling-group",
943+
"Key": "TestKey",
944+
"Value": "TestValue"}]}]
945+
client.describe_auto_scaling_groups.return_value = {
946+
"AutoScalingGroups": [{
947+
"AutoScalingGroupName": "AutoScalingGroup-A",
948+
"Instances": [
949+
{
950+
"InstanceId": "i-012345678901",
951+
"HealthStatus": "Healthy"
952+
},
953+
{
954+
"InstanceId": "i-012345678902",
955+
"HealthStatus": "Healthy"
956+
},
957+
{
958+
"InstanceId": "i-012345678903",
959+
"HealthStatus": "Unhealthy"
960+
}],
961+
"Tags": [{
962+
"ResourceId": "AutoScalingGroup-A",
963+
"Key": "TestKey",
964+
"Value": "TestValue"}]
965+
}]}
966+
response = instance_count_by_health(tags=tags)
967+
client.get_paginator.return_value.paginate.assert_called_with(
968+
Filters=[{'Name': 'TestKey', 'Values': ['TestValue']}])
969+
client.describe_auto_scaling_groups.assert_called_with(
970+
AutoScalingGroupNames=["AutoScalingGroup-A"])
971+
assert response == 2
972+
973+
974+
@patch('chaosaws.asg.probes.aws_client', autospec=True)
975+
def test_instance_unhealthy_count_names(aws_client):
976+
client = MagicMock()
977+
aws_client.return_value = client
978+
asg_names = ['AutoScalingGroup-A']
979+
client.describe_auto_scaling_groups.return_value = {
980+
"AutoScalingGroups": [{
981+
"AutoScalingGroupName": "AutoScalingGroup-A",
982+
"Instances": [
983+
{
984+
"InstanceId": "i-012345678901",
985+
"HealthStatus": "Healthy"
986+
},
987+
{
988+
"InstanceId": "i-012345678902",
989+
"HealthStatus": "Healthy"
990+
},
991+
{
992+
"InstanceId": "i-012345678903",
993+
"HealthStatus": "Unhealthy"
994+
}
995+
]}]}
996+
response = instance_count_by_health(asg_names, count_healthy=False)
997+
client.describe_auto_scaling_groups.assert_called_with(
998+
AutoScalingGroupNames=asg_names)
999+
assert response == 1
1000+
1001+
1002+
@patch('chaosaws.asg.probes.aws_client', autospec=True)
1003+
def test_instance_unhealthy_count_tags(aws_client):
1004+
client = MagicMock()
1005+
aws_client.return_value = client
1006+
tags = [{'Key': 'TestKey', 'Value': 'TestValue'}]
1007+
client.get_paginator.return_value.paginate.return_value = [{
1008+
"Tags": [{
1009+
"ResourceId": "AutoScalingGroup-A",
1010+
"ResourceType": "auto-scaling-group",
1011+
"Key": "TestKey",
1012+
"Value": "TestValue"}]}]
1013+
1014+
client.describe_auto_scaling_groups.return_value = {
1015+
"AutoScalingGroups": [{
1016+
"AutoScalingGroupName": "AutoScalingGroup-A",
1017+
"Instances": [
1018+
{
1019+
"InstanceId": "i-012345678901",
1020+
"HealthStatus": "Healthy"
1021+
},
1022+
{
1023+
"InstanceId": "i-012345678902",
1024+
"HealthStatus": "Healthy"
1025+
},
1026+
{
1027+
"InstanceId": "i-012345678903",
1028+
"HealthStatus": "Unhealthy"
1029+
}],
1030+
"Tags": [{
1031+
"ResourceId": "AutoScalingGroup-A",
1032+
"Key": "TestKey",
1033+
"Value": "TestValue"}]
1034+
}]}
1035+
response = instance_count_by_health(tags=tags, count_healthy=False)
1036+
client.get_paginator.return_value.paginate.assert_called_with(
1037+
Filters=[{'Name': 'TestKey', 'Values': ['TestValue']}])
1038+
client.describe_auto_scaling_groups.assert_called_with(
1039+
AutoScalingGroupNames=["AutoScalingGroup-A"])
1040+
assert response == 1

‎tests/test_utils.py

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#
2+
from unittest import TestCase
3+
from chaosaws.utils import breakup_iterable
4+
5+
6+
class TestUtilities(TestCase):
7+
def test_breakup_iterable(self):
8+
iterable = []
9+
for i in range(0, 100):
10+
iterable.append('Object%s' % i)
11+
12+
iteration = []
13+
for group in breakup_iterable(iterable, 25):
14+
iteration.append(group)
15+
self.assertEqual(len(group), 25)
16+
self.assertEqual(len(iteration), 4)

0 commit comments

Comments
 (0)