From 1d1879906c4d13d226aa8f4e81a17fa0c0f41a82 Mon Sep 17 00:00:00 2001 From: serosset Date: Sat, 15 Jan 2022 11:07:17 -0800 Subject: [PATCH 1/8] Initial policy statement for opensearch service --- aws/policy/data-services.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/aws/policy/data-services.yaml b/aws/policy/data-services.yaml index 8c4164d4..0c73ada5 100644 --- a/aws/policy/data-services.yaml +++ b/aws/policy/data-services.yaml @@ -134,7 +134,7 @@ Statement: - 'arn:aws:elasticache:{{ aws_region }}:{{ aws_account_id }}:parametergroup:*' - 'arn:aws:elasticache:{{ aws_region }}:{{ aws_account_id }}:securitygroup:*' - 'arn:aws:redshift:{{ aws_region }}:{{ aws_account_id }}:cluster:*' - # This allows AWS Services to autmatically create their Default Service Linked Roles + # This allows AWS Services to automatically create their Default Service Linked Roles # These have fixed policies and can only be assumed by the service itself. - Sid: AllowServiceLinkedRoleCreation Effect: Allow @@ -186,3 +186,8 @@ Statement: - kafka:UntagResource - kafka:ListClusterOperations Resource: "*" + - Sid: OpenSearchCluster + Effect: Allow + Action: + - es:DescribeElasticsearchDomain + Resource: "*" From c2727d508590436a5344252f14109a0ae157baa0 Mon Sep 17 00:00:00 2001 From: serosset Date: Sat, 15 Jan 2022 11:18:42 -0800 Subject: [PATCH 2/8] Add OpenSearch permissions --- aws/policy/data-services.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/aws/policy/data-services.yaml b/aws/policy/data-services.yaml index 0c73ada5..c5d38b8d 100644 --- a/aws/policy/data-services.yaml +++ b/aws/policy/data-services.yaml @@ -189,5 +189,22 @@ Statement: - Sid: OpenSearchCluster Effect: Allow Action: + - es:AddTags + - es:CreateDomain - es:DescribeElasticsearchDomain + - es:DescribeDomainAutoTunes + - es:DescribeDomain + - es:DescribeDomainConfig + - es:DescribeDomains + - es:GetCompatibleVersions + - es:GetUpgradeHistory + - es:GetUpgradeStatus + - es:ListDomainNames + - es:ListInstanceTypeDetails + - es:ListTags + - es:ListVersions + - es:DeleteDomain + - es:RemoveTags + - es:UpdateDomainConfig + - es:UpgradeDomain Resource: "*" From 42778ba1f75efaf1674a63b8885329cef2e2b65c Mon Sep 17 00:00:00 2001 From: serosset Date: Thu, 27 Jan 2022 23:50:30 +0000 Subject: [PATCH 3/8] create a more compact list of permissions --- aws/policy/data-services.yaml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/aws/policy/data-services.yaml b/aws/policy/data-services.yaml index c5d38b8d..46ed96b0 100644 --- a/aws/policy/data-services.yaml +++ b/aws/policy/data-services.yaml @@ -191,18 +191,9 @@ Statement: Action: - es:AddTags - es:CreateDomain - - es:DescribeElasticsearchDomain - - es:DescribeDomainAutoTunes - - es:DescribeDomain - - es:DescribeDomainConfig - - es:DescribeDomains - - es:GetCompatibleVersions - - es:GetUpgradeHistory - - es:GetUpgradeStatus - - es:ListDomainNames - - es:ListInstanceTypeDetails - - es:ListTags - - es:ListVersions + - es:Describe* + - es:Get* + - es:List* - es:DeleteDomain - es:RemoveTags - es:UpdateDomainConfig From 81cc31712ad53d9935b0ea1da9146c564e916d32 Mon Sep 17 00:00:00 2001 From: serosset Date: Thu, 27 Jan 2022 23:56:46 +0000 Subject: [PATCH 4/8] create a more compact list of permissions --- aws/policy/data-services.yaml | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/aws/policy/data-services.yaml b/aws/policy/data-services.yaml index 46ed96b0..ccb12d94 100644 --- a/aws/policy/data-services.yaml +++ b/aws/policy/data-services.yaml @@ -17,6 +17,9 @@ Statement: - glue:GetConnections - rds:DescribeDB* - rds:List* + - es:Describe* + - es:Get* + - es:List* Resource: "*" - Sid: AllowGlobalResourceRestrictedActionsWhichIncurNoFees Effect: Allow @@ -127,6 +130,12 @@ Statement: - rds:CreateDBCluster - elasticache:CreateCacheCluster - redshift:CreateCluster + - es:AddTags + - es:CreateDomain + - es:DeleteDomain + - es:RemoveTags + - es:UpdateDomainConfig + - es:UpgradeDomain Resource: - 'arn:aws:rds:{{ aws_region }}:{{ aws_account_id }}:cluster:*' - 'arn:aws:elasticache:{{ aws_region }}:{{ aws_account_id }}:cluster:*' @@ -134,6 +143,7 @@ Statement: - 'arn:aws:elasticache:{{ aws_region }}:{{ aws_account_id }}:parametergroup:*' - 'arn:aws:elasticache:{{ aws_region }}:{{ aws_account_id }}:securitygroup:*' - 'arn:aws:redshift:{{ aws_region }}:{{ aws_account_id }}:cluster:*' + - 'arn:aws:es:{{ aws_region }}:{{ aws_account_id }}:domain:*' # This allows AWS Services to automatically create their Default Service Linked Roles # These have fixed policies and can only be assumed by the service itself. - Sid: AllowServiceLinkedRoleCreation @@ -186,16 +196,3 @@ Statement: - kafka:UntagResource - kafka:ListClusterOperations Resource: "*" - - Sid: OpenSearchCluster - Effect: Allow - Action: - - es:AddTags - - es:CreateDomain - - es:Describe* - - es:Get* - - es:List* - - es:DeleteDomain - - es:RemoveTags - - es:UpdateDomainConfig - - es:UpgradeDomain - Resource: "*" From d0750dac8edc25e4d4726d73048bc141c2b2f041 Mon Sep 17 00:00:00 2001 From: serosset Date: Fri, 28 Jan 2022 02:11:08 +0000 Subject: [PATCH 5/8] add class to delete opensearch domains --- aws/terminator/data_services.py | 43 +++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/aws/terminator/data_services.py b/aws/terminator/data_services.py index 11a58a56..b5b43fff 100644 --- a/aws/terminator/data_services.py +++ b/aws/terminator/data_services.py @@ -1,5 +1,7 @@ import datetime +import botocore.exceptions + from . import DbTerminator, Terminator, get_tag_dict_from_tag_list @@ -297,3 +299,44 @@ def age_limit(self): def terminate(self): self.client.delete_cluster(ClusterArn=self.id) + + +class OpenSearch(Terminator): + + @staticmethod + def create(credentials): + def get_available_clusters(client): + domains = [] + for domain in client.list_domain_names()['DomainNames']: + try: + domain_status = client.describe_domain(DomainName=domain['DomainName'])['DomainStatus'] + if not domain_status['Deleted']: + # 'Deleted' is true if a delete request has been received for the domain + # but resource cleanup is still in progress. + domain_config = client.describe_domain_config(DomainName=domain['DomainName']) + domain_status["CreationDate"] = domain_config['DomainConfig']['Status']['CreationDate'] + domains.append(domain_status) + except botocore.exceptions.ClientError as ex: + pass + return domains + + return Terminator._create(credentials, OpenSearch, 'opensearch', get_available_clusters) + + @property + def id(self): + return self.instance['DomainId'] + + @property + def name(self): + return self.instance['DomainName'] + + @property + def created_time(self): + return self.instance['CreationDate'] + + @property + def age_limit(self): + return datetime.timedelta(minutes=60) + + def terminate(self): + self.client.delete_domain(DomainName=self.name) From b43a9e063eecb12b1745dac56b40596e3eac4d82 Mon Sep 17 00:00:00 2001 From: serosset Date: Fri, 28 Jan 2022 22:01:52 +0000 Subject: [PATCH 6/8] add error handling --- aws/terminator/data_services.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/aws/terminator/data_services.py b/aws/terminator/data_services.py index b5b43fff..815dbbc8 100644 --- a/aws/terminator/data_services.py +++ b/aws/terminator/data_services.py @@ -3,7 +3,7 @@ import botocore.exceptions from . import DbTerminator, Terminator, get_tag_dict_from_tag_list - +from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code class DmsSubnetGroup(DbTerminator): @staticmethod @@ -316,7 +316,11 @@ def get_available_clusters(client): domain_config = client.describe_domain_config(DomainName=domain['DomainName']) domain_status["CreationDate"] = domain_config['DomainConfig']['Status']['CreationDate'] domains.append(domain_status) - except botocore.exceptions.ClientError as ex: + except is_boto3_error_code('ResourceNotFoundException'): + # Unlikely, but possible. The domain may have been deleted after invoking + # list_domain_names(). + pass + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError): # pylint: disable=duplicate-except pass return domains From 9428cbcc6d1563782592c9f7e40940354092220e Mon Sep 17 00:00:00 2001 From: serosset Date: Fri, 28 Jan 2022 22:17:06 +0000 Subject: [PATCH 7/8] add error handling --- aws/terminator/data_services.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws/terminator/data_services.py b/aws/terminator/data_services.py index 815dbbc8..695c8e22 100644 --- a/aws/terminator/data_services.py +++ b/aws/terminator/data_services.py @@ -5,6 +5,7 @@ from . import DbTerminator, Terminator, get_tag_dict_from_tag_list from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code + class DmsSubnetGroup(DbTerminator): @staticmethod def create(credentials): From d9cb5fac6663a1d0a966a79630bb2e54688577b7 Mon Sep 17 00:00:00 2001 From: serosset Date: Fri, 28 Jan 2022 22:23:48 +0000 Subject: [PATCH 8/8] add error handling --- aws/terminator/data_services.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/aws/terminator/data_services.py b/aws/terminator/data_services.py index 695c8e22..e48b91d2 100644 --- a/aws/terminator/data_services.py +++ b/aws/terminator/data_services.py @@ -3,7 +3,6 @@ import botocore.exceptions from . import DbTerminator, Terminator, get_tag_dict_from_tag_list -from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code class DmsSubnetGroup(DbTerminator): @@ -317,12 +316,10 @@ def get_available_clusters(client): domain_config = client.describe_domain_config(DomainName=domain['DomainName']) domain_status["CreationDate"] = domain_config['DomainConfig']['Status']['CreationDate'] domains.append(domain_status) - except is_boto3_error_code('ResourceNotFoundException'): + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError): # pylint: disable=duplicate-except # Unlikely, but possible. The domain may have been deleted after invoking # list_domain_names(). pass - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError): # pylint: disable=duplicate-except - pass return domains return Terminator._create(credentials, OpenSearch, 'opensearch', get_available_clusters)