feat(terraform-init): use account tags for cloudformation tags (#5239)

* feat(terraform-init): use account tags for cloudformation tags
* refactor(terraform-init): update tests tag values

Signed-off-by: Di Wang <dwan@redhat.com>

Author: hemslo

reconcile/gql_definitions/fragments/aws_organization.gql

@@ -0,0 +1,8 @@
+# qenerate: plugin=pydantic_v1
+
+fragment AWSOrganization on AWSOrganization_v1 {
+  payerAccount {
+    organizationAccountTags
+  }
+  tags
+}

reconcile/gql_definitions/fragments/aws_organization.py

@@ -0,0 +1,33 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class AWSAccountV1(ConfiguredBaseModel):
+    organization_account_tags: Optional[Json] = Field(..., alias="organizationAccountTags")
+
+
+class AWSOrganization(ConfiguredBaseModel):
+    payer_account: AWSAccountV1 = Field(..., alias="payerAccount")
+    tags: Json = Field(..., alias="tags")

reconcile/gql_definitions/terraform_init/aws_accounts.gql

@@ -15,5 +15,8 @@ query TerraformInitAWSAccounts {
     disable {
       integrations
     }
+    organization {
+      ...AWSOrganization
+    }
   }
 }

reconcile/gql_definitions/terraform_init/aws_accounts.py

@@ -17,10 +17,18 @@
     Json,
 )
 
+from reconcile.gql_definitions.fragments.aws_organization import AWSOrganization
 from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
 
 
 DEFINITION = """
+fragment AWSOrganization on AWSOrganization_v1 {
+  payerAccount {
+    organizationAccountTags
+  }
+  tags
+}
+
 fragment VaultSecret on VaultSecret_v1 {
   path
   field
@@ -43,6 +51,9 @@
     disable {
       integrations
     }
+    organization {
+      ...AWSOrganization
+    }
   }
 }
 """
@@ -70,6 +81,7 @@ class AWSAccountV1(ConfiguredBaseModel):
     resources_default_region: str = Field(..., alias="resourcesDefaultRegion")
     automation_token: VaultSecret = Field(..., alias="automationToken")
     disable: Optional[DisableClusterAutomationsV1] = Field(..., alias="disable")
+    organization: Optional[AWSOrganization] = Field(..., alias="organization")
 
 
 class TerraformInitAWSAccountsQueryData(ConfiguredBaseModel):

reconcile/terraform_init/integration.py

@@ -11,6 +11,7 @@
 from reconcile.terraform_init.merge_request import Renderer, create_parser
 from reconcile.terraform_init.merge_request_manager import MergeRequestManager, MrData
 from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.aws_account_tags import get_aws_account_tags
 from reconcile.typed_queries.external_resources import get_settings
 from reconcile.typed_queries.github_orgs import get_github_orgs
 from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
@@ -115,7 +116,7 @@ def reconcile_account(
         state_template: str,
         cloudformation_template: str,
         cloudformation_import_template: str,
-        tags: dict[str, str],
+        default_tags: dict[str, str],
     ) -> None:
         """
         Reconcile the terraform state for a given account.
@@ -140,7 +141,7 @@ def reconcile_account(
             state_template: str: Jinja2 template for the Terraform state configuration.
             cloudformation_template: str: CloudFormation template to create the S3 bucket.
             cloudformation_import_template: str: CloudFormation template to import existing S3 bucket.
-            tags: dict[str, str]: Tags to apply to the CloudFormation stack.
+            default_tags: dict[str, str]: Default tags to apply to the CloudFormation stack.
 
         Returns:
             None
@@ -151,6 +152,8 @@ def reconcile_account(
             else f"terraform-{account.name}"
         )
 
+        tags = default_tags | get_aws_account_tags(account.organization)
+
         if account.terraform_state is None:
             return self._provision_terraform_state(
                 aws_api=aws_api,
@@ -324,5 +327,5 @@ def run(self, dry_run: bool, defer: Callable | None = None) -> None:
                     state_template=state_template,
                     cloudformation_template=cloudformation_template,
                     cloudformation_import_template=cloudformation_import_template,
-                    tags=default_tags,
+                    default_tags=default_tags,
                 )

reconcile/test/fixtures/terraform_init/accounts.yml

@@ -21,6 +21,10 @@ accounts:
   resourcesDefaultRegion: us-east-1
   automationToken:
     path: path/to/creds
+  organization:
+    payerAccount:
+      organizationAccountTags: '{"common_key": "not_final_common_value", "payer_key": "payer_value"}'
+    tags: '{"common_key": "final_common_value", "account_key": "account_value"}'
 
 # all accounts below must be ignored
 - name: no-terraform-username

reconcile/test/terraform_init/test_terraform_init_integration.py

@@ -85,7 +85,7 @@ def test_terraform_init_integration_reconcile_account_with_new_account(
         state_template="{{bucket_name}}",
         cloudformation_template="cloudformation_template",
         cloudformation_import_template="cloudformation_import_template",
-        tags={"env": "test"},
+        default_tags={"env": "test"},
     )
 
     aws_api.cloudformation.create_stack.assert_called_once_with(
@@ -119,7 +119,7 @@ def test_terraform_init_integration_reconcile_account_with_new_account_dry_run(
         state_template="{{bucket_name}}",
         cloudformation_template="cloudformation_template",
         cloudformation_import_template="cloudformation_import_template",
-        tags={"env": "test"},
+        default_tags={"env": "test"},
     )
 
     aws_api.cloudformation.create_stack.assert_not_called()
@@ -149,7 +149,7 @@ def test_terraform_init_integration_reconcile_account_when_import_stack(
         state_template="{{bucket_name}}",
         cloudformation_template="cloudformation_template",
         cloudformation_import_template="cloudformation_import_template",
-        tags={"env": "test"},
+        default_tags={"env": "test"},
     )
 
     aws_api.cloudformation.get_stack.assert_called_once_with(
@@ -160,13 +160,23 @@ def test_terraform_init_integration_reconcile_account_when_import_stack(
         change_set_name="import-existing-bucket",
         template_body="cloudformation_import_template",
         parameters={"BucketName": "existing-bucket"},
-        tags={"env": "test"},
+        tags={
+            "account_key": "account_value",
+            "common_key": "final_common_value",
+            "env": "test",
+            "payer_key": "payer_value",
+        },
     )
     aws_api.cloudformation.update_stack.assert_called_once_with(
         stack_name="existing-bucket",
         template_body="cloudformation_template",
         parameters={"BucketName": "existing-bucket"},
-        tags={"env": "test"},
+        tags={
+            "account_key": "account_value",
+            "common_key": "final_common_value",
+            "env": "test",
+            "payer_key": "payer_value",
+        },
     )
     merge_request_manager.create_merge_request.assert_not_called()
 
@@ -188,7 +198,7 @@ def test_terraform_init_integration_reconcile_account_when_import_stack_dry_run(
         state_template="{{bucket_name}}",
         cloudformation_template="cloudformation_template",
         cloudformation_import_template="cloudformation_import_template",
-        tags={"env": "test"},
+        default_tags={"env": "test"},
     )
 
     aws_api.cloudformation.get_stack.assert_called_once_with(
@@ -208,7 +218,12 @@ def test_terraform_init_integration_reconcile_account_when_tags_mismatch(
     account = next(a for a in aws_accounts if a.name == "terraform-state-already-set")
     aws_api.cloudformation.get_stack.return_value = {
         "StackName": "terraform-terraform-state-already-set",
-        "Tags": [{"Key": "env", "Value": "test-old"}],
+        "Tags": [
+            {"Key": "account_key", "Value": "account_value"},
+            {"Key": "common_key", "Value": "final_common_value"},
+            {"Key": "env", "Value": "test-old"},
+            {"Key": "payer_key", "Value": "payer_value"},
+        ],
     }
 
     intg.reconcile_account(
@@ -219,7 +234,7 @@ def test_terraform_init_integration_reconcile_account_when_tags_mismatch(
         state_template="{{bucket_name}}",
         cloudformation_template="cloudformation_template",
         cloudformation_import_template="cloudformation_import_template",
-        tags={"env": "test"},
+        default_tags={"env": "test"},
     )
 
     aws_api.cloudformation.get_stack.assert_called_once_with(
@@ -229,7 +244,12 @@ def test_terraform_init_integration_reconcile_account_when_tags_mismatch(
         stack_name="existing-bucket",
         template_body="cloudformation_template",
         parameters={"BucketName": "existing-bucket"},
-        tags={"env": "test"},
+        tags={
+            "account_key": "account_value",
+            "common_key": "final_common_value",
+            "env": "test",
+            "payer_key": "payer_value",
+        },
     )
     aws_api.cloudformation.create_stack.assert_not_called()
     aws_api.cloudformation.get_template.assert_not_called()
@@ -245,7 +265,12 @@ def test_terraform_init_integration_reconcile_account_when_tags_mismatch_dry_run
     account = next(a for a in aws_accounts if a.name == "terraform-state-already-set")
     aws_api.cloudformation.get_stack.return_value = {
         "StackName": "terraform-terraform-state-already-set",
-        "Tags": [{"Key": "env", "Value": "test-old"}],
+        "Tags": [
+            {"Key": "account_key", "Value": "account_value"},
+            {"Key": "common_key", "Value": "final_common_value"},
+            {"Key": "env", "Value": "test-old"},
+            {"Key": "payer_key", "Value": "payer_value"},
+        ],
     }
 
     intg.reconcile_account(
@@ -256,7 +281,7 @@ def test_terraform_init_integration_reconcile_account_when_tags_mismatch_dry_run
         state_template="{{bucket_name}}",
         cloudformation_template="cloudformation_template",
         cloudformation_import_template="cloudformation_import_template",
-        tags={"env": "test"},
+        default_tags={"env": "test"},
     )
 
     aws_api.cloudformation.get_stack.assert_called_once_with(
@@ -277,7 +302,12 @@ def test_terraform_init_integration_reconcile_account_when_template_body_mismatc
     account = next(a for a in aws_accounts if a.name == "terraform-state-already-set")
     aws_api.cloudformation.get_stack.return_value = {
         "StackName": "terraform-terraform-state-already-set",
-        "Tags": [{"Key": "env", "Value": "test"}],
+        "Tags": [
+            {"Key": "account_key", "Value": "account_value"},
+            {"Key": "common_key", "Value": "final_common_value"},
+            {"Key": "env", "Value": "test"},
+            {"Key": "payer_key", "Value": "payer_value"},
+        ],
     }
     aws_api.cloudformation.get_template_body.return_value = "old_template_body"
 
@@ -289,7 +319,7 @@ def test_terraform_init_integration_reconcile_account_when_template_body_mismatc
         state_template="{{bucket_name}}",
         cloudformation_template="cloudformation_template",
         cloudformation_import_template="cloudformation_import_template",
-        tags={"env": "test"},
+        default_tags={"env": "test"},
     )
 
     aws_api.cloudformation.get_stack.assert_called_once_with(
@@ -302,7 +332,12 @@ def test_terraform_init_integration_reconcile_account_when_template_body_mismatc
         stack_name="existing-bucket",
         template_body="cloudformation_template",
         parameters={"BucketName": "existing-bucket"},
-        tags={"env": "test"},
+        tags={
+            "account_key": "account_value",
+            "common_key": "final_common_value",
+            "env": "test",
+            "payer_key": "payer_value",
+        },
     )
     aws_api.cloudformation.create_stack.assert_not_called()
     merge_request_manager.create_merge_request.assert_not_called()
@@ -317,7 +352,12 @@ def test_terraform_init_integration_reconcile_account_when_template_body_mismatc
     account = next(a for a in aws_accounts if a.name == "terraform-state-already-set")
     aws_api.cloudformation.get_stack.return_value = {
         "StackName": "terraform-terraform-state-already-set",
-        "Tags": [{"Key": "env", "Value": "test"}],
+        "Tags": [
+            {"Key": "account_key", "Value": "account_value"},
+            {"Key": "env", "Value": "test"},
+            {"Key": "common_key", "Value": "final_common_value"},
+            {"Key": "payer_key", "Value": "payer_value"},
+        ],
     }
     aws_api.cloudformation.get_template_body.return_value = "old_template_body"
 
@@ -329,7 +369,7 @@ def test_terraform_init_integration_reconcile_account_when_template_body_mismatc
         state_template="{{bucket_name}}",
         cloudformation_template="cloudformation_template",
         cloudformation_import_template="cloudformation_import_template",
-        tags={"env": "test"},
+        default_tags={"env": "test"},
     )
 
     aws_api.cloudformation.get_stack.assert_called_once_with(
@@ -352,7 +392,12 @@ def test_terraform_init_integration_reconcile_account_when_no_changes(
     account = next(a for a in aws_accounts if a.name == "terraform-state-already-set")
     aws_api.cloudformation.get_stack.return_value = {
         "StackName": "terraform-terraform-state-already-set",
-        "Tags": [{"Key": "env", "Value": "test"}],
+        "Tags": [
+            {"Key": "account_key", "Value": "account_value"},
+            {"Key": "common_key", "Value": "final_common_value"},
+            {"Key": "env", "Value": "test"},
+            {"Key": "payer_key", "Value": "payer_value"},
+        ],
     }
     aws_api.cloudformation.get_template_body.return_value = "cloudformation_template"
 
@@ -364,7 +409,7 @@ def test_terraform_init_integration_reconcile_account_when_no_changes(
         state_template="{{bucket_name}}",
         cloudformation_template="cloudformation_template",
         cloudformation_import_template="cloudformation_import_template",
-        tags={"env": "test"},
+        default_tags={"env": "test"},
     )
 
     aws_api.cloudformation.get_stack.assert_called_once_with(