Merge remote-tracking branch 'refs/remotes/origin/main'

Includes a (new) Reference Module and updates to the testing sections for the new module

Author: SohamMajumder

data-collection/deploy/account-collector.yaml

@@ -128,7 +128,9 @@ Resources:
           ZipFile: |
             import os
             import json
+            import uuid
             import logging
+            from datetime import datetime
             from functools import partial
 
             import boto3
@@ -141,6 +143,9 @@ Resources:
             EXCLUDED_ACCOUNT_LIST_KEY = os.environ.get('EXCLUDED_ACCOUNT_LIST_KEY')
             EUC_ACCOUNTS = os.environ.get('EUC_ACCOUNT_IDS', '').strip()
             TMP_FILE = "/tmp/data.json"
+            START_TIME = str(datetime.now().isoformat())
+            LINKED_ACCOUNT_LIST_KEY = os.environ.get('LINKED_ACCOUNT_LIST_KEY', 'linked-account-list.json')
+            PAYER_ACCOUNT_LIST_KEY = os.environ.get('PAYER_ACCOUNT_LIST_KEY', 'payer-account-list.json')
 
             logger = logging.getLogger(__name__)
             logger.setLevel(getattr(logging, os.environ.get('LOG_LEVEL', 'INFO').upper(), logging.INFO))
@@ -156,6 +161,8 @@ Resources:
                     logger.error(message)
                     raise Exception(message) #pylint: disable=broad-exception-raised
 
+                module = event.get("module", '').lower()
+                params = event.get("params", '')
                 functions = { # keep keys same as boto3 services
                     'linked': iterate_linked_accounts,
                     'euc': partial(iterate_accounts_with_filter, EUC_ACCOUNTS),
@@ -164,7 +171,7 @@ Resources:
                     'compute-optimizer': partial(iterate_admins_accounts, 'compute-optimizer'),
                     'backup': partial(iterate_admins_accounts, 'backup'),
                 }
-                account_type = event.get("Type", '').lower()
+                account_type = event.get("type", '').lower()
                 if account_type not in functions:
                     raise Exception(f"Lambda event must have 'Type' parameter with value = ({list(functions.keys())})") #pylint: disable=broad-exception-raised
 
@@ -173,6 +180,14 @@ Resources:
                     count = 0
                     f.write("[\n")
                     for account in account_iterator():
+                        account['main_exe_uuid'] = event.get("main_exe_uuid", str(uuid.uuid4()))
+                        account['module'] = module
+                        account['bucket'] = BUCKET
+                        account['dc_account'] = boto3.client('sts').get_caller_identity()['Account']
+                        account['dc_region'] = boto3.session.Session().region_name
+                        account['params'] = params
+                        account['prefix'] = RESOURCE_PREFIX
+                        account['stack_version'] = event.get("stack_version", '')
                         if count > 0:
                             f.write(",\n")
                         f.write(json.dumps(account))
@@ -183,6 +198,7 @@ Resources:
                     raise Exception('No accounts found. Check the log.') #pylint: disable=broad-exception-raised
 
                 key = f"account-list/{account_type}-account-list.json"
+                key = f"account-collector/{module+'-'+(params+'-' if params else '')+(LINKED_ACCOUNT_LIST_KEY if account_type == 'linked' else PAYER_ACCOUNT_LIST_KEY)}"
                 s3 = boto3.client('s3')
                 s3.upload_file(TMP_FILE, Bucket=BUCKET, Key=key)
 
@@ -285,6 +301,7 @@ Resources:
                     aws_secret_access_key=credentials['SecretAccessKey'],
                     aws_session_token=credentials['SessionToken'],
                 )
+
       Handler: 'index.lambda_handler'
       MemorySize: 2688
       Timeout: 600
@@ -298,6 +315,7 @@ Resources:
           PREDEF_ACCOUNT_LIST_KEY: "account-list/account-list"
           EXCLUDED_ACCOUNT_LIST_KEY: "account-list/excluded-linked-account-list.csv"
           EUC_ACCOUNT_IDS: !Ref EUCAccountIDs
+
     Metadata:
       cfn_nag:
         rules_to_suppress:
@@ -310,4 +328,4 @@ Resources:
     Type: AWS::Logs::LogGroup
     Properties:
       LogGroupName: !Sub "/aws/lambda/${LambdaFunction}"
-      RetentionInDays: 60
+      RetentionInDays: 60

data-collection/deploy/deploy-data-collection.yaml

@@ -1,6 +1,6 @@
-# https://github.com/awslabs/cid-data-collection-framework/blob/main/data-collection/v3.10.0/deploy/deploy-data-read-permissions.yaml
+# https://github.com/awslabs/cid-data-collection-framework/blob/main/data-collection/v3.11.0/deploy/deploy-data-read-permissions.yaml
 AWSTemplateFormatVersion: '2010-09-09'
-Description: CID Data Collection - All-in-One for Management Account v3.10.0 - AWS Solution SO9011
+Description: CID Data Collection - All-in-One for Management Account v3.11.0 - AWS Solution SO9011
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -190,7 +190,7 @@ Resources:
   DataCollectorMgmtAccountReadStack:
     Type: AWS::CloudFormation::Stack
     Properties:
-      TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/v3.10.0/deploy-in-management-account.yaml"
+      TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/v3.11.0/deploy-in-management-account.yaml"
       Parameters:
         DataCollectionAccountID: !Ref DataCollectionAccountID
         ManagementAccountRole: !Ref ManagementAccountRole
@@ -206,7 +206,7 @@ Resources:
     Type: AWS::CloudFormation::Stack
     Condition: DeployModuleReadInMgmt
     Properties:
-      TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/v3.10.0/deploy-in-linked-account.yaml"
+      TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/v3.11.0/deploy-in-linked-account.yaml"
       Parameters:
         DataCollectionAccountID: !Ref DataCollectionAccountID
         MultiAccountRoleName: !Ref MultiAccountRoleName
@@ -223,7 +223,7 @@ Resources:
   DataCollectorOrgAccountModulesReadStackSet:
     Type: AWS::CloudFormation::StackSet
     Properties:
-      Description: "StackSet in charge of deploying read roles across organization accounts v3.10.0"
+      Description: "StackSet in charge of deploying read roles across organization accounts v3.11.0"
       PermissionModel: SERVICE_MANAGED
       AutoDeployment:
         Enabled: true
@@ -268,4 +268,4 @@ Resources:
         - CAPABILITY_IAM
         - CAPABILITY_NAMED_IAM
       StackSetName: !Sub "StackSet-${AWS::AccountId}-OptimizationDataRole"
-      TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/v3.10.0/deploy-in-linked-account.yaml"
+      TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/v3.11.0/deploy-in-linked-account.yaml"

data-collection/deploy/deploy-data-read-permissions.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: CID Data Collection - Role for Linked Account v3.10.0
+Description: CID Data Collection - Role for Linked Account v3.11.0
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:

data-collection/deploy/deploy-in-linked-account.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: CID Data Collection - Role for Management Account v3.10.0
+Description: CID Data Collection - Role for Management Account v3.11.0
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:

data-collection/deploy/deploy-in-management-account.yaml

@@ -932,4 +932,4 @@ Resources:
     Type: Custom::LambdaAnalyticsExecutor
     Properties:
       ServiceToken: !Ref LambdaAnalyticsARN
-      Name: !Ref CFDataName
+      Name: !Ref CFDataName

data-collection/deploy/module-aws-feeds.yaml

@@ -320,6 +320,7 @@ Resources:
             DeployRegion: !Ref AWS::Region
             Account: !Ref AWS::AccountId
             Prefix: !Ref ResourcePrefix
+            Bucket: !Ref DestinationBucket
       'RefreshSchedule${AwsObject}':
         Type: AWS::Scheduler::Schedule
         Properties:
@@ -338,4 +339,4 @@ Resources:
     Type: Custom::LambdaAnalyticsExecutor
     Properties:
       ServiceToken: !Ref LambdaAnalyticsARN
-      Name: !Ref CFDataName
+      Name: !Ref CFDataName

data-collection/deploy/module-backup.yaml

@@ -290,6 +290,7 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
+        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-budgets.yaml

@@ -501,6 +501,7 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
+        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-compute-optimizer.yaml

@@ -425,6 +425,7 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
+        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: "AWS::Scheduler::Schedule"