Pre-main merge instrumentation

Author: esc1144

data-collection/deploy/account-collector.yaml

@@ -159,10 +159,11 @@ Resources:
                 global MODULE
                 logger.info(f"Incoming event: {event}")
                 sub_uuid = {"lambda-request-id": context.aws_request_id, "lambda-log-group": context.log_group_name, "lambda-log-stream": context.log_stream_name}
-                account_type = event.get("Type", '').lower()
-                MODULE = event.get("Module", '').lower()
-                main_exe_uuid = event.get("MainExeUuid", str(uuid.uuid4()))
-                params = event.get("Params", "")
+                account_type = event.get("type", '').lower()
+                MODULE = event.get("module", '').lower()
+                main_exe_uuid = event.get("main_exe_uuid", str(uuid.uuid4()))
+                params = event.get("params", "")
+                stack_version = event.get("stack_version", "")
                 try:
                     # need to confirm that the Lambda concurrency limit is sufficient to avoid throttling
                     lambda_limit = boto3.client('lambda').get_account_settings()['AccountLimit']['ConcurrentExecutions']
@@ -182,6 +183,8 @@ Resources:
                         account_type = None
                         raise Exception(STATUS_NOT_ACCEPTABLE) #pylint: disable=broad-exception-raised
 
+                    context = "linked" if account_type == "linked" or account_type == "euc" else "payer"
+
                     account_iterator = functions[account_type]
                     logger.info(f"Looking for accounts")
                     with open(TMP_FILE, "w", encoding='utf-8') as f:
@@ -195,6 +198,7 @@ Resources:
                             account['dc_region'] = DC_REGION
                             account['params'] = params
                             account['prefix'] = PREFIX
+                            account['stack_version'] = stack_version
                             if count > 0:
                                 f.write(",\n")
                             f.write(json.dumps(account))
@@ -209,7 +213,7 @@ Resources:
                     s3.upload_file(TMP_FILE, Bucket=BUCKET, Key=key)
                     location = f"s3://{BUCKET}/{key}"
                     log_entry = create_log_entry(module_function='account-collector-lambda', params=params, region="us-east-1", record_count=count,
-                                                location=location, main_exe_uuid=main_exe_uuid, record_context="account", sub_uuid=sub_uuid)
+                                                location=location, main_exe_uuid=main_exe_uuid, record_context=f"{context} account(s)", sub_uuid=sub_uuid, stack_version=stack_version)
                     return {'statusCode': 200, 'accountList': key, 'bucket': BUCKET, 'logEntry': log_entry['logEntry']}
                 except Exception as exc: #pylint: disable=broad-exception-caught
                     exc_msg = str(exc)
@@ -227,7 +231,8 @@ Resources:
                     else:
                         status_code = None
                         description = None
-                    create_log_entry(module_function='account-collector-lambda', params=params, region="us-east-1", status_code=status_code, description=description, error=exc, main_exe_uuid=main_exe_uuid, sub_uuid=sub_uuid)
+                    create_log_entry(module_function='account-collector-lambda', params=params, region="us-east-1", status_code=status_code,
+                                    description=description, error=exc, main_exe_uuid=main_exe_uuid, sub_uuid=sub_uuid, stack_version=stack_version)
                     raise exc
 
             def get_all_payers():
@@ -244,8 +249,6 @@ Resources:
                             account_id = ssm.get_parameter(Name=ssm_key)['Parameter']['Value']
                         except ssm.exceptions.ParameterNotFound:
                             logger.info(f'Not found ssm parameter {ssm_key}. Will use Management Account Id {payer_id}')
-                    yield {"account": json.dumps({'account_id': account_id, 'account_name': '', 'payer_id': payer_id})}
-                            logger.warning(f'Not found ssm parameter {ssm_key}. Will use Management Account Id {payer_id}')
                     yield {"account": json.dumps({'account_id': account_id, 'account_name': '', 'payer_id': payer_id}), "main_exe_uuid": ""}
 
             def iterate_linked_accounts():
@@ -334,7 +337,7 @@ Resources:
                 )
 
             def create_log_entry(payer_id="", account_id=None, status_code=None, region="", module_function="data-collection-lambda", sub_code="",
-                                params="", record_count=0, record_context="", description=None, location="", error=None, main_exe_uuid="", sub_uuid={}, is_summary=False, store_it=True): # pylint: disable=too-many-locals
+                                params="", record_count=0, record_context="", description=None, location="", error=None, main_exe_uuid="", sub_uuid={}, is_summary=False, store_it=True, stack_version="", subversion=""): # pylint: disable=too-many-locals
                 """Format log entry for logging."""
                 status_code, description = status_handler(error, record_count, is_summary, status_code, description, record_context)
                 log_entry = {
@@ -354,7 +357,9 @@ Resources:
                     "DataLocation": location if record_count > 0 else "",
                     "MainExeUuid": main_exe_uuid,
                     "SubUuid": sub_uuid,
-                    "Service": "Lambda"
+                    "Service": "Lambda",
+                    "StackVersion": stack_version,
+                    "SubVersion": subversion
                 }
                 if status_code >= 400:
                     logger.error(description)

data-collection/deploy/deploy-data-collection.yaml

@@ -120,14 +120,14 @@ Mappings:
     eu-west-3:      {CodeBucket: aws-managed-cost-intelligence-dashboards-eu-west-3 }
     sa-east-1:      {CodeBucket: aws-managed-cost-intelligence-dashboards-sa-east-1 }
     us-east-1:      {CodeBucket: aws-managed-cost-intelligence-dashboards-us-east-1 }
-    us-east-2:      {CodeBucket: aws-managed-cost-intelligence-dashboards-us-east-2 }
+    us-east-2:      {CodeBucket: aws-managed-cost-intel ligence-dashboards-us-east-2 }
     us-west-1:      {CodeBucket: aws-managed-cost-intelligence-dashboards-us-west-1 }
     us-west-2:      {CodeBucket: aws-managed-cost-intelligence-dashboards-us-west-2 }
   StepFunctionCode:
     main-state-machine-v4:           {TemplatePath: cfn/data-collection/source/step-functions/main-state-machine-v4.json}
     crawler-v2:                      {TemplatePath: cfn/data-collection/source/step-functions/crawler-state-machine-v2.json}
     standalone-v2:                   {TemplatePath: cfn/data-collection/source/step-functions/standalone-state-machine-v2.json}
-    health-detail-state-machine-v2-1:  {TemplatePath: cfn/data-collection/source/step-functions/health-detail-state-machine-v2-1.json}
+    health-detail-state-machine-v2:  {TemplatePath: cfn/data-collection/source/step-functions/health-detail-state-machine-v2.json}
 
 Parameters:
   DestinationBucket:
@@ -897,14 +897,26 @@ Resources:
                   - s3:GetObject
                   - s3:PutObject
                 Resource: !Sub '${S3Bucket.Arn}/*'
-        - PolicyName: "CloudWatch-ErrorLogging" #Used for logging processing errors to CloudWatch
+        - PolicyName: "CloudWatchErrorLogging" #Used for logging processing errors to CloudWatch
           PolicyDocument:
             Version: "2012-10-17"
             Statement:
               - Effect: "Allow"
                 Action:
                   - cloudwatch:PutMetricData
                 Resource: '*'
+        - PolicyName: "CloudFormationStackVersionRead" #Used for reading the deployed version number of the stack
+          PolicyDocument:
+            Version: "2012-10-17"
+            Statement:
+              - Effect: "Allow"
+                Action:
+                  - cloudformation:ListStacks
+                Resource: '*'
+              - Effect: "Allow"
+                Action:
+                  - cloudformation:DescribeStacks
+                Resource: !Ref "AWS::StackId"
 
   StepFunctionExecutionRoleInvokeAccountCollectorPolicy:
     Type: 'AWS::IAM::Policy'
@@ -984,6 +996,8 @@ Resources:
           - { Name: mainexeuuid,                 Type: string }
           - { Name: subuuid,                     Type: string }
           - { Name: service,                     Type: string }
+          - { Name: stackversion,                Type: string }
+          - { Name: subversion,                  Type: string }
           InputFormat: org.apache.hadoop.mapred.TextInputFormat
           OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
           Location: !Sub "s3://${DestinationBucket}${AWS::AccountId}/logs/modules/"
@@ -1022,6 +1036,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   RightsizeModule:
     Type: AWS::CloudFormation::Stack
@@ -1043,6 +1058,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   CostAnomalyModule:
     Type: AWS::CloudFormation::Stack
@@ -1065,6 +1081,7 @@ Resources:
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   SupportCasesModule:
     Type: AWS::CloudFormation::Stack
@@ -1086,6 +1103,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   BackupModule:
     Type: AWS::CloudFormation::Stack
@@ -1107,6 +1125,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   InventoryCollectorModule:
     Type: AWS::CloudFormation::Stack
@@ -1134,6 +1153,7 @@ Resources:
             - RegionsInScopeIsEmpty
             - !Sub "${AWS::Region}"
             - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
+        StackID: !Ref "AWS::StackId"
 
   PricingModule:
     Type: AWS::CloudFormation::Stack
@@ -1157,6 +1177,7 @@ Resources:
             - RegionsInScopeIsEmpty
             - !Sub "${AWS::Region}"
             - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
+        StackID: !Ref "AWS::StackId"
 
   ComputeOptimizerModule:
     Type: AWS::CloudFormation::Stack
@@ -1182,6 +1203,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   EcsChargebackModule:
     Type: AWS::CloudFormation::Stack
@@ -1208,6 +1230,7 @@ Resources:
             - RegionsInScopeIsEmpty
             - !Sub "${AWS::Region}"
             - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
+        StackID: !Ref "AWS::StackId"
 
   RDSUsageModule:
     Type: AWS::CloudFormation::Stack
@@ -1234,6 +1257,7 @@ Resources:
             - RegionsInScopeIsEmpty
             - !Sub "${AWS::Region}"
             - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
+        StackID: !Ref "AWS::StackId"
 
   EUCUsageModule:
     Type: AWS::CloudFormation::Stack
@@ -1260,6 +1284,7 @@ Resources:
             - RegionsInScopeIsEmpty
             - !Sub "${AWS::Region}"
             - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
+        StackID: !Ref "AWS::StackId"
 
   OrgDataModule:
     Type: AWS::CloudFormation::Stack
@@ -1281,6 +1306,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   BudgetsModule:
     Type: AWS::CloudFormation::Stack
@@ -1302,6 +1328,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   TransitGatewayModule:
     Type: AWS::CloudFormation::Stack
@@ -1328,6 +1355,7 @@ Resources:
             - RegionsInScopeIsEmpty
             - !Sub "${AWS::Region}"
             - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
+        StackID: !Ref "AWS::StackId"
 
   AWSFeedsModule:
     Type: AWS::CloudFormation::Stack
@@ -1347,6 +1375,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, standalone-v2, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   HealthEventsModule:
     Type: AWS::CloudFormation::Stack
@@ -1368,7 +1397,8 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
-        DetailStepFunctionTemplate: !FindInMap [StepFunctionCode, health-detail-state-machine-v2-1, TemplatePath]
+        DetailStepFunctionTemplate: !FindInMap [StepFunctionCode, health-detail-state-machine-v2, TemplatePath]
+        StackID: !Ref "AWS::StackId"
 
   LicenseManagerModule:
     Type: AWS::CloudFormation::Stack
@@ -1390,6 +1420,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   ServiceQuotasModule:
     Type: AWS::CloudFormation::Stack
@@ -1416,6 +1447,7 @@ Resources:
             - RegionsInScopeIsEmpty
             - !Sub "${AWS::Region}"
             - !Join [ '', !Split [ ' ', !Ref RegionsInScope  ] ] # remove spaces
+        StackID: !Ref "AWS::StackId"
 
   QuickSightModule:
     Type: AWS::CloudFormation::Stack
@@ -1435,6 +1467,7 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, standalone-v2, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
+        StackID: !Ref "AWS::StackId"
 
   AccountCollector:
     Type: AWS::CloudFormation::Stack

data-collection/deploy/module-aws-feeds.yaml

@@ -1,6 +1,9 @@
 AWSTemplateFormatVersion: '2010-09-09'
 Description: Restrieves AWS Feeds like what's new, blog posts, youtube videos and security bulletin
 Parameters:
+  StackID:
+    Type: String
+    Description: Arn of the main stack
   DatabaseName:
     Type: String
     Description: Name of the Athena database to be created

data-collection/deploy/module-backup.yaml

@@ -2,6 +2,9 @@ AWSTemplateFormatVersion: "2010-09-09"
 Description: Retrieves AWS Backup details across AWS organization
 Transform: 'AWS::LanguageExtensions'
 Parameters:
+  StackID:
+    Type: String
+    Description: Arn of the main stack
   DatabaseName:
     Type: String
     Description: Name of the Athena database to be created to hold lambda information
@@ -215,6 +218,7 @@ Resources:
               count = 0
               region = "us-east-1" #FIXME: what about other regions?
               main_exe_uuid = event.get("main_exe_uuid", str(uuid.uuid4()))
+              stack_version = event.get("stack_version", "")
 
               try:
                   account = json.loads(event.get("account","{}"))
@@ -254,19 +258,19 @@ Resources:
                   count, location = store_to_s3(flatten_data_iterator, s3_prefix)
                   description = f"Searched range from {start_date} to {end_date}"
                   return create_log_entry(payer_id=payer_id, account_id=account_id, region=region, params=name, record_count=count, location=location,
-                                          main_exe_uuid=main_exe_uuid, sub_uuid=sub_uuid, description=description)
+                                          main_exe_uuid=main_exe_uuid, sub_uuid=sub_uuid, description=description, stack_version=stack_version)
 
               except Exception as exc: #pylint: disable=broad-exception-caught
                   is_not_activated = 'Insufficient privileges to perform this action.' in str(exc)
                   status_code = STATUS_FORBIDDEN if is_not_activated else None
                   description = None if not is_not_activated else ('You need to activate cross account jobs monitoring. '
                                                                   'See https://docs.aws.amazon.com/aws-backup/latest/devguide/manage-cross-account.html#enable-cross-account')
                   create_log_entry(payer_id=payer_id, account_id=account_id, region=region, params=name, record_count=count,
-                                  main_exe_uuid=main_exe_uuid, sub_uuid=sub_uuid, description=description, status_code=status_code, error=exc)
+                                  main_exe_uuid=main_exe_uuid, sub_uuid=sub_uuid, description=description, status_code=status_code, error=exc, stack_version=stack_version)
                   raise exc
 
           def create_log_entry(payer_id="", account_id=None, status_code=None, region="", module_function="data-collection-lambda", sub_code="",
-                              params="", record_count=0, description=None, location="", error=None, main_exe_uuid="", sub_uuid={}, is_summary=False, store_it=True): # pylint: disable=too-many-locals
+                              params="", record_count=0, record_context="", description=None, location="", error=None, main_exe_uuid="", sub_uuid={}, is_summary=False, store_it=True, stack_version="", subversion=""): # pylint: disable=too-many-locals
               """Format log entry for logging."""
               status_code, description = status_handler(error, record_count, is_summary, status_code, description)
               log_entry = {
@@ -286,7 +290,9 @@ Resources:
                   "DataLocation": location if record_count > 0 else "",
                   "MainExeUuid": main_exe_uuid,
                   "SubUuid": sub_uuid,
-                  "Service": "Lambda"
+                  "Service": "Lambda",
+                  "StackVersion": stack_version,
+                  "SubVersion": subversion
               }
               if status_code >= 400:
                   logger.error(description)
@@ -403,6 +409,7 @@ Resources:
             Account: !Ref AWS::AccountId
             Prefix: !Ref ResourcePrefix
             Bucket: !Ref DestinationBucket
+            DataCollectionStackID: !Ref StackID
       'RefreshSchedule${AwsObject}':
         Type: AWS::Scheduler::Schedule
         Properties: