Instrumentation rollback update (#362)

* Data Collection logging phase 1

* codify new SF version name

* tweak account collector logging

* Sync account collector with fix for manual account list

* Add backoff/retry to standalone state machine

* add more retry buffer

* Misc cleanup

* Temporarily re-indent the over indent to make the diff easier to read

* Switch to pre-defined table

* Fix merge issue

* Merge main, normalize linked sf code file names

* Interim commit

* Checkpoint

* beta

* Pre-main merge instrumentation

* Post merge main and cleanup

* Fix misplaced space insertion

* Refine Health detail execution logging

* Add policy condition for CW logging

* Update data-collection/deploy/deploy-data-collection.yaml

* Tweak to output error message

* Scale back Lambda-based monitoring

* Renenable testing deploy of CaseSummary

* Misc cleanup for easier merge

* Misc cleanup for easier merge, 2

* Health SF correction from reversion of code

* Health mod cleanup

* Merge for push

* post-merge

* fix parameter

* Update data-collection/deploy/module-health-events.yaml

---------

Co-authored-by: Iakov Gan <[email protected]>
Co-authored-by: Iakov GAN <[email protected]>

Author: esc1144

data-collection/deploy/account-collector.yaml

@@ -128,7 +128,9 @@ Resources:
           ZipFile: |
             import os
             import json
+            import uuid
             import logging
+            from datetime import datetime
             from functools import partial
 
             import boto3
@@ -141,6 +143,9 @@ Resources:
             EXCLUDED_ACCOUNT_LIST_KEY = os.environ.get('EXCLUDED_ACCOUNT_LIST_KEY')
             EUC_ACCOUNTS = os.environ.get('EUC_ACCOUNT_IDS', '').strip()
             TMP_FILE = "/tmp/data.json"
+            START_TIME = str(datetime.now().isoformat())
+            LINKED_ACCOUNT_LIST_KEY = os.environ.get('LINKED_ACCOUNT_LIST_KEY', 'linked-account-list.json')
+            PAYER_ACCOUNT_LIST_KEY = os.environ.get('PAYER_ACCOUNT_LIST_KEY', 'payer-account-list.json')
 
             logger = logging.getLogger(__name__)
             logger.setLevel(getattr(logging, os.environ.get('LOG_LEVEL', 'INFO').upper(), logging.INFO))
@@ -156,6 +161,8 @@ Resources:
                     logger.error(message)
                     raise Exception(message) #pylint: disable=broad-exception-raised
 
+                module = event.get("module", '').lower()
+                params = event.get("params", '')
                 functions = { # keep keys same as boto3 services
                     'linked': iterate_linked_accounts,
                     'euc': partial(iterate_accounts_with_filter, EUC_ACCOUNTS),
@@ -164,7 +171,7 @@ Resources:
                     'compute-optimizer': partial(iterate_admins_accounts, 'compute-optimizer'),
                     'backup': partial(iterate_admins_accounts, 'backup'),
                 }
-                account_type = event.get("Type", '').lower()
+                account_type = event.get("type", '').lower()
                 if account_type not in functions:
                     raise Exception(f"Lambda event must have 'Type' parameter with value = ({list(functions.keys())})") #pylint: disable=broad-exception-raised
 
@@ -173,6 +180,14 @@ Resources:
                     count = 0
                     f.write("[\n")
                     for account in account_iterator():
+                        account['main_exe_uuid'] = event.get("main_exe_uuid", str(uuid.uuid4()))
+                        account['module'] = module
+                        account['bucket'] = BUCKET
+                        account['dc_account'] = boto3.client('sts').get_caller_identity()['Account']
+                        account['dc_region'] = boto3.session.Session().region_name
+                        account['params'] = params
+                        account['prefix'] = RESOURCE_PREFIX
+                        account['stack_version'] = event.get("stack_version", '')
                         if count > 0:
                             f.write(",\n")
                         f.write(json.dumps(account))
@@ -183,6 +198,7 @@ Resources:
                     raise Exception('No accounts found. Check the log.') #pylint: disable=broad-exception-raised
 
                 key = f"account-list/{account_type}-account-list.json"
+                key = f"account-collector/{module+'-'+(params+'-' if params else '')+(LINKED_ACCOUNT_LIST_KEY if account_type == 'linked' else PAYER_ACCOUNT_LIST_KEY)}"
                 s3 = boto3.client('s3')
                 s3.upload_file(TMP_FILE, Bucket=BUCKET, Key=key)
 
@@ -285,6 +301,7 @@ Resources:
                     aws_secret_access_key=credentials['SecretAccessKey'],
                     aws_session_token=credentials['SessionToken'],
                 )
+
       Handler: 'index.lambda_handler'
       MemorySize: 2688
       Timeout: 600
@@ -298,6 +315,7 @@ Resources:
           PREDEF_ACCOUNT_LIST_KEY: "account-list/account-list"
           EXCLUDED_ACCOUNT_LIST_KEY: "account-list/excluded-linked-account-list.csv"
           EUC_ACCOUNT_IDS: !Ref EUCAccountIDs
+
     Metadata:
       cfn_nag:
         rules_to_suppress:
@@ -310,4 +328,4 @@ Resources:
     Type: AWS::Logs::LogGroup
     Properties:
       LogGroupName: !Sub "/aws/lambda/${LambdaFunction}"
-      RetentionInDays: 60
+      RetentionInDays: 60

data-collection/deploy/deploy-data-collection.yaml

@@ -932,4 +932,4 @@ Resources:
     Type: Custom::LambdaAnalyticsExecutor
     Properties:
       ServiceToken: !Ref LambdaAnalyticsARN
-      Name: !Ref CFDataName
+      Name: !Ref CFDataName

data-collection/deploy/module-aws-feeds.yaml

@@ -320,6 +320,7 @@ Resources:
             DeployRegion: !Ref AWS::Region
             Account: !Ref AWS::AccountId
             Prefix: !Ref ResourcePrefix
+            Bucket: !Ref DestinationBucket
       'RefreshSchedule${AwsObject}':
         Type: AWS::Scheduler::Schedule
         Properties:
@@ -338,4 +339,4 @@ Resources:
     Type: Custom::LambdaAnalyticsExecutor
     Properties:
       ServiceToken: !Ref LambdaAnalyticsARN
-      Name: !Ref CFDataName
+      Name: !Ref CFDataName

data-collection/deploy/module-backup.yaml

@@ -290,6 +290,7 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
+        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-budgets.yaml

@@ -501,6 +501,7 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
+        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-compute-optimizer.yaml

@@ -425,6 +425,7 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
+        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: "AWS::Scheduler::Schedule"

data-collection/deploy/module-cost-anomaly.yaml

@@ -262,6 +262,7 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
+        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-cost-explorer-rightsizing.yaml

@@ -274,6 +274,7 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
+        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-ecs-chargeback.yaml

@@ -51,6 +51,9 @@ Parameters:
     Type: String
     Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
     Default: ""
+  DetailStepFunctionTemplate:
+    Type: String
+    Description: JSON representation of the detail retrieval StepFunction template
 
 Conditions:
   NeedDataBucketsKms: !Not [ !Equals [ !Ref DataBucketsKmsKeysArns, "" ] ]
@@ -327,16 +330,17 @@ Resources:
               """
               logger.info(f"Event data: {event}")
               account = event.get('account')
-              batch_input = event.get('BatchInput')
-              items = event.get('Items')
-              if not (account or batch_input):
+              items = event.get('items')
+              if not (account):
                   raise ValueError(
                       "Please do not trigger this Lambda manually."
                       "Find the corresponding state machine in Step Functions and Trigger from there."
                   )
-              is_summary_mode = batch_input == None
+              is_summary_mode = items is None
               logger.info(f"Executing in {'summary' if is_summary_mode else 'detail'} mode flow")
-              account = json.loads(account) if is_summary_mode else batch_input.get('account')
+              main_exe_uuid = event.get("main_exe_uuid", "")
+              stack_version = event.get("stack_version", "")
+              account = account if isinstance(account, dict) else json.loads(account)
               account_id = account["account_id"]
               region = get_active_health_region()
               partition = boto3.session.Session().get_partition_for_region(region_name=region)
@@ -387,7 +391,10 @@ Resources:
                               "bucket": BUCKET_NAME,
                               "file": key,
                               "account": account,
-                              "ingestion_time": int(round(ingestion_time.timestamp()))
+                              "ingestion_time": int(round(ingestion_time.timestamp())),
+                              "main_exe_uuid": main_exe_uuid,
+                              "params": "detail",
+                              "stack_version": stack_version
                           }
                           sf_input = json.dumps(sf_input).replace('"', '\"') #need to escape the json for SF
                           sf.start_execution(stateMachineArn=DETAIL_SM_ARN, input=sf_input)
@@ -400,7 +407,7 @@ Resources:
                           logger.error(f"Error: {exc}")
 
               elif items:
-                  ingestion_time = datetime.fromtimestamp(int(batch_input.get('ingestion_time')))
+                  ingestion_time = datetime.fromtimestamp(int(event.get('ingestion_time')))
 
                   with open(TMP_FILE, "w", encoding='utf-8') as f:
                       for item in items:
@@ -479,6 +486,7 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
+        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'
@@ -500,81 +508,9 @@ Resources:
       StateMachineName: !Sub '${ResourcePrefix}${CFDataName}-detail-StateMachine'
       StateMachineType: STANDARD
       RoleArn: !Ref StepFunctionExecutionRoleARN
-      DefinitionString: |
-        {
-          "Comment": "Collects Health Events",
-          "StartAt": "DetailProcessor Map",
-          "States": {
-            "DetailProcessor Map": {
-              "Type": "Map",
-              "ItemProcessor": {
-                "ProcessorConfig": {
-                  "Mode": "DISTRIBUTED",
-                  "ExecutionType": "STANDARD"
-                },
-                "StartAt": "DetailLambda Invoke",
-                "States": {
-                  "DetailLambda Invoke": {
-                    "Type": "Task",
-                    "Resource": "arn:${Partition}:states:::lambda:invoke",
-                    "OutputPath": "$.Payload",
-                    "Parameters": {
-                      "Payload.$": "$",
-                      "FunctionName": "${ModuleLambdaARN}"
-                    },
-                    "Retry": [
-                      {
-                        "ErrorEquals": [
-                          "Lambda.ServiceException",
-                          "Lambda.AWSLambdaException",
-                          "Lambda.SdkClientException",
-                          "Lambda.TooManyRequestsException"
-                        ],
-                        "IntervalSeconds": 1,
-                        "MaxAttempts": 3,
-                        "BackoffRate": 2
-                      }
-                    ],
-                    "End": true
-                  }
-                }
-              },
-              "Label": "DetailProcessorMap",
-              "MaxConcurrency": ${MaxConcurrentBatches},
-              "ItemReader": {
-                "Resource": "arn:${Partition}:states:::s3:getObject",
-                "ReaderConfig": {
-                  "InputType": "CSV",
-                  "CSVHeaderLocation": "FIRST_ROW"
-                },
-                "Parameters": {
-                  "Bucket.$": "$.bucket",
-                  "Key.$": "$.file"
-                }
-              },
-              "ItemBatcher": {
-                "MaxItemsPerBatch": ${ItemsPerBatch},
-                "BatchInput": {
-                  "account.$": "$.account",
-                  "ingestion_time.$": "$.ingestion_time"
-                }
-              },
-              "Next": "CrawlerStepFunctionStartExecution"
-            },
-            "CrawlerStepFunctionStartExecution": {
-              "Type": "Task",
-              "Resource": "arn:${Partition}:states:::states:startExecution.sync:2",
-              "Parameters": {
-                "StateMachineArn": "arn:${Partition}:states:${DeployRegion}:${Account}:stateMachine:${Prefix}CrawlerExecution-StateMachine",
-                "Input": {
-                  "crawlers": ${Crawlers}
-                }
-              },
-              "End": true
-            }
-          },
-          "TimeoutSeconds": 14400
-        }
+      DefinitionS3Location:
+        Bucket: !Ref CodeBucket
+        Key: !Ref DetailStepFunctionTemplate
       DefinitionSubstitutions:
         ModuleLambdaARN: !GetAtt LambdaFunction.Arn
         Crawlers: !Sub '["${ResourcePrefix}${CFDataName}-detail-Crawler"]'
@@ -587,6 +523,7 @@ Resources:
         ItemsPerBatch: 50
         MaxConcurrentBatches: 1
         Partition: !Ref AWS::Partition
+        Bucket: !Ref DestinationBucket
     Metadata:
       cfn-lint:
         config:
@@ -596,4 +533,4 @@ Resources:
     Type: Custom::LambdaAnalyticsExecutor
     Properties:
       ServiceToken: !Ref LambdaAnalyticsARN
-      Name: !Ref CFDataName
+      Name: !Ref CFDataName