Revert "Instrumentation phase 1 (#275)" (#356)

This reverts commit 7746a4d.

Author: iakov-aws

data-collection/deploy/account-collector.yaml

@@ -128,9 +128,7 @@ Resources:
           ZipFile: |
             import os
             import json
-            import uuid
             import logging
-            from datetime import datetime
             from functools import partial
 
             import boto3
@@ -143,7 +141,6 @@ Resources:
             EXCLUDED_ACCOUNT_LIST_KEY = os.environ.get('EXCLUDED_ACCOUNT_LIST_KEY')
             EUC_ACCOUNTS = os.environ.get('EUC_ACCOUNT_IDS', '').strip()
             TMP_FILE = "/tmp/data.json"
-            START_TIME = str(datetime.now().isoformat())
 
             logger = logging.getLogger(__name__)
             logger.setLevel(getattr(logging, os.environ.get('LOG_LEVEL', 'INFO').upper(), logging.INFO))
@@ -167,7 +164,7 @@ Resources:
                     'compute-optimizer': partial(iterate_admins_accounts, 'compute-optimizer'),
                     'backup': partial(iterate_admins_accounts, 'backup'),
                 }
-                account_type = event.get("type", '').lower()
+                account_type = event.get("Type", '').lower()
                 if account_type not in functions:
                     raise Exception(f"Lambda event must have 'Type' parameter with value = ({list(functions.keys())})") #pylint: disable=broad-exception-raised
 
@@ -176,14 +173,6 @@ Resources:
                     count = 0
                     f.write("[\n")
                     for account in account_iterator():
-                        account['main_exe_uuid'] = event.get("main_exe_uuid", str(uuid.uuid4()))
-                        account['module'] = event.get("module", '').lower()
-                        account['bucket'] = BUCKET
-                        account['dc_account'] = boto3.client('sts').get_caller_identity()['Account']
-                        account['dc_region'] = boto3.session.Session().region_name
-                        account['params'] = event.get("params", '')
-                        account['prefix'] = RESOURCE_PREFIX
-                        account['stack_version'] = event.get("stack_version", '')
                         if count > 0:
                             f.write(",\n")
                         f.write(json.dumps(account))
@@ -296,7 +285,6 @@ Resources:
                     aws_secret_access_key=credentials['SecretAccessKey'],
                     aws_session_token=credentials['SessionToken'],
                 )
-
       Handler: 'index.lambda_handler'
       MemorySize: 2688
       Timeout: 600
@@ -310,7 +298,6 @@ Resources:
           PREDEF_ACCOUNT_LIST_KEY: "account-list/account-list"
           EXCLUDED_ACCOUNT_LIST_KEY: "account-list/excluded-linked-account-list.csv"
           EUC_ACCOUNT_IDS: !Ref EUCAccountIDs
-
     Metadata:
       cfn_nag:
         rules_to_suppress:
@@ -323,4 +310,4 @@ Resources:
     Type: AWS::Logs::LogGroup
     Properties:
       LogGroupName: !Sub "/aws/lambda/${LambdaFunction}"
-      RetentionInDays: 60
+      RetentionInDays: 60

data-collection/deploy/deploy-data-collection.yaml

@@ -932,4 +932,4 @@ Resources:
     Type: Custom::LambdaAnalyticsExecutor
     Properties:
       ServiceToken: !Ref LambdaAnalyticsARN
-      Name: !Ref CFDataName
+      Name: !Ref CFDataName

data-collection/deploy/module-aws-feeds.yaml

@@ -320,7 +320,6 @@ Resources:
             DeployRegion: !Ref AWS::Region
             Account: !Ref AWS::AccountId
             Prefix: !Ref ResourcePrefix
-            Bucket: !Ref DestinationBucket
       'RefreshSchedule${AwsObject}':
         Type: AWS::Scheduler::Schedule
         Properties:
@@ -339,4 +338,4 @@ Resources:
     Type: Custom::LambdaAnalyticsExecutor
     Properties:
       ServiceToken: !Ref LambdaAnalyticsARN
-      Name: !Ref CFDataName
+      Name: !Ref CFDataName

data-collection/deploy/module-backup.yaml

@@ -290,7 +290,6 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
-        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-budgets.yaml

@@ -501,7 +501,6 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
-        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-compute-optimizer.yaml

@@ -425,7 +425,6 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
-        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: "AWS::Scheduler::Schedule"

data-collection/deploy/module-cost-anomaly.yaml

@@ -262,7 +262,6 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
-        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-cost-explorer-rightsizing.yaml

@@ -274,7 +274,6 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
-        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'

data-collection/deploy/module-ecs-chargeback.yaml

@@ -51,9 +51,6 @@ Parameters:
     Type: String
     Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
     Default: ""
-  DetailStepFunctionTemplate:
-    Type: String
-    Description: JSON representation of the detail retrieval StepFunction template
 
 Conditions:
   NeedDataBucketsKms: !Not [ !Equals [ !Ref DataBucketsKmsKeysArns, "" ] ]
@@ -482,7 +479,6 @@ Resources:
         DeployRegion: !Ref AWS::Region
         Account: !Ref AWS::AccountId
         Prefix: !Ref ResourcePrefix
-        Bucket: !Ref DestinationBucket
 
   ModuleRefreshSchedule:
     Type: 'AWS::Scheduler::Schedule'
@@ -501,12 +497,84 @@ Resources:
   StepFunctionDetail:
     Type: AWS::StepFunctions::StateMachine
     Properties:
-      StateMachineName: !Sub '${ResourcePrefix}${CFDataName}-StateMachine'
+      StateMachineName: !Sub '${ResourcePrefix}${CFDataName}-detail-StateMachine'
       StateMachineType: STANDARD
       RoleArn: !Ref StepFunctionExecutionRoleARN
-      DefinitionS3Location:
-        Bucket: !Ref CodeBucket
-        Key: !Ref DetailStepFunctionTemplate
+      DefinitionString: |
+        {
+          "Comment": "Collects Health Events",
+          "StartAt": "DetailProcessor Map",
+          "States": {
+            "DetailProcessor Map": {
+              "Type": "Map",
+              "ItemProcessor": {
+                "ProcessorConfig": {
+                  "Mode": "DISTRIBUTED",
+                  "ExecutionType": "STANDARD"
+                },
+                "StartAt": "DetailLambda Invoke",
+                "States": {
+                  "DetailLambda Invoke": {
+                    "Type": "Task",
+                    "Resource": "arn:${Partition}:states:::lambda:invoke",
+                    "OutputPath": "$.Payload",
+                    "Parameters": {
+                      "Payload.$": "$",
+                      "FunctionName": "${ModuleLambdaARN}"
+                    },
+                    "Retry": [
+                      {
+                        "ErrorEquals": [
+                          "Lambda.ServiceException",
+                          "Lambda.AWSLambdaException",
+                          "Lambda.SdkClientException",
+                          "Lambda.TooManyRequestsException"
+                        ],
+                        "IntervalSeconds": 1,
+                        "MaxAttempts": 3,
+                        "BackoffRate": 2
+                      }
+                    ],
+                    "End": true
+                  }
+                }
+              },
+              "Label": "DetailProcessorMap",
+              "MaxConcurrency": ${MaxConcurrentBatches},
+              "ItemReader": {
+                "Resource": "arn:${Partition}:states:::s3:getObject",
+                "ReaderConfig": {
+                  "InputType": "CSV",
+                  "CSVHeaderLocation": "FIRST_ROW"
+                },
+                "Parameters": {
+                  "Bucket.$": "$.bucket",
+                  "Key.$": "$.file"
+                }
+              },
+              "ItemBatcher": {
+                "MaxItemsPerBatch": ${ItemsPerBatch},
+                "BatchInput": {
+                  "account.$": "$.account",
+                  "ingestion_time.$": "$.ingestion_time"
+                }
+              },
+              "Next": "CrawlerStepFunctionStartExecution"
+            },
+            "CrawlerStepFunctionStartExecution": {
+              "Type": "Task",
+              "Resource": "arn:${Partition}:states:::states:startExecution.sync:2",
+              "Parameters": {
+                "StateMachineArn": "arn:${Partition}:states:${DeployRegion}:${Account}:stateMachine:${Prefix}CrawlerExecution-StateMachine",
+                "Input": {
+                  "crawlers": ${Crawlers}
+                }
+              },
+              "End": true
+            }
+          },
+          "TimeoutSeconds": 14400
+        }
       DefinitionSubstitutions:
         ModuleLambdaARN: !GetAtt LambdaFunction.Arn
         Crawlers: !Sub '["${ResourcePrefix}${CFDataName}-detail-Crawler"]'
@@ -519,7 +587,6 @@ Resources:
         ItemsPerBatch: 50
         MaxConcurrentBatches: 1
         Partition: !Ref AWS::Partition
-        Bucket: !Ref DestinationBucket
     Metadata:
       cfn-lint:
         config:
@@ -529,4 +596,4 @@ Resources:
     Type: Custom::LambdaAnalyticsExecutor
     Properties:
       ServiceToken: !Ref LambdaAnalyticsARN
-      Name: !Ref CFDataName
+      Name: !Ref CFDataName