Merge branch 'main' into instrumentation

Author: esc1144

data-collection/deploy/account-collector.yaml

@@ -22,6 +22,11 @@ Parameters:
     Type: String
     Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
     Default: ""
+  EUCAccountIDs:
+    Type: String
+    Description: "Comma-separated list of account IDs that user has identified to have WorkSpaces running, used to collect WorkSpaces metrics from. If left blank, metrics will be checked from all linked accounts in Organization."
+    Default: ""
+
 Outputs:
   LambdaFunctionName:
     Value: !Ref LambdaFunction
@@ -143,6 +148,7 @@ Resources:
             LINKED_ACCOUNT_LIST_KEY = os.environ.get('LINKED_ACCOUNT_LIST_KEY', 'linked-account-list.json')
             PAYER_ACCOUNT_LIST_KEY = os.environ.get('PAYER_ACCOUNT_LIST_KEY', 'payer-account-list.json')
             EXCLUDED_ACCOUNT_LIST_KEY = os.environ.get('EXCLUDED_ACCOUNT_LIST_KEY', 'excluded-linked-account-list.csv')
+            EUC_ACCOUNTS = os.environ.get('EUC_ACCOUNT_IDS', '').strip()
 
             MODULE = ""
 
@@ -165,6 +171,7 @@ Resources:
 
                     functions = { # keep keys same as boto3 services
                         'linked': iterate_linked_accounts,
+                        'euc': partial(iterate_accounts_with_filter, EUC_ACCOUNTS),
                         'payers': partial(iterate_admins_accounts, None),
                         'organizations': partial(iterate_admins_accounts, 'organizations'),
                         'compute-optimizer': partial(iterate_admins_accounts, 'compute-optimizer'),
@@ -236,6 +243,8 @@ Resources:
                         try:
                             account_id = ssm.get_parameter(Name=ssm_key)['Parameter']['Value']
                         except ssm.exceptions.ParameterNotFound:
+                            logger.info(f'Not found ssm parameter {ssm_key}. Will use Management Account Id {payer_id}')
+                    yield {"account": json.dumps({'account_id': account_id, 'account_name': '', 'payer_id': payer_id})}
                             logger.warning(f'Not found ssm parameter {ssm_key}. Will use Management Account Id {payer_id}')
                     yield {"account": json.dumps({'account_id': account_id, 'account_name': '', 'payer_id': payer_id}), "main_exe_uuid": ""}
 
@@ -271,6 +280,16 @@ Resources:
                             yield format_account(account_id, account.get('Name'), payer_id)
                         logger.info(f'Found {count} accounts for payer {payer_id}')
 
+            def iterate_accounts_with_filter(filter_accounts):
+                ''' same as iterate_linked_accounts but with additional filtering
+                filter_accounts: a comma separated list of accounts OR '*'
+                '''
+                for acc in iterate_linked_accounts():
+                    account_id = json.loads(acc['account'])['account_id']
+                    if account_id in filter_accounts.split(',') or filter_accounts.strip() == '*':
+                        yield acc
+
+
             def get_defined_list(bucket, key):
                 s3 = boto3.client("s3")
                 exts = [".json", ".csv"]
@@ -400,6 +419,7 @@ Resources:
           PAYER_ACCOUNT_LIST_KEY: "payer-account-list.json"
           EXCLUDED_ACCOUNT_LIST_KEY: "excluded-linked-account-list.csv"
           DC_ACCOUNT: !Ref AWS::AccountId
+          EUC_ACCOUNT_IDS: !Ref EUCAccountIDs
     Metadata:
       cfn_nag:
         rules_to_suppress:

data-collection/deploy/deploy-data-collection.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: CID Data Collection Stack v3.8.0
+Description: CID Data Collection Stack v3.9.0
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -30,20 +30,16 @@ Metadata:
           - IncludeInventoryCollectorModule
           - IncludeOrgDataModule
           - IncludeRDSUtilizationModule
-          - IncludeEUCUtilizationModule
           - IncludeRightsizingModule
           - IncludeTAModule
           - IncludeTransitGatewayModule
           - IncludeAWSFeedsModule
           - IncludeLicenseManagerModule
           - IncludeQuickSightModule
           - IncludeServiceQuotasModule
-      - Label:
-          default: 'EUC Module Configuration'
-        Parameters:
           - IncludeEUCUtilizationModule
       - Label:
-          default: 'EUC Module Settings'
+          default: 'EUC (End User Compute) Module Configuration'
         Parameters:
           - EUCAccountIDs
     ParameterLabels:
@@ -225,11 +221,11 @@ Parameters:
     Type: String
     Description: Collects WorkSpaces CloudWatch metrics from your accounts
     AllowedValues: ['yes', 'no']
-    Default: 'no'  
+    Default: 'no'
   EUCAccountIDs:
     Type: String
-    Description: "Optional, If you enable EUC Utilization or Inventory module and you use Amazon WorkSpaces, please provide a comma-separated list of account IDs where WorkSpaces are deployed. If left blank, metrics will be collected from all linked accounts in the Organization."
-    Default: ""
+    Description: "Optional, If you enable EUC Utilization or Inventory module and you use Amazon WorkSpaces, please provide a comma-separated list of account IDs where WorkSpaces are deployed. Or you can set * to collect from all linked accounts in the Organization."
+    Default: "*"
   IncludeOrgDataModule:
     Type: String
     Description: Collects AWS Organizations data such as account Id, account name, organization parent and specified tags
@@ -1259,7 +1255,6 @@ Resources:
         StepFunctionTemplate: !FindInMap [StepFunctionCode, main-state-machine-v4, TemplatePath]
         StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn
         SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn
-        EUCAccountIDs: !Ref EUCAccountIDs
         RegionsInScope:
           Fn::If:
             - RegionsInScopeIsEmpty
@@ -1453,6 +1448,7 @@ Resources:
         DestinationBucket: !Ref S3Bucket
         DestinationBucketARN: !GetAtt S3Bucket.Arn
         DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
+        EUCAccountIDs: !Ref EUCAccountIDs
 
   DataCollectionReadAccess:
     Type: AWS::IAM::ManagedPolicy

data-collection/deploy/deploy-data-read-permissions.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: "2010-09-09"
-Description: CID Data Collection - All-in-One for Management Account v3.8.0
+Description: CID Data Collection - All-in-One for Management Account v3.9.0
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -196,7 +196,6 @@ Resources:
         ResourcePrefix: !Ref ResourcePrefix
         IncludeComputeOptimizerModule: !Ref IncludeComputeOptimizerModule
         IncludeCostAnomalyModule: !Ref IncludeCostAnomalyModule
-        IncludeSupportCasesModule: !Ref IncludeSupportCasesModule
         IncludeRightsizingModule: !Ref IncludeRightsizingModule
         IncludeBackupModule: !Ref IncludeBackupModule
         IncludeHealthEventsModule: !Ref IncludeHealthEventsModule
@@ -216,14 +215,14 @@ Resources:
         IncludeInventoryCollectorModule: !Ref IncludeInventoryCollectorModule
         IncludeECSChargebackModule: !Ref IncludeECSChargebackModule
         IncludeRDSUtilizationModule: !Ref IncludeRDSUtilizationModule
-        IncludeEUCUtilizationModule: !Ref IncludeEUCUtilizationModule       
+        IncludeEUCUtilizationModule: !Ref IncludeEUCUtilizationModule
         IncludeBudgetsModule: !Ref IncludeBudgetsModule
         IncludeTransitGatewayModule: !Ref IncludeTransitGatewayModule
         IncludeServiceQuotasModule: !Ref IncludeServiceQuotasModule
   DataCollectorOrgAccountModulesReadStackSet:
     Type: AWS::CloudFormation::StackSet
     Properties:
-      Description: "StackSet in charge of deploying read roles across organization accounts v3.8.0"
+      Description: "StackSet in charge of deploying read roles across organization accounts v3.9.0"
       PermissionModel: SERVICE_MANAGED
       AutoDeployment:
         Enabled: true
@@ -252,7 +251,7 @@ Resources:
         - ParameterKey: IncludeRDSUtilizationModule
           ParameterValue: !Ref IncludeRDSUtilizationModule
         - ParameterKey: IncludeEUCUtilizationModule
-          ParameterValue: !Ref IncludeEUCUtilizationModule          
+          ParameterValue: !Ref IncludeEUCUtilizationModule
         - ParameterKey: IncludeBudgetsModule
           ParameterValue: !Ref IncludeBudgetsModule
         - ParameterKey: IncludeTransitGatewayModule