Skip to content

Commit a7b6c5d

Browse files
marcodb-awsmarcodb-aws
authored
Add support for KMS Keys in Data Collection
1 parent 97a348e commit a7b6c5d

20 files changed

+408
-8
lines changed

‎data-collection/deploy/account-collector.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ Parameters:
1818
DestinationBucketARN:
1919
Type: String
2020
Description: ARN of the S3 Bucket that exists or needs to be created to hold rightsizing information
21+
DataBucketsKmsKeysArns:
22+
Type: String
23+
Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
24+
Default: ""
2125
Outputs:
2226
LambdaFunctionName:
2327
Value: !Ref LambdaFunction
@@ -26,6 +30,8 @@ Outputs:
2630
Value: !GetAtt LambdaFunction.Arn
2731
Export:
2832
Name: !Sub ${ResourcePrefix}AccountCollectorLambdaARN
33+
Conditions:
34+
NeedDataBucketsKms: !Not [ !Equals [ !Ref DataBucketsKmsKeysArns, "" ] ]
2935
Resources:
3036
LambdaRole:
3137
Type: AWS::IAM::Role
@@ -88,6 +94,18 @@ Resources:
8894
- "s3:PutObject"
8995
Resource:
9096
- !Sub "${DestinationBucketARN}/*"
97+
- !If
98+
- NeedDataBucketsKms
99+
- PolicyName: "KMS"
100+
PolicyDocument:
101+
Version: "2012-10-17"
102+
Statement:
103+
- Effect: "Allow"
104+
Action:
105+
- "kms:GenerateDataKey"
106+
Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ]
107+
- !Ref AWS::NoValue
108+
91109
Metadata:
92110
cfn_nag:
93111
rules_to_suppress:

‎data-collection/deploy/deploy-data-collection.yaml‎

Lines changed: 67 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ Metadata:
1616
- Schedule
1717
- ScheduleFrequent
1818
- CFNSourceBucket
19+
- DataBucketsKmsKeysArns
1920
- Label:
2021
default: 'Available modules'
2122
Parameters:
@@ -57,6 +58,8 @@ Metadata:
5758
default: "Role Prefix"
5859
CFNSourceBucket:
5960
default: "DO NOT CHANGE - A bucket that contains WA-Labs CloudFormation templates. Must be always 'aws-managed-cost-intelligence-dashboards'"
61+
DataBucketsKmsKeysArns:
62+
default: ""
6063
IncludeTAModule:
6164
default: 'Include AWS Trusted Advisor Data Collection Module'
6265
IncludeRightsizingModule:
@@ -160,6 +163,10 @@ Parameters:
160163
Type: String
161164
Description: "DO NOT CHANGE - A bucket that contains WA-Labs CloudFormation templates. Must be always 'aws-managed-cost-intelligence-dashboards'"
162165
Default: "aws-managed-cost-intelligence-dashboards"
166+
DataBucketsKmsKeysArns:
167+
Type: String
168+
Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
169+
Default: ""
163170
IncludeTAModule:
164171
Type: String
165172
Description: Collects AWS Trusted Advisor recommendations data
@@ -290,6 +297,7 @@ Conditions:
290297
- !Join [ '', !Split [ ' ', !Ref RegionsInScope ] ] # remove spaces
291298
- ""
292299
ProdCFNTemplateUsed: !Equals [ !Ref CFNSourceBucket, 'aws-managed-cost-intelligence-dashboards' ]
300+
NeedDataBucketsKms: !Not [ !Equals [ !Ref DataBucketsKmsKeysArns, "" ] ]
293301

294302
Resources:
295303
S3Bucket:
@@ -394,12 +402,17 @@ Resources:
394402
- s3:GetObject
395403
Resource:
396404
- !Sub "arn:${AWS::Partition}:s3:::${DestinationBucket}${AWS::AccountId}/*"
397-
## Uncomment if bucket is encrypted by Custom KMS Key
398-
#- Effect: Allow
399-
# Action:
400-
# - kms:Decrypt
401-
# Resource:
402-
# - !Sub "arn:${AWS::Partition}:kms:${AWS::Region}:${AWS::AccountId}:key/key-id"
405+
- !If
406+
- NeedDataBucketsKms
407+
- PolicyName: "KMS"
408+
PolicyDocument:
409+
Version: "2012-10-17"
410+
Statement:
411+
- Effect: "Allow"
412+
Action:
413+
- "kms:Decrypt"
414+
Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ]
415+
- !Ref AWS::NoValue
403416
- PolicyName: "Glue"
404417
PolicyDocument:
405418
Version: "2012-10-17"
@@ -460,6 +473,7 @@ Resources:
460473
- sts:AssumeRole
461474
ManagedPolicyArns:
462475
- !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
476+
463477
LambdaAnalytics:
464478
Type: AWS::Lambda::Function
465479
Properties:
@@ -556,6 +570,25 @@ Resources:
556570
- !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog"
557571
- !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}"
558572
- !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${DatabaseName}/*"
573+
574+
KmsPolicyForCidResources:
575+
Type: AWS::IAM::Policy
576+
Condition: NeedDataBucketsKms
577+
Properties:
578+
PolicyName: !Sub "${ResourcePrefix}AwsDataCollectionKmsDecryption"
579+
PolicyDocument:
580+
Version: 2012-10-17
581+
Statement:
582+
- Effect: Allow
583+
Action:
584+
- 'kms:Decrypt'
585+
Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ]
586+
Roles:
587+
- !Ref LambdaInitRole
588+
- !Ref StepFunctionExecutionRole
589+
- !Ref LambdaManageGlueTableRole
590+
- !Ref GlueRole
591+
559592
LambdaInit:
560593
Type: AWS::Lambda::Function
561594
Properties:
@@ -680,6 +713,7 @@ Resources:
680713
- !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:catalog"
681714
- !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}"
682715
- !Sub "arn:${AWS::Partition}:glue:${AWS::Region}:${AWS::AccountId}:table/${DatabaseName}/*"
716+
683717
LambdaManageGlueTable:
684718
Type: AWS::Lambda::Function
685719
Properties:
@@ -894,6 +928,7 @@ Resources:
894928
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-trusted-advisor.yaml"
895929
Parameters:
896930
DatabaseName: !Ref DatabaseName
931+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
897932
DestinationBucket: !Ref S3Bucket
898933
DestinationBucketARN: !GetAtt S3Bucket.Arn
899934
GlueRoleARN: !GetAtt GlueRole.Arn
@@ -914,6 +949,7 @@ Resources:
914949
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-cost-explorer-rightsizing.yaml"
915950
Parameters:
916951
DatabaseName: !Ref DatabaseName
952+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
917953
DestinationBucket: !Ref S3Bucket
918954
DestinationBucketARN: !GetAtt S3Bucket.Arn
919955
ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}"
@@ -934,6 +970,7 @@ Resources:
934970
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-cost-anomaly.yaml"
935971
Parameters:
936972
DatabaseName: !Ref DatabaseName
973+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
937974
DestinationBucket: !Ref S3Bucket
938975
DestinationBucketARN: !GetAtt S3Bucket.Arn
939976
ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}"
@@ -955,6 +992,7 @@ Resources:
955992
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-support-cases.yaml"
956993
Parameters:
957994
DatabaseName: !Ref DatabaseName
995+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
958996
DestinationBucket: !Ref S3Bucket
959997
DestinationBucketARN: !GetAtt S3Bucket.Arn
960998
MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}"
@@ -975,6 +1013,7 @@ Resources:
9751013
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-backup.yaml"
9761014
Parameters:
9771015
DatabaseName: !Ref DatabaseName
1016+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
9781017
DestinationBucket: !Ref S3Bucket
9791018
DestinationBucketARN: !GetAtt S3Bucket.Arn
9801019
ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}"
@@ -997,6 +1036,7 @@ Resources:
9971036
DatabaseName: !Ref DatabaseName
9981037
DestinationBucket: !Ref S3Bucket
9991038
DestinationBucketARN: !GetAtt S3Bucket.Arn
1039+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
10001040
GlueRoleARN: !GetAtt GlueRole.Arn
10011041
MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}"
10021042
Schedule: !Ref Schedule
@@ -1023,6 +1063,7 @@ Resources:
10231063
DatabaseName: !Ref DatabaseName
10241064
DestinationBucket: !Ref S3Bucket
10251065
DestinationBucketARN: !GetAtt S3Bucket.Arn
1066+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
10261067
GlueRoleARN: !GetAtt GlueRole.Arn
10271068
CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ]
10281069
Schedule: !Ref Schedule
@@ -1045,6 +1086,7 @@ Resources:
10451086
DestinationBucket: !Ref S3Bucket
10461087
ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}"
10471088
ManagementAccountID: !Ref ManagementAccountID
1089+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
10481090
Schedule: !Ref Schedule
10491091
ResourcePrefix: !Ref ResourcePrefix
10501092
BucketPrefix: !Ref DestinationBucket
@@ -1067,6 +1109,7 @@ Resources:
10671109
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-ecs-chargeback.yaml"
10681110
Parameters:
10691111
DatabaseName: !Ref DatabaseName
1112+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
10701113
DestinationBucket: !Ref S3Bucket
10711114
DestinationBucketARN: !GetAtt S3Bucket.Arn
10721115
GlueRoleARN: !GetAtt GlueRole.Arn
@@ -1092,6 +1135,7 @@ Resources:
10921135
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-rds-usage.yaml"
10931136
Parameters:
10941137
DatabaseName: !Ref DatabaseName
1138+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
10951139
DestinationBucket: !Ref S3Bucket
10961140
DestinationBucketARN: !GetAtt S3Bucket.Arn
10971141
GlueRoleARN: !GetAtt GlueRole.Arn
@@ -1117,6 +1161,7 @@ Resources:
11171161
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-organization.yaml"
11181162
Parameters:
11191163
DatabaseName: !Ref DatabaseName
1164+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
11201165
DestinationBucket: !Ref S3Bucket
11211166
DestinationBucketARN: !GetAtt S3Bucket.Arn
11221167
GlueRoleARN: !GetAtt GlueRole.Arn
@@ -1140,6 +1185,7 @@ Resources:
11401185
DestinationBucket: !Ref S3Bucket
11411186
DestinationBucketARN: !GetAtt S3Bucket.Arn
11421187
GlueRoleARN: !GetAtt GlueRole.Arn
1188+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
11431189
MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}"
11441190
Schedule: !Ref ScheduleFrequent
11451191
ResourcePrefix: !Ref ResourcePrefix
@@ -1157,6 +1203,7 @@ Resources:
11571203
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-transit-gateway.yaml"
11581204
Parameters:
11591205
DatabaseName: !Ref DatabaseName
1206+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
11601207
DestinationBucket: !Ref S3Bucket
11611208
DestinationBucketARN: !GetAtt S3Bucket.Arn
11621209
GlueRoleARN: !GetAtt GlueRole.Arn
@@ -1184,6 +1231,7 @@ Resources:
11841231
DatabaseName: !Ref DatabaseName
11851232
DestinationBucket: !Ref S3Bucket
11861233
DestinationBucketARN: !GetAtt S3Bucket.Arn
1234+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
11871235
Schedule: !Ref ScheduleFrequent
11881236
GlueRoleARN: !GetAtt GlueRole.Arn
11891237
ResourcePrefix: !Ref ResourcePrefix
@@ -1200,6 +1248,7 @@ Resources:
12001248
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-health-events.yaml"
12011249
Parameters:
12021250
DatabaseName: !Ref DatabaseName
1251+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
12031252
DestinationBucket: !Ref S3Bucket
12041253
DestinationBucketARN: !GetAtt S3Bucket.Arn
12051254
Schedule: !Ref ScheduleFrequent
@@ -1220,6 +1269,7 @@ Resources:
12201269
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-license-manager.yaml"
12211270
Parameters:
12221271
DatabaseName: !Ref DatabaseName
1272+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
12231273
DestinationBucket: !Ref S3Bucket
12241274
DestinationBucketARN: !GetAtt S3Bucket.Arn
12251275
ManagementRoleName: !Sub "${ResourcePrefix}${ManagementAccountRole}"
@@ -1244,6 +1294,7 @@ Resources:
12441294
DestinationBucketARN: !GetAtt S3Bucket.Arn
12451295
MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}"
12461296
Schedule: !Ref ScheduleFrequent
1297+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
12471298
GlueRoleARN: !GetAtt GlueRole.Arn
12481299
ResourcePrefix: !Ref ResourcePrefix
12491300
LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn
@@ -1265,6 +1316,7 @@ Resources:
12651316
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/module-quicksight.yaml"
12661317
Parameters:
12671318
DatabaseName: !Ref DatabaseName
1319+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
12681320
DestinationBucket: !Ref S3Bucket
12691321
DestinationBucketARN: !GetAtt S3Bucket.Arn
12701322
Schedule: !Ref ScheduleFrequent
@@ -1287,6 +1339,7 @@ Resources:
12871339
ResourcePrefix: !Ref ResourcePrefix
12881340
DestinationBucket: !Ref S3Bucket
12891341
DestinationBucketARN: !GetAtt S3Bucket.Arn
1342+
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
12901343

12911344
DataCollectionReadAccess:
12921345
Type: AWS::IAM::ManagedPolicy
@@ -1321,6 +1374,14 @@ Resources:
13211374
- s3:GetObjectVersion
13221375
Resource:
13231376
- !Sub ${S3Bucket.Arn}/*
1377+
- !If
1378+
- NeedDataBucketsKms
1379+
- Sid: AllowKmsDecrypt
1380+
Effect: "Allow"
1381+
Action:
1382+
- "kms:Decrypt"
1383+
Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ]
1384+
- !Ref AWS::NoValue
13241385

13251386
Outputs:
13261387
Bucket:

‎data-collection/deploy/module-aws-feeds.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,13 @@ Parameters:
4141
LambdaAnalyticsARN:
4242
Type: String
4343
Description: Arn of lambda for Analytics
44+
DataBucketsKmsKeysArns:
45+
Type: String
46+
Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
47+
Default: ""
48+
49+
Conditions:
50+
NeedDataBucketsKms: !Not [ !Equals [ !Ref DataBucketsKmsKeysArns, "" ] ]
4451

4552
Resources:
4653

@@ -70,6 +77,17 @@ Resources:
7077
- "s3:PutObject"
7178
Resource:
7279
- !Sub "${DestinationBucketARN}/*"
80+
- !If
81+
- NeedDataBucketsKms
82+
- PolicyName: "KMS"
83+
PolicyDocument:
84+
Version: "2012-10-17"
85+
Statement:
86+
- Effect: "Allow"
87+
Action:
88+
- "kms:GenerateDataKey"
89+
Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ]
90+
- !Ref AWS::NoValue
7391
Metadata:
7492
cfn_nag:
7593
rules_to_suppress:

‎data-collection/deploy/module-backup.yaml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ Parameters:
66
Type: String
77
Description: Name of the Athena database to be created to hold lambda information
88
Default: optimization_data
9+
DataBucketsKmsKeysArns:
10+
Type: String
11+
Description: KMS Key ARNs used for encrypting data in S3 buckets (comma separated)
912
DestinationBucket:
1013
Type: String
1114
Description: Name of the S3 Bucket that exists or needs to be created to hold backup information
@@ -62,6 +65,9 @@ Mappings:
6265
CopyJobs:
6366
path: copy-jobs
6467

68+
Conditions:
69+
NeedDataBucketsKms: !Not [!Equals [!Ref DataBucketsKmsKeysArns, '']]
70+
6571
Resources:
6672
LambdaRole:
6773
Type: AWS::IAM::Role
@@ -80,6 +86,17 @@ Resources:
8086
- !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole"
8187
Path: /
8288
Policies:
89+
- !If
90+
- NeedDataBucketsKms
91+
- PolicyName: "KMS"
92+
PolicyDocument:
93+
Version: "2012-10-17"
94+
Statement:
95+
- Effect: "Allow"
96+
Action:
97+
- "kms:GenerateDataKey"
98+
Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ]
99+
- !Ref AWS::NoValue
83100
- PolicyName: !Sub "${CFDataName}-ManagementAccount-LambdaRole"
84101
PolicyDocument:
85102
Version: "2012-10-17"

0 commit comments

Comments
 (0)