Skip to content

Commit c2f505f

Browse files
iakov-ganiakov-gan
committed
more fixes
1 parent 7506418 commit c2f505f

File tree

4 files changed

+77
-26
lines changed

4 files changed

+77
-26
lines changed

‎data-collection/deploy/deploy-data-collection.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1479,10 +1479,11 @@ Resources:
14791479
Properties:
14801480
TemplateURL: !Sub "https://${CFNSourceBucket}.s3.${AWS::URLSuffix}/cfn/data-collection/v3.11.0/module-resilience-hub.yaml"
14811481
Parameters:
1482-
DestinationBucket: !Ref S3Bucket # this includes account id
1483-
BucketPrefix: !Ref DestinationBucket # this is just a prefix
1482+
DatabaseName: !Ref DatabaseName
1483+
DestinationBucket: !Ref S3Bucket
14841484
MultiAccountRoleName: !Sub "${ResourcePrefix}${MultiAccountRoleName}"
14851485
DataBucketsKmsKeysArns: !Ref DataBucketsKmsKeysArns
1486+
GlueRoleARN: !GetAtt GlueRole.Arn
14861487
Schedule: !Ref ScheduleFrequent
14871488
ResourcePrefix: !Ref ResourcePrefix
14881489
RegionsInScope:

‎data-collection/deploy/deploy-data-read-permissions.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ Metadata:
8080
default: "Include Service Quotas Module"
8181
IncludeResilienceHubModule:
8282
default: "Include ResilienceHub Module"
83-
83+
8484
Parameters:
8585
ManagementAccountRole:
8686
Type: String

‎data-collection/deploy/module-resilience-hub.yaml‎

Lines changed: 72 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,6 @@ Parameters:
44
DatabaseName:
55
Type: String
66
Description: Name of the Athena database to be created to hold lambda information
7-
BucketPrefix:
8-
Type: String
9-
Description: A prefix for S3 Bucket
10-
AllowedPattern: (?=^.{3,63}$)(?!^(\d+\.)+\d+$)(^(([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])\.)*([a-z0-9]|[a-z0-9][a-z0-9\-]*[a-z0-9])$) #FIXME
117
DestinationBucket:
128
Type: String
139
Description: Name of the S3 Bucket to be created to hold data information.
@@ -19,6 +15,9 @@ Parameters:
1915
Type: String
2016
Description: The name of what this cf is doing.
2117
Default: resilience-hub
18+
GlueRoleARN:
19+
Type: String
20+
Description: Arn for the Glue Crawler role
2221
Schedule:
2322
Type: String
2423
Description: EventBridge Schedule to trigger the data collection
@@ -47,10 +46,13 @@ Parameters:
4746
SchedulerExecutionRoleARN:
4847
Type: String
4948
Description: Common role for module Scheduler execution
50-
# DataBucketsKmsKeysArns:
51-
# Type: String
52-
# Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
53-
# Default: ""
49+
DataBucketsKmsKeysArns:
50+
Type: String
51+
Description: "ARNs of KMS Keys for data buckets and/or Glue Catalog. Comma separated list, no spaces. Keep empty if data Buckets and Glue Catalog are not Encrypted with KMS. You can also set it to '*' to grant decrypt permission for all the keys."
52+
Default: ""
53+
54+
Conditions:
55+
NeedDataBucketsKms: !Not [!Equals [!Ref DataBucketsKmsKeysArns, '']]
5456

5557
Outputs:
5658
StepFunctionARN:
@@ -83,6 +85,33 @@ Resources:
8385
- Effect: "Allow"
8486
Action: "sts:AssumeRole"
8587
Resource: !Sub "arn:${AWS::Partition}:iam::*:role/${MultiAccountRoleName}" # Need to assume a Read role in all Accounts
88+
- !If
89+
- NeedDataBucketsKms
90+
- PolicyName: "KMS"
91+
PolicyDocument:
92+
Version: "2012-10-17"
93+
Statement:
94+
- Effect: "Allow"
95+
Action:
96+
- "kms:GenerateDataKey"
97+
Resource: !Split [ ',', !Ref DataBucketsKmsKeysArns ]
98+
- !Ref AWS::NoValue
99+
- PolicyName: "S3-Access"
100+
PolicyDocument:
101+
Version: "2012-10-17"
102+
Statement:
103+
- Effect: "Allow"
104+
Action:
105+
- "s3:PutObject"
106+
- "s3:GetObject"
107+
- "s3:PutObjectAcl"
108+
Resource:
109+
- !Sub 'arn:${AWS::Partition}:s3:::${DestinationBucket}/*'
110+
- Effect: "Allow"
111+
Action:
112+
- "s3:ListBucket"
113+
Resource:
114+
- !Sub 'arn:${AWS::Partition}:s3:::${DestinationBucket}'
86115
Metadata:
87116
cfn_nag:
88117
rules_to_suppress:
@@ -101,11 +130,13 @@ Resources:
101130
Environment:
102131
Variables:
103132
REGIONS: !Ref RegionsInScope
104-
BUCKET_PREFIX: !Sub "${BucketPrefix}${AWS::AccountId}"
105133
ROLE_NAME: !Ref MultiAccountRoleName
106134
DESTINATION_BUCKET: !Ref DestinationBucket
107135
Code:
108136
ZipFile: |
137+
''' This code will go through all regions in given linked account and pull data from Resilience Hub (only applications with assessment updated since last pull)
138+
to preform full pull remove 'resilience-hub' folder on s3 and rerun StepFunction
139+
'''
109140
import os
110141
import json
111142
import logging
@@ -133,7 +164,7 @@ Resources:
133164
yield from response.get(result_key, [])
134165
next_token = response.get('NextToken') or response.get('nextToken')
135166
if not next_token:
136-
break
167+
break
137168
138169
def json_converter(obj):
139170
""" Help json encode date"""
@@ -192,7 +223,7 @@ Resources:
192223
RoleArn=f"arn:aws:iam::{account_id}:role/{ROLE_NAME}",
193224
RoleSessionName="ResHub_CID_Lambda"
194225
)['Credentials']
195-
assumed_session = boto3.session(
226+
assumed_session = boto3.session.Session(
196227
aws_access_key_id=creds["AccessKeyId"],
197228
aws_secret_access_key=creds["SecretAccessKey"],
198229
aws_session_token=creds["SessionToken"]
@@ -266,15 +297,15 @@ Resources:
266297
try:
267298
policy_response = resilience_client.describe_resiliency_policy(policyArn=policy_arn)
268299
app_data['resiliency_policy_name'] = policy_response['policy'].get('policyName', '')
269-
with s3_uploader(f'{module_name}/{module_name}-resiliency_policy/payer={payer_id}/account={account_id}/region_code={region}/{policy_id}.json') as write_policy:
300+
with s3_uploader(f'{module_name}/{module_name}-resiliency_policy/payer_id={payer_id}/account_id={account_id}/region_code={region}/{policy_id}.json') as write_policy:
270301
write_policy(policy_response['policy'])
271302
272303
except Exception as e:
273304
print(f"Error getting policy details: {str(e)}")
274305
275306
# Loop over list of assessments to get the latest successful
276307
latest_assessment = None
277-
with s3_uploader(f'{module_name}/{module_name}-assessments/payer={payer_id}/account={account_id}/region_code={region}/app={app_id}/all.json') as write_assessment:
308+
with s3_uploader(f'{module_name}/{module_name}-assessments/payer_id={payer_id}/account_id={account_id}/region_code={region}/app={app_id}/all.json') as write_assessment:
278309
for assessment in paginate(resilience_client.list_app_assessments, 'assessmentSummaries', appArn=app_arn):
279310
if assessment['assessmentStatus'] == 'Success':
280311
if not latest_assessment or latest_assessment['endTime'] < assessment['endTime']:
@@ -288,12 +319,11 @@ Resources:
288319
app_data['last_assessment_arn'] = latest_assessment['assessmentArn']
289320
app_data['resiliency_score'] = latest_assessment.get('resiliencyScore')
290321
291-
with s3_uploader(f'{module_name}/{module_name}-app_component_recommendations/payer={payer_id}/account={account_id}/region_code={region}/app={app_id}/latest.json') as write:
322+
with s3_uploader(f'{module_name}/{module_name}-app_component_recommendations/payer_id={payer_id}/account_id={account_id}/region_code={region}/app={app_id}/latest.json') as write:
292323
for rec in paginate(resilience_client.list_app_component_recommendations, 'componentRecommendations', assessmentArn=latest_assessment['assessmentArn']):
293324
write(rec)
294325
295-
296-
with s3_uploader(f'{module_name}/{module_name}-alarm_recommendations/payer={payer_id}/account={account_id}/region_code={region}/app={app_id}/latest.json') as write:
326+
with s3_uploader(f'{module_name}/{module_name}-alarm_recommendations/payer_id={payer_id}/account_id={account_id}/region_code={region}/app={app_id}/latest.json') as write:
297327
total = outstanding = excluded = 0
298328
for rec in paginate(resilience_client.list_alarm_recommendations, 'alarmRecommendations', assessmentArn=latest_assessment['assessmentArn']):
299329
status = rec.get('recommendationStatus', '') # 'recommendationStatus': 'Implemented'|'Inactive'|'NotImplemented'|'Excluded',
@@ -307,7 +337,7 @@ Resources:
307337
app_data['alarms_outstanding'] = str(outstanding)
308338
app_data['alarms_excluded'] = str(excluded)
309339
310-
with s3_uploader(f'{module_name}/{module_name}-sop_recommendations/payer={payer_id}/account={account_id}/region_code={region}/app={app_id}/latest.json') as write:
340+
with s3_uploader(f'{module_name}/{module_name}-sop_recommendations/payer_id={payer_id}/account_id={account_id}/region_code={region}/app={app_id}/latest.json') as write:
311341
total = outstanding = excluded = 0
312342
for rec in paginate(resilience_client.list_sop_recommendations, 'sopRecommendations', assessmentArn=latest_assessment['assessmentArn']):
313343
status = rec.get('recommendationStatus', '') # 'recommendationStatus': 'Implemented'|'Inactive'|'NotImplemented'|'Excluded',
@@ -321,7 +351,7 @@ Resources:
321351
app_data['sops_outstanding'] = str(outstanding)
322352
app_data['sops_excluded'] = str(excluded)
323353
324-
with s3_uploader(f'{module_name}/{module_name}-test_recommendations/payer={payer_id}/account={account_id}/region_code={region}/app={app_id}/latest.json') as write:
354+
with s3_uploader(f'{module_name}/{module_name}-test_recommendations/payer_id={payer_id}/account_id={account_id}/region_code={region}/app={app_id}/latest.json') as write:
325355
total = outstanding = excluded = 0
326356
for rec in paginate(resilience_client.list_test_recommendations, 'testRecommendations', assessmentArn=latest_assessment['assessmentArn']):
327357
test_type = rec.get('type', '') # 'type': 'Software'|'Hardware'|'AZ'|'Region'
@@ -337,7 +367,7 @@ Resources:
337367
app_data['fis_tests_outstanding'] = str(outstanding)
338368
app_data['fis_tests_excluded'] = str(excluded)
339369
340-
with s3_uploader(f'{module_name}/{module_name}-compliance_drift/payer={payer_id}/account={account_id}/region_code={region}/app={app_id}/latest.json') as write:
370+
with s3_uploader(f'{module_name}/{module_name}-compliance_drift/payer_id={payer_id}/account_id={account_id}/region_code={region}/app={app_id}/latest.json') as write:
341371
app_component_drifts = 0
342372
for rec in paginate(resilience_client.list_app_assessment_compliance_drift, 'complianceDrifts', assessmentArn=latest_assessment['assessmentArn']):
343373
drift_type = rec.get('driftType', '') # 'driftType': 'ApplicationCompliance'|'AppComponentResiliencyComplianceStatus',
@@ -346,8 +376,10 @@ Resources:
346376
write(rec)
347377
app_data['app_component_drifts'] = str(app_component_drifts)
348378
349-
with s3_uploader(f'{module_name}/{module_name}-applications/payer={payer_id}/account={account_id}/{region}-{app_id}.json') as write_app:
379+
with s3_uploader(f'{module_name}/{module_name}-applications/payer_id={payer_id}/account_id={account_id}/{region}-{app_id}.json') as write_app:
350380
write_app(app_data)
381+
with s3_uploader(f'{module_name}/{module_name}-application-details/payer_id={payer_id}/account_id={account_id}/{region}-{app_id}.json') as write_app:
382+
write_app(app)
351383
352384
# Write the time to s3
353385
status["last_read"] = collection_time
@@ -383,13 +415,14 @@ Resources:
383415
DefinitionSubstitutions:
384416
AccountCollectorLambdaARN: !Ref AccountCollectorLambdaARN
385417
ModuleLambdaARN: !GetAtt LambdaFunction.Arn
386-
Crawlers: '[]'
418+
Crawlers: !Sub '["${ResourcePrefix}${CFDataName}-Crawler"]'
387419
CollectionType: "LINKED"
388420
Params: ''
389421
Module: !Ref CFDataName
390422
DeployRegion: !Ref AWS::Region
391423
Account: !Ref AWS::AccountId
392424
Prefix: !Ref ResourcePrefix
425+
Bucket: !Ref DestinationBucket
393426

394427
ModuleRefreshSchedule:
395428
Type: 'AWS::Scheduler::Schedule'
@@ -411,6 +444,24 @@ Resources:
411444
ServiceToken: !Ref LambdaAnalyticsARN
412445
Name: !Ref CFDataName
413446

447+
Crawler:
448+
Type: AWS::Glue::Crawler
449+
Properties:
450+
Name: !Sub '${ResourcePrefix}${CFDataName}-Crawler'
451+
Role: !Ref GlueRoleARN
452+
DatabaseName: !Ref DatabaseName
453+
Targets:
454+
S3Targets:
455+
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-applications/"
456+
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-application-details/"
457+
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-resiliency_policy/"
458+
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-assessments/"
459+
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-app_component_recommendations/"
460+
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-alarm_recommendations/"
461+
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-sop_recommendations/"
462+
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-test_recommendations/"
463+
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-compliance_drift/"
464+
414465
# ResilienceHubApplicationsTable:
415466
# Type: AWS::Glue::Table
416467
# Properties:

‎test/utils.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -408,8 +408,7 @@ def trigger_update(account_id):
408408
f"arn:{partition}:states:{region}:{account_id}:stateMachine:{PREFIX}quicksight-StateMachine",
409409
f"arn:{partition}:states:{region}:{account_id}:stateMachine:{PREFIX}service-quotas-StateMachine",
410410
f"arn:{partition}:states:{region}:{account_id}:stateMachine:{PREFIX}workspaces-metrics-StateMachine",
411-
f"arn:{partition}:states:{region}:{account_id}:stateMachine:{PREFIX}resiliency-hub-StateMachine",
412-
411+
f"arn:{partition}:states:{region}:{account_id}:stateMachine:{PREFIX}resilience-hub-StateMachine",
413412
]
414413
lambda_arns = []
415414
lambda_norun_arns = []

0 commit comments

Comments
 (0)