Skip to content

Commit c99e89c

Browse files
iakov-ganiakov-gan
committed
refactor python
1 parent efafc02 commit c99e89c

File tree

2 files changed

+10
-183
lines changed

2 files changed

+10
-183
lines changed

‎data-collection/deploy/module-resilience-hub.yaml‎

Lines changed: 8 additions & 181 deletions
Original file line numberDiff line numberDiff line change
@@ -192,10 +192,10 @@ Resources:
192192
if not temp_file.closed:
193193
temp_file.close()
194194
s3_client.upload_file(temp_file.name, bucket, s3_path)
195-
print(f"Uploaded records to s3://{bucket}/{s3_path}")
195+
logger.info(f"Uploaded records to s3://{bucket}/{s3_path}")
196196
197197
except Exception as e:
198-
print(f"Error during S3 upload s3://{bucket}/{s3_path}: {str(e)}")
198+
logger.error(f"Error during S3 upload s3://{bucket}/{s3_path}: {str(e)}")
199199
raise
200200
201201
finally:
@@ -205,7 +205,7 @@ Resources:
205205
try:
206206
os.unlink(temp_file.name)
207207
except OSError as e:
208-
print(f"Warning: Could not delete temporary file {temp_file.name}: {e}")
208+
logger.info(f"Warning: Could not delete temporary file {temp_file.name}: {e}")
209209
210210
def lambda_handler(event, context): #pylint: disable=unused-argument,too-many-branches,too-many-locals,too-many-statements
211211
logger.info(f"Event: {event}")
@@ -254,54 +254,16 @@ Resources:
254254
app_id = app_arn.split('/')[-1]
255255
app = resilience_client.describe_app(appArn=app_arn)['app']
256256
257-
# Initialize data structure
258-
app_data = {
259-
'region': region,
260-
'account_id': account_id,
261-
'application_id': app_id,
262-
'application_arn': app_arn,
263-
'application_name': app.get('name', ''),
264-
'resiliency_policy_id': '',
265-
'resiliency_policy_arn': '',
266-
'resiliency_policy_name': '',
267-
'last_assessment_time': '',
268-
'last_assessment_arn': '',
269-
'resiliency_status': app.get('status', ''),
270-
'resiliency_drift_status': app.get('driftStatus', ''),
271-
'compliance_total': '0',
272-
'fis_tests_total': '0',
273-
'alarms_total': '0',
274-
'sops_total': '0',
275-
'compliance_outstanding': '0', # FIXME
276-
'fis_tests_outstanding': '0',
277-
'alarms_outstanding': '0',
278-
'sops_outstanding': '0',
279-
'compliance_excluded': '0',
280-
'fis_tests_excluded': '0',
281-
'alarms_excluded': '0',
282-
'sops_excluded': '0',
283-
'operational_recommendations_total': '0', # FIXME
284-
'operational_recommendations_outstanding': '0', # FIXME
285-
'operational_recommendations_excluded': '0', # FIXME
286-
'infrastructure_recommendations_total': '0', # FIXME
287-
'input_source_drifts': '0', # FIXME
288-
'app_component_drifts': '0', # FIXME
289-
'resiliency_score': '0' #from latest assessment
290-
}
291-
292257
# Get policy information
293258
policy_arn = app['policyArn']
294259
policy_id = policy_arn.split('/')[-1]
295-
app_data['resiliency_policy_arn'] = policy_arn
296-
app_data['resiliency_policy_id'] = policy_arn.split('/')[-1]
297260
try:
298261
policy_response = resilience_client.describe_resiliency_policy(policyArn=policy_arn)
299-
app_data['resiliency_policy_name'] = policy_response['policy'].get('policyName', '')
300262
with s3_uploader(f'{module_name}/{module_name}-resiliency_policy/payer_id={payer_id}/account_id={account_id}/region_code={region}/{policy_id}.json') as write_policy:
301263
write_policy(policy_response['policy'])
302264
303265
except Exception as e: #pylint: disable=broad-exception-caught
304-
print(f"Error getting policy details: {str(e)}")
266+
logger.warning(f"Error getting policy details: {str(e)}")
305267
306268
# Loop over list of assessments to get the latest successful
307269
latest_assessment = None
@@ -314,73 +276,32 @@ Resources:
314276
315277
# Get info from the latest successful assessment
316278
if latest_assessment:
317-
318-
app_data['last_assessment_time'] = latest_assessment['endTime']
319-
app_data['last_assessment_arn'] = assessment_arn = latest_assessment['assessmentArn']
320-
app_data['resiliency_score'] = latest_assessment.get('resiliencyScore')
279+
assessment_arn = latest_assessment['assessmentArn']
321280
322281
with s3_uploader(f'{module_name}/{module_name}-app_component_recommendations_latest/payer_id={payer_id}/account_id={account_id}/region_code={region}/app_id={app_id}/latest.json') as write:
323282
for rec in paginate(resilience_client.list_app_component_recommendations, 'componentRecommendations', assessmentArn=latest_assessment['assessmentArn']):
324283
rec['assessment_arn'] = assessment_arn
325-
print(f'Record Value Test {rec}')
326284
write(rec)
327285
328286
with s3_uploader(f'{module_name}/{module_name}-alarm_recommendations_latest/payer_id={payer_id}/account_id={account_id}/region_code={region}/app_id={app_id}/latest.json') as write:
329-
total = outstanding = excluded = 0
330287
for rec in paginate(resilience_client.list_alarm_recommendations, 'alarmRecommendations', assessmentArn=latest_assessment['assessmentArn']):
331-
status = rec.get('recommendationStatus', '') # 'recommendationStatus': 'Implemented'|'Inactive'|'NotImplemented'|'Excluded',
332-
total += 1
333-
if status == 'NotImplemented':
334-
outstanding += 1
335-
elif status == 'Excluded':
336-
excluded += 1
337288
rec['assessment_arn'] = assessment_arn
338289
write(rec)
339-
app_data['alarms_total'] = str(total)
340-
app_data['alarms_outstanding'] = str(outstanding)
341-
app_data['alarms_excluded'] = str(excluded)
342290
343291
with s3_uploader(f'{module_name}/{module_name}-sop_recommendations_latest/payer_id={payer_id}/account_id={account_id}/region_code={region}/app_id={app_id}/latest.json') as write:
344-
total = outstanding = excluded = 0
345292
for rec in paginate(resilience_client.list_sop_recommendations, 'sopRecommendations', assessmentArn=latest_assessment['assessmentArn']):
346-
status = rec.get('recommendationStatus', '') # 'recommendationStatus': 'Implemented'|'Inactive'|'NotImplemented'|'Excluded',
347-
total += 1
348-
if status == 'NotImplemented':
349-
outstanding += 1
350-
elif status == 'Excluded':
351-
excluded += 1
352293
rec['assessment_arn'] = assessment_arn
353294
write(rec)
354-
app_data['sops_total'] = str(total)
355-
app_data['sops_outstanding'] = str(outstanding)
356-
app_data['sops_excluded'] = str(excluded)
357295
358296
with s3_uploader(f'{module_name}/{module_name}-test_recommendations_latest/payer_id={payer_id}/account_id={account_id}/region_code={region}/app_id={app_id}/latest.json') as write:
359-
total = outstanding = excluded = 0
360297
for rec in paginate(resilience_client.list_test_recommendations, 'testRecommendations', assessmentArn=latest_assessment['assessmentArn']):
361-
test_type = rec.get('type', '') # 'type': 'Software'|'Hardware'|'AZ'|'Region'
362-
status = rec.get('recommendationStatus', '') # 'recommendationStatus': 'Implemented'|'Inactive'|'NotImplemented'|'Excluded',
363-
if test_type == 'Software': # FIS tests
364-
total += 1
365-
if status == 'NotImplemented':
366-
outstanding += 1
367-
elif status == 'Excluded':
368-
excluded += 1
369298
rec['assessment_arn'] = assessment_arn
370299
write(rec)
371-
app_data['fis_tests_total'] = str(total)
372-
app_data['fis_tests_outstanding'] = str(outstanding)
373-
app_data['fis_tests_excluded'] = str(excluded)
374300
375301
with s3_uploader(f'{module_name}/{module_name}-compliance_drifts_latest/payer_id={payer_id}/account_id={account_id}/region_code={region}/app_id={app_id}/latest.json') as write:
376-
app_component_drifts = 0
377302
for rec in paginate(resilience_client.list_app_assessment_compliance_drifts, 'complianceDrifts', assessmentArn=latest_assessment['assessmentArn']):
378-
drift_type = rec.get('driftType', '') # 'driftType': 'ApplicationCompliance'|'AppComponentResiliencyComplianceStatus',
379-
if drift_type == 'ApplicationCompliance':
380-
app_component_drifts += 1
381303
rec['assessment_arn'] = assessment_arn
382304
write(rec)
383-
app_data['app_component_drifts'] = str(app_component_drifts)
384305
385306
386307
with s3_uploader(f'{module_name}/{module_name}-app_assessment_latest/payer_id={payer_id}/account_id={account_id}/region_code={region}/app_id={app_id}/latest.json') as write:
@@ -394,9 +315,6 @@ Resources:
394315
rec['app_arn'] = app_arn
395316
write(rec)
396317
397-
398-
with s3_uploader(f'{module_name}/{module_name}-applications/payer_id={payer_id}/account={account_id}/{region}-{app_id}.json') as write_app:
399-
write_app(app_data)
400318
with s3_uploader(f'{module_name}/{module_name}-application-details/payer_id={payer_id}/account_id={account_id}/{region}-{app_id}.json') as write_app:
401319
write_app(app)
402320
@@ -405,12 +323,12 @@ Resources:
405323
s3.put_object(Bucket=BUCKET, Key=status_key, Body=json.dumps(status_obj), ContentType='application/json')
406324
407325
except Exception as e: #pylint: disable=broad-exception-caught
408-
print(f"Error pulling data in region {region}: {str(e)}")
326+
logger.error(f"Error pulling data in {account_id} {region}: {str(e)}")
409327
410328
return {
411329
'statusCode': 200,
412330
'body': {
413-
'message': 'Exports and copy completed successfully',
331+
'message': 'Completed successfully',
414332
'account': account_id,
415333
'bucket': BUCKET,
416334
}
@@ -471,7 +389,6 @@ Resources:
471389
DatabaseName: !Ref DatabaseName
472390
Targets:
473391
S3Targets:
474-
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-applications/"
475392
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-application-details/"
476393
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-resiliency_policy/"
477394
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-assessments/"
@@ -482,7 +399,7 @@ Resources:
482399
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-compliance_drifts_latest/"
483400
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-app_assessment_latest/"
484401
- Path: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-app_version_resources_latest/"
485-
402+
486403
SchemaChangePolicy:
487404
UpdateBehavior: UPDATE_IN_DATABASE
488405
DeleteBehavior: DEPRECATE_IN_DATABASE
@@ -497,96 +414,6 @@ Resources:
497414
}
498415
499416
# Resilience Hub Table Definitions
500-
ResilienceHubApplicationsTable:
501-
Type: AWS::Glue::Table
502-
Properties:
503-
CatalogId: !Ref AWS::AccountId
504-
DatabaseName: !Ref DatabaseName
505-
TableInput:
506-
Name: 'resilience_hub_applications'
507-
TableType: EXTERNAL_TABLE
508-
Parameters:
509-
EXTERNAL: "TRUE"
510-
classification: "json"
511-
typeOfData: "file"
512-
UPDATED_BY_CRAWLER: !Ref Crawler
513-
StorageDescriptor:
514-
Location: !Sub "s3://${DestinationBucket}/${CFDataName}/${CFDataName}-applications/"
515-
InputFormat: "org.apache.hadoop.mapred.TextInputFormat"
516-
OutputFormat: "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"
517-
SerdeInfo:
518-
SerializationLibrary: "org.openx.data.jsonserde.JsonSerDe"
519-
Parameters:
520-
paths: "account_id,alarms_excluded,alarms_outstanding,alarms_total,app_component_drifts,application_arn,application_id,application_name,compliance_excluded,compliance_outstanding,compliance_total,fis_tests_excluded,fis_tests_outstanding,fis_tests_total,infrastructure_recommendations_total,input_source_drifts,last_assessment_arn,last_assessment_time,operational_recommendations_excluded,operational_recommendations_outstanding,operational_recommendations_total,region,resiliency_drift_status,resiliency_policy_arn,resiliency_policy_id,resiliency_policy_name,resiliency_score,resiliency_status,sops_excluded,sops_outstanding,sops_total"
521-
Columns:
522-
- Name: account_id
523-
Type: string
524-
- Name: alarms_excluded
525-
Type: string
526-
- Name: alarms_outstanding
527-
Type: string
528-
- Name: alarms_total
529-
Type: string
530-
- Name: app_component_drifts
531-
Type: string
532-
- Name: application_arn
533-
Type: string
534-
- Name: application_id
535-
Type: string
536-
- Name: application_name
537-
Type: string
538-
- Name: compliance_excluded
539-
Type: string
540-
- Name: compliance_outstanding
541-
Type: string
542-
- Name: compliance_total
543-
Type: string
544-
- Name: fis_tests_excluded
545-
Type: string
546-
- Name: fis_tests_outstanding
547-
Type: string
548-
- Name: fis_tests_total
549-
Type: string
550-
- Name: infrastructure_recommendations_total
551-
Type: string
552-
- Name: input_source_drifts
553-
Type: string
554-
- Name: last_assessment_arn
555-
Type: string
556-
- Name: last_assessment_time
557-
Type: string
558-
- Name: operational_recommendations_excluded
559-
Type: string
560-
- Name: operational_recommendations_outstanding
561-
Type: string
562-
- Name: operational_recommendations_total
563-
Type: string
564-
- Name: region
565-
Type: string
566-
- Name: resiliency_drift_status
567-
Type: string
568-
- Name: resiliency_policy_arn
569-
Type: string
570-
- Name: resiliency_policy_id
571-
Type: string
572-
- Name: resiliency_policy_name
573-
Type: string
574-
- Name: resiliency_score
575-
Type: double
576-
- Name: resiliency_status
577-
Type: string
578-
- Name: sops_excluded
579-
Type: string
580-
- Name: sops_outstanding
581-
Type: string
582-
- Name: sops_total
583-
Type: string
584-
PartitionKeys:
585-
- Name: payer_id
586-
Type: string
587-
- Name: account
588-
Type: string
589-
590417
ResilienceHubApplicationDetailsTable:
591418
Type: AWS::Glue::Table
592419
Properties:

‎test/test_from_scratch.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -262,8 +262,8 @@ def test_circle_ci_feeds_data(athena):
262262
data = athena_query(athena=athena, sql_query='SELECT * FROM "optimization_data"."circle_ci_feeds_whats_new" LIMIT 10;')
263263
assert len(data) > 0, 'circle_ci_feeds_data is empty'
264264

265-
def test_resiliencehub_daily_assessment_data(athena):
266-
data = athena_query(athena=athena, sql_query='SELECT * FROM "optimization_data"."resiliencehub_daily_assessments" LIMIT 10;')
265+
def test_resilience_hub_application_details(athena):
266+
data = athena_query(athena=athena, sql_query='SELECT * FROM "optimization_data"."resilience_hub_application_details" LIMIT 10;')
267267
assert len(data) > 0, 'resiliencehub_daily_assessments is empty'
268268

269269
def test_content_of_summary_not_empty(s3):

0 commit comments

Comments
 (0)