Skip to content

Commit fdfc9c6

Browse files
esc1144esc1144
committed
Checkpoint
1 parent 5e70edd commit fdfc9c6

17 files changed

+3344
-548
lines changed

‎data-collection/deploy/account-collector.yaml‎

Lines changed: 41 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -130,31 +130,32 @@ Resources:
130130
131131
import boto3
132132
133+
BUCKET = os.environ.get("BUCKET_NAME")
133134
ROLE_NAME = os.environ.get('ROLE_NAME')
134-
RESOURCE_PREFIX = os.environ.get('RESOURCE_PREFIX')
135-
MANAGEMENT_ACCOUNT_IDS = os.environ.get('MANAGEMENT_ACCOUNT_IDS')
136-
BUCKET = os.environ.get('BUCKET_NAME')
137-
ROLE_NAME = os.environ.get('ROLE_NAME')
138-
RESOURCE_PREFIX = os.environ.get('RESOURCE_PREFIX')
139-
MANAGEMENT_ACCOUNT_IDS = os.environ.get('MANAGEMENT_ACCOUNT_IDS')
140-
BUCKET = os.environ.get('BUCKET_NAME')
141-
PREDEF_ACCOUNT_LIST_KEY = os.environ.get('PREDEF_ACCOUNT_LIST_KEY')
142-
LINKED_ACCOUNT_LIST_KEY = os.environ.get('LINKED_ACCOUNT_LIST_KEY')
143-
PAYER_ACCOUNT_LIST_KEY = os.environ.get('PAYER_ACCOUNT_LIST_KEY')
144-
EXCLUDED_ACCOUNT_LIST_KEY = os.environ.get('EXCLUDED_ACCOUNT_LIST_KEY')
135+
PREFIX = os.environ.get("PREFIX")
145136
TMP_FILE = "/tmp/data.json"
146-
PREFIX = "account-collector"
147137
START_TIME = str(datetime.now().isoformat())
138+
DC_REGION = boto3.session.Session().region_name
139+
DC_ACCOUNT = boto3.client('sts').get_caller_identity()['Account']
140+
141+
MANAGEMENT_ACCOUNT_IDS = os.environ.get('MANAGEMENT_ACCOUNT_IDS')
142+
PREDEF_ACCOUNT_LIST_KEY = os.environ.get('PREDEF_ACCOUNT_LIST_KEY', 'account-list/account-list')
143+
LINKED_ACCOUNT_LIST_KEY = os.environ.get('LINKED_ACCOUNT_LIST_KEY', 'linked-account-list.json')
144+
PAYER_ACCOUNT_LIST_KEY = os.environ.get('PAYER_ACCOUNT_LIST_KEY', 'payer-account-list.json')
145+
EXCLUDED_ACCOUNT_LIST_KEY = os.environ.get('EXCLUDED_ACCOUNT_LIST_KEY', 'excluded-linked-account-list.csv')
146+
147+
MODULE = ""
148148
149149
logger = logging.getLogger(__name__)
150150
logger.setLevel(getattr(logging, os.environ.get('LOG_LEVEL', 'INFO').upper(), logging.INFO))
151151
152152
def lambda_handler(event, context): #pylint: disable=unused-argument
153+
global MODULE
153154
logger.info(f"Incoming event: {event}")
154-
sub_uuid = [context.aws_request_id, context.log_group_name, context.log_stream_name]
155+
sub_uuid = {"lambda-request-id": context.aws_request_id, "lambda-log-group": context.log_group_name, "lambda-log-stream": context.log_stream_name}
155156
account_type = event.get("Type", '').lower()
156-
module = event.get("Module", '').lower()
157-
run_uuid = event.get("RunUUID", str(uuid.uuid4()))
157+
MODULE = event.get("Module", '').lower()
158+
main_exe_uuid = event.get("MainExeUuid", str(uuid.uuid4()))
158159
params = event.get("Params", "")
159160
try:
160161
# need to confirm that the Lambda concurrency limit is sufficient to avoid throttling
@@ -180,7 +181,13 @@ Resources:
180181
count = 0
181182
f.write("[\n")
182183
for account in account_iterator():
183-
account['run_uuid'] = run_uuid
184+
account['main_exe_uuid'] = main_exe_uuid
185+
account['module'] = MODULE
186+
account['bucket'] = BUCKET
187+
account['dc_account'] = DC_ACCOUNT
188+
account['dc_region'] = DC_REGION
189+
account['params'] = params
190+
account['prefix'] = PREFIX
184191
if count > 0:
185192
f.write(",\n")
186193
f.write(json.dumps(account))
@@ -190,12 +197,12 @@ Resources:
190197
if count == 0:
191198
raise Exception(STATUS_NOT_FOUND) #pylint: disable=broad-exception-raised
192199
193-
key = f"{PREFIX}/{module+'-'+(params+'-' if params else '')+(LINKED_ACCOUNT_LIST_KEY if account_type == 'linked' else PAYER_ACCOUNT_LIST_KEY)}"
200+
key = f"account-collector/{MODULE+'-'+(params+'-' if params else '')+(LINKED_ACCOUNT_LIST_KEY if account_type == 'linked' else PAYER_ACCOUNT_LIST_KEY)}"
194201
s3 = boto3.client('s3')
195202
s3.upload_file(TMP_FILE, Bucket=BUCKET, Key=key)
196203
location = f"s3://{BUCKET}/{key}"
197-
log_entry = create_log_entry(module=module, module_function=PREFIX+'-lambda', params=params, region="us-east-1", record_count=count,
198-
location=location, run_uuid=run_uuid, record_context="account", sub_uuid=sub_uuid)
204+
log_entry = create_log_entry(module_function='account-collector-lambda', params=params, region="us-east-1", record_count=count,
205+
location=location, main_exe_uuid=main_exe_uuid, record_context="account", sub_uuid=sub_uuid)
199206
return {'statusCode': 200, 'accountList': key, 'bucket': BUCKET, 'logEntry': log_entry['logEntry']}
200207
except Exception as exc: #pylint: disable=broad-exception-caught
201208
exc_msg = str(exc)
@@ -213,7 +220,7 @@ Resources:
213220
else:
214221
status_code = None
215222
description = None
216-
create_log_entry(module=module, module_function=PREFIX+'-lambda', region="us-east-1", status_code=status_code, description=description, error=exc, run_uuid=run_uuid, sub_uuid=sub_uuid)
223+
create_log_entry(module_function='account-collector-lambda', params=params, region="us-east-1", status_code=status_code, description=description, error=exc, main_exe_uuid=main_exe_uuid, sub_uuid=sub_uuid)
217224
raise exc
218225
219226
def get_all_payers():
@@ -225,12 +232,12 @@ Resources:
225232
for payer_id in get_all_payers():
226233
account_id = payer_id # default
227234
if service:
228-
ssm_key = f'/cid/{RESOURCE_PREFIX}config/delegated-admin/{service}/{payer_id}'
235+
ssm_key = f'/cid/{PREFIX}config/delegated-admin/{service}/{payer_id}'
229236
try:
230237
account_id = ssm.get_parameter(Name=ssm_key)['Parameter']['Value']
231238
except ssm.exceptions.ParameterNotFound:
232239
logger.warning(f'Not found ssm parameter {ssm_key}. Will use Management Account Id {payer_id}')
233-
yield {"account": json.dumps({'account_id': account_id, 'account_name': '', 'payer_id': payer_id}), "runuuid": ""}
240+
yield {"account": json.dumps({'account_id': account_id, 'account_name': '', 'payer_id': payer_id}), "main_exe_uuid": ""}
234241
235242
def iterate_linked_accounts():
236243
defined_accounts, ext = get_defined_list(BUCKET, PREDEF_ACCOUNT_LIST_KEY)
@@ -307,25 +314,15 @@ Resources:
307314
aws_session_token=credentials['SessionToken'],
308315
)
309316
310-
def create_log_entry(payer_id="", account_id=None, start_time=None, status_code=None, region="", module=None, module_function="module-lambda", sub_code="",
311-
params="", record_count=0, record_context="", description=None, location="", error=None, run_uuid="", sub_uuid=[], is_summary=False, store_it=True): # pylint: disable=too-many-locals
317+
def create_log_entry(payer_id="", account_id=None, status_code=None, region="", module_function="data-collection-lambda", sub_code="",
318+
params="", record_count=0, record_context="", description=None, location="", error=None, main_exe_uuid="", sub_uuid={}, is_summary=False, store_it=True): # pylint: disable=too-many-locals
312319
"""Format log entry for logging."""
313-
try:
314-
# get the local account and region
315-
dc_region = boto3.session.Session().region_name
316-
dc_account_id = boto3.client('sts').get_caller_identity()['Account']
317-
except Exception as exc: #pylint: disable=broad-exception-caught
318-
dc_region = ""
319-
dc_account_id = ""
320-
logger.error(f"{type(exc).__name__}: When trying to obtain local region and account information. Message: {str(exc)}")
321-
322320
status_code, description = status_handler(error, record_count, is_summary, status_code, description, record_context)
323321
log_entry = {
324-
"StartTime": start_time if start_time else START_TIME,
325-
"EndTime": str(datetime.now().isoformat()),
326-
"DataCollectionRegion": dc_region,
327-
"DataCollectionAccountId": dc_account_id,
328-
"Module": module if module else PREFIX,
322+
"Timestamp": str(datetime.now().isoformat()),
323+
"DataCollectionRegion": DC_REGION,
324+
"DataCollectionAccountId": DC_ACCOUNT,
325+
"Module": MODULE,
329326
"ModuleFunction": module_function,
330327
"Params": params,
331328
"PayerId": payer_id,
@@ -336,8 +333,8 @@ Resources:
336333
"RecordCount": record_count,
337334
"Description": description,
338335
"DataLocation": location if record_count > 0 else "",
339-
"RunUUID": run_uuid,
340-
"SubUUID": sub_uuid if isinstance(sub_uuid, list) else [sub_uuid],
336+
"MainExeUuid": main_exe_uuid,
337+
"SubUuid": sub_uuid,
341338
"Service": "Lambda"
342339
}
343340
if status_code >= 400:
@@ -371,7 +368,7 @@ Resources:
371368
372369
def store_log_entry(log_entry):
373370
"""Store the log entry to S3."""
374-
key = datetime.now().strftime(f"logs/%Y/%m/%d/{PREFIX}-{uuid.uuid4()}.json")
371+
key = datetime.now().strftime(f"logs/modules/%Y/%m/%d/{PREFIX}-{uuid.uuid4()}.json")
375372
try:
376373
boto3.client('s3').put_object(Body=json.dumps(log_entry), Bucket=BUCKET, Key=key)
377374
except Exception as exc: #pylint: disable=broad-exception-caught
@@ -396,12 +393,13 @@ Resources:
396393
Variables:
397394
ROLE_NAME: !Ref ManagementRoleName
398395
MANAGEMENT_ACCOUNT_IDS: !Ref ManagementAccountID
399-
RESOURCE_PREFIX: !Ref ResourcePrefix
400-
BUCKET_NAME: !Ref DestinationBucket
396+
PREFIX: !Ref ResourcePrefix
397+
BUCKET: !Ref DestinationBucket
401398
PREDEF_ACCOUNT_LIST_KEY: "account-list/account-list"
402399
LINKED_ACCOUNT_LIST_KEY: "linked-account-list.json"
403400
PAYER_ACCOUNT_LIST_KEY: "payer-account-list.json"
404401
EXCLUDED_ACCOUNT_LIST_KEY: "excluded-linked-account-list.csv"
402+
DC_ACCOUNT: !Ref AWS::AccountId
405403
Metadata:
406404
cfn_nag:
407405
rules_to_suppress:

0 commit comments

Comments
 (0)