Skip to content

Commit b53c687

Browse files
barnesm707barnesm707
authored
Update 09-spreadsheet-imports-from-g-drive.tf (#2446)
Bailiff Weekly data update 10-09-2025
1 parent 58a9c43 commit b53c687

File tree

1 file changed

+144
-0
lines changed

1 file changed

+144
-0
lines changed

‎terraform/etl/09-spreadsheet-imports-from-g-drive.tf‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1330,3 +1330,147 @@ module "bailiff_dashboard_link" {
13301330
}
13311331
}
13321332
}
1333+
module "Equita_Weekly_Data" {
1334+
count = local.is_live_environment ? 1 : 0
1335+
source = "../modules/import-spreadsheet-file-from-g-drive"
1336+
is_production_environment = local.is_production_environment
1337+
is_live_environment = local.is_live_environment
1338+
department = module.department_parking_data_source
1339+
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
1340+
glue_catalog_database_name = module.department_parking_data_source.raw_zone_catalog_database_name
1341+
glue_temp_storage_bucket_id = module.glue_temp_storage_data_source.bucket_url
1342+
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
1343+
secrets_manager_kms_key = data.aws_kms_key.secrets_manager_key
1344+
glue_role_arn = data.aws_iam_role.glue_role.arn
1345+
helper_module_key = data.aws_s3_object.helpers.key
1346+
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
1347+
jars_key = data.aws_s3_object.jars.key
1348+
spreadsheet_import_script_key = aws_s3_object.spreadsheet_import_script.key
1349+
identifier_prefix = local.short_identifier_prefix
1350+
lambda_artefact_storage_bucket = module.lambda_artefact_storage_data_source.bucket_id
1351+
landing_zone_bucket_id = module.landing_zone_data_source.bucket_id
1352+
landing_zone_kms_key_arn = module.landing_zone_data_source.kms_key_arn
1353+
landing_zone_bucket_arn = module.landing_zone_data_source.bucket_arn
1354+
google_drive_document_id = "1LR7b94HdpVylXHfONVhwAUwucipYYaMs"
1355+
glue_job_name = "Equita_Weekly_Data"
1356+
output_folder_name = "g-drive"
1357+
raw_zone_bucket_id = module.raw_zone_data_source.bucket_id
1358+
input_file_name = "Equita_Weekly_Data/equita_08_09_2025.csv"
1359+
ingestion_schedule = "cron(0 21 * * ? *)"
1360+
enable_bookmarking = true
1361+
tags = module.tags.values
1362+
worksheets = {
1363+
sheet1 : {
1364+
header_row_number = 0
1365+
worksheet_name = "Equita_Weekly_Data"
1366+
}
1367+
}
1368+
}
1369+
module "Marston_Weekly_Data" {
1370+
count = local.is_live_environment ? 1 : 0
1371+
source = "../modules/import-spreadsheet-file-from-g-drive"
1372+
is_production_environment = local.is_production_environment
1373+
is_live_environment = local.is_live_environment
1374+
department = module.department_parking_data_source
1375+
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
1376+
glue_catalog_database_name = module.department_parking_data_source.raw_zone_catalog_database_name
1377+
glue_temp_storage_bucket_id = module.glue_temp_storage_data_source.bucket_url
1378+
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
1379+
secrets_manager_kms_key = data.aws_kms_key.secrets_manager_key
1380+
glue_role_arn = data.aws_iam_role.glue_role.arn
1381+
helper_module_key = data.aws_s3_object.helpers.key
1382+
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
1383+
jars_key = data.aws_s3_object.jars.key
1384+
spreadsheet_import_script_key = aws_s3_object.spreadsheet_import_script.key
1385+
identifier_prefix = local.short_identifier_prefix
1386+
lambda_artefact_storage_bucket = module.lambda_artefact_storage_data_source.bucket_id
1387+
landing_zone_bucket_id = module.landing_zone_data_source.bucket_id
1388+
landing_zone_kms_key_arn = module.landing_zone_data_source.kms_key_arn
1389+
landing_zone_bucket_arn = module.landing_zone_data_source.bucket_arn
1390+
google_drive_document_id = "1jXskOkzjqgmQ81T4LFNZ4-qu2fl9yk2b"
1391+
glue_job_name = "Marston_Weekly_Data"
1392+
output_folder_name = "g-drive"
1393+
raw_zone_bucket_id = module.raw_zone_data_source.bucket_id
1394+
input_file_name = "Marston_Weekly_Data/marston_10_09_2025.csv"
1395+
ingestion_schedule = "cron(0 21 * * ? *)"
1396+
enable_bookmarking = true
1397+
tags = module.tags.values
1398+
worksheets = {
1399+
sheet1 : {
1400+
header_row_number = 0
1401+
worksheet_name = "Marston_Weekly_Data"
1402+
}
1403+
}
1404+
}
1405+
module "Newlyn_Weekly_Data" {
1406+
count = local.is_live_environment ? 1 : 0
1407+
source = "../modules/import-spreadsheet-file-from-g-drive"
1408+
is_production_environment = local.is_production_environment
1409+
is_live_environment = local.is_live_environment
1410+
department = module.department_parking_data_source
1411+
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
1412+
glue_catalog_database_name = module.department_parking_data_source.raw_zone_catalog_database_name
1413+
glue_temp_storage_bucket_id = module.glue_temp_storage_data_source.bucket_url
1414+
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
1415+
secrets_manager_kms_key = data.aws_kms_key.secrets_manager_key
1416+
glue_role_arn = data.aws_iam_role.glue_role.arn
1417+
helper_module_key = data.aws_s3_object.helpers.key
1418+
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
1419+
jars_key = data.aws_s3_object.jars.key
1420+
spreadsheet_import_script_key = aws_s3_object.spreadsheet_import_script.key
1421+
identifier_prefix = local.short_identifier_prefix
1422+
lambda_artefact_storage_bucket = module.lambda_artefact_storage_data_source.bucket_id
1423+
landing_zone_bucket_id = module.landing_zone_data_source.bucket_id
1424+
landing_zone_kms_key_arn = module.landing_zone_data_source.kms_key_arn
1425+
landing_zone_bucket_arn = module.landing_zone_data_source.bucket_arn
1426+
google_drive_document_id = "1If-AGk_htDmbCtsiXdBz6qwseAoF7Yyr"
1427+
glue_job_name = "Newlyn_Weekly_Data"
1428+
output_folder_name = "g-drive"
1429+
raw_zone_bucket_id = module.raw_zone_data_source.bucket_id
1430+
input_file_name = "Newlyn_Weekly_Data/newlyn_12_09_2025.csv"
1431+
ingestion_schedule = "cron(0 21 * * ? *)"
1432+
enable_bookmarking = true
1433+
tags = module.tags.values
1434+
worksheets = {
1435+
sheet1 : {
1436+
header_row_number = 0
1437+
worksheet_name = "Newlyn_Weekly_Data"
1438+
}
1439+
}
1440+
}
1441+
module "Whyte_Weekly_Data" {
1442+
count = local.is_live_environment ? 1 : 0
1443+
source = "../modules/import-spreadsheet-file-from-g-drive"
1444+
is_production_environment = local.is_production_environment
1445+
is_live_environment = local.is_live_environment
1446+
department = module.department_parking_data_source
1447+
glue_scripts_bucket_id = module.glue_scripts_data_source.bucket_id
1448+
glue_catalog_database_name = module.department_parking_data_source.raw_zone_catalog_database_name
1449+
glue_temp_storage_bucket_id = module.glue_temp_storage_data_source.bucket_url
1450+
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
1451+
secrets_manager_kms_key = data.aws_kms_key.secrets_manager_key
1452+
glue_role_arn = data.aws_iam_role.glue_role.arn
1453+
helper_module_key = data.aws_s3_object.helpers.key
1454+
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
1455+
jars_key = data.aws_s3_object.jars.key
1456+
spreadsheet_import_script_key = aws_s3_object.spreadsheet_import_script.key
1457+
identifier_prefix = local.short_identifier_prefix
1458+
lambda_artefact_storage_bucket = module.lambda_artefact_storage_data_source.bucket_id
1459+
landing_zone_bucket_id = module.landing_zone_data_source.bucket_id
1460+
landing_zone_kms_key_arn = module.landing_zone_data_source.kms_key_arn
1461+
landing_zone_bucket_arn = module.landing_zone_data_source.bucket_arn
1462+
google_drive_document_id = "1-RGssHGSKGpis4Bode6Iw9e6lStnM5TR"
1463+
glue_job_name = "Whyte_Weekly_Data"
1464+
output_folder_name = "g-drive"
1465+
raw_zone_bucket_id = module.raw_zone_data_source.bucket_id
1466+
input_file_name = "Whyte_Weekly_Data/whyte_10_09_2025.csv"
1467+
ingestion_schedule = "cron(0 21 * * ? *)"
1468+
enable_bookmarking = true
1469+
tags = module.tags.values
1470+
worksheets = {
1471+
sheet1 : {
1472+
header_row_number = 0
1473+
worksheet_name = "Whyte_Weekly_Data"
1474+
}
1475+
}
1476+
}

0 commit comments

Comments
 (0)