Merge branch 'main' into di-447-update-data-quality-tests

Author: annajgibson

terraform/etl/08-google-sheets-imports.tf

@@ -948,3 +948,25 @@ module "parking_cycle_hangar_manual_waiting_list" {
   google_sheet_import_schedule    = "cron(0 4 ? * * *)"
   spark_ui_output_storage_id      = module.spark_ui_output_storage_data_source.bucket_id
 }
+
+module "diplomatic_country_vrm_prefix_data_load" {
+  count                           = local.is_live_environment ? 1 : 0
+  source                          = "../modules/google-sheets-glue-job"
+  is_production_environment       = local.is_production_environment
+  identifier_prefix               = local.short_identifier_prefix
+  is_live_environment             = local.is_live_environment
+  glue_scripts_bucket_id          = module.glue_scripts_data_source.bucket_id
+  helper_module_key               = data.aws_s3_object.helpers.key
+  pydeequ_zip_key                 = data.aws_s3_object.pydeequ.key
+  glue_catalog_database_name      = module.department_parking_data_source.raw_zone_catalog_database_name
+  glue_temp_storage_bucket_url    = module.glue_temp_storage_data_source.bucket_url
+  glue_crawler_excluded_blobs     = local.glue_crawler_excluded_blobs
+  google_sheets_import_script_key = aws_s3_object.google_sheets_import_script.key
+  bucket_id                       = module.raw_zone_data_source.bucket_id
+  google_sheets_document_id       = "1ui4i40sEp_wVlxVjcOhcQnhp2ucPpAQ9hALGJ7K76i0"
+  google_sheets_worksheet_name    = "diplomatic_country_vrm_prefix_data_load"
+  department                      = module.department_parking_data_source
+  dataset_name                    = "diplomatic_country_vrm_prefix_data_load"
+  google_sheet_import_schedule    = "cron(0 4 * * 1)"
+  spark_ui_output_storage_id      = module.spark_ui_output_storage_data_source.bucket_id
+}

terraform/etl/09-spreadsheet-imports-from-g-drive.tf

@@ -747,11 +747,11 @@ module "bailiff_return" {
   landing_zone_bucket_id         = module.landing_zone_data_source.bucket_id
   landing_zone_kms_key_arn       = module.landing_zone_data_source.kms_key_arn
   landing_zone_bucket_arn        = module.landing_zone_data_source.bucket_arn
-  google_drive_document_id       = "1vV_DAzaPk-QfsUAa7qIWkuEqFAiCyDG_"
+  google_drive_document_id       = "1M6tYUhEgrz7VeSXwBiZONm6K7h7993Uo"
   glue_job_name                  = "bailiff_return"
   output_folder_name             = "g-drive"
   raw_zone_bucket_id             = module.raw_zone_data_source.bucket_id
-  input_file_name                = "bailiff_return/Bailiff_return @ 30-10-2024.csv"
+  input_file_name                = "bailiff_return/Bailiff_return @ 02-12-2024.csv"
   ingestion_schedule             = "cron(0 21 * * ? *)"
   enable_bookmarking             = true
   worksheets = {
@@ -782,11 +782,11 @@ module "bailiff_allocation" {
   landing_zone_bucket_id         = module.landing_zone_data_source.bucket_id
   landing_zone_kms_key_arn       = module.landing_zone_data_source.kms_key_arn
   landing_zone_bucket_arn        = module.landing_zone_data_source.bucket_arn
-  google_drive_document_id       = "1FBY6mmfrNPD9Wzo2njqxNHvfU78AC3CS"
+  google_drive_document_id       = "1qxVzla3QRUu5B-WvPG6TVxK27rcVkAIL"
   glue_job_name                  = "bailiff_allocation"
   output_folder_name             = "g-drive"
   raw_zone_bucket_id             = module.raw_zone_data_source.bucket_id
-  input_file_name                = "bailiff_allocation/Bailiff Allocation @ 30-10-2024.csv"
+  input_file_name                = "bailiff_allocation/Bailiff Allocation @ 02-12-2024.csv"
   ingestion_schedule             = "cron(0 21 * * ? *)"
   enable_bookmarking             = true
   worksheets = {