@@ -1752,3 +1752,27 @@ module "parking_cycle_hangar_interim_wait_list" {
17521752 " --conf" = " spark.sql.legacy.timeParserPolicy=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInWrite=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=LEGACY"
17531753 }
17541754}
1755+ # MRB 08-11-2024 job created
1756+ module "Parking_interim_cycle_hangar_waiting_list" {
1757+ source = " ../modules/aws-glue-job"
1758+ is_live_environment = local. is_live_environment
1759+ is_production_environment = local. is_production_environment
1760+ department = module. department_parking_data_source
1761+ job_name = " ${ local . short_identifier_prefix } Parking_interim_cycle_hangar_waiting_list"
1762+ helper_module_key = data. aws_s3_object . helpers . key
1763+ pydeequ_zip_key = data. aws_s3_object . pydeequ . key
1764+ spark_ui_output_storage_id = module. spark_ui_output_storage_data_source . bucket_id
1765+ script_name = " Parking_interim_cycle_hangar_waiting_list"
1766+ triggered_by_job = " ${ local . short_identifier_prefix } Copy parking Liberator landing zone to raw"
1767+ job_description = " format the interim cycle hangar waiting list from Michael W. to remove the unwanted waiting list items"
1768+ workflow_name = " ${ local . short_identifier_prefix } parking-liberator-data-workflow"
1769+ trigger_enabled = local. is_production_environment
1770+ number_of_workers_for_glue_job = 2 # 2 minimum which is enough for this job
1771+ glue_job_worker_type = " G.1X"
1772+ glue_version = " 4.0"
1773+ job_parameters = {
1774+ " --job-bookmark-option" = " job-bookmark-disable"
1775+ " --environment" = var.environment
1776+ " --conf" = " spark.sql.legacy.timeParserPolicy=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInWrite=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=LEGACY"
1777+ }
1778+ }
0 commit comments