Skip to content

Commit 0cd761f

Browse files
authored
Update 38-aws-glue-job-parking.tf (#2043)
add Parking_interim_cycle_hangar_waiting_list code
1 parent 7c25d8e commit 0cd761f

File tree

1 file changed

+24
-0
lines changed

1 file changed

+24
-0
lines changed

terraform/etl/38-aws-glue-job-parking.tf

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1752,3 +1752,27 @@ module "parking_cycle_hangar_interim_wait_list" {
17521752
"--conf" = "spark.sql.legacy.timeParserPolicy=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInWrite=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=LEGACY"
17531753
}
17541754
}
1755+
# MRB 08-11-2024 job created
1756+
module "Parking_interim_cycle_hangar_waiting_list" {
1757+
source = "../modules/aws-glue-job"
1758+
is_live_environment = local.is_live_environment
1759+
is_production_environment = local.is_production_environment
1760+
department = module.department_parking_data_source
1761+
job_name = "${local.short_identifier_prefix}Parking_interim_cycle_hangar_waiting_list"
1762+
helper_module_key = data.aws_s3_object.helpers.key
1763+
pydeequ_zip_key = data.aws_s3_object.pydeequ.key
1764+
spark_ui_output_storage_id = module.spark_ui_output_storage_data_source.bucket_id
1765+
script_name = "Parking_interim_cycle_hangar_waiting_list"
1766+
triggered_by_job = "${local.short_identifier_prefix}Copy parking Liberator landing zone to raw"
1767+
job_description = "format the interim cycle hangar waiting list from Michael W. to remove the unwanted waiting list items"
1768+
workflow_name = "${local.short_identifier_prefix}parking-liberator-data-workflow"
1769+
trigger_enabled = local.is_production_environment
1770+
number_of_workers_for_glue_job = 2 # 2 minimum which is enough for this job
1771+
glue_job_worker_type = "G.1X"
1772+
glue_version = "4.0"
1773+
job_parameters = {
1774+
"--job-bookmark-option" = "job-bookmark-disable"
1775+
"--environment" = var.environment
1776+
"--conf" = "spark.sql.legacy.timeParserPolicy=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.int96RebaseModeInWrite=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInRead=LEGACY --conf spark.sql.legacy.parquet.datetimeRebaseModeInWrite=LEGACY"
1777+
}
1778+
}

0 commit comments

Comments
 (0)