Temporary fix for Glue jobs impacted by the Address API (#2065)

Tian-2017 · web-flow · commit 4caf1771d1b2 · 2025-01-16T09:06:42.000Z
diff --git a/scripts/jobs/parking/parking_vouchers_approved_summary_gds.py b/scripts/jobs/parking/parking_vouchers_approved_summary_gds.py
@@ -48,7 +48,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
 # Script generated for node Amazon S3 - raw-zone-unrestricted-address-api - unrestricted_address_api_dbo_hackney_address
 AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032 = glueContext.create_dynamic_frame.from_catalog(
     database="dataplatform-" + environment + "-raw-zone-unrestricted-address-api",
-    push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)",
+    push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 30)",
     table_name="unrestricted_address_api_dbo_hackney_address",
     transformation_ctx="AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032",
 )
diff --git a/scripts/jobs/parking/parking_vouchers_approved_summary_gds_lbh.py b/scripts/jobs/parking/parking_vouchers_approved_summary_gds_lbh.py
@@ -32,7 +32,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
 # Script generated for node Amazon S3 - unrestricted_address_api_dbo_hackney_xref
 AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724077428047 = glueContext.create_dynamic_frame.from_catalog(
     database="dataplatform-" + environment + "-raw-zone-unrestricted-address-api",
-    push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)",
+    push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 30)",
     table_name="unrestricted_address_api_dbo_hackney_xref",
     transformation_ctx="AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724077428047",
 )

Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra`
`48`	`48`	`# Script generated for node Amazon S3 - raw-zone-unrestricted-address-api - unrestricted_address_api_dbo_hackney_address`
`49`	`49`	`AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032 = glueContext.create_dynamic_frame.from_catalog(`
`50`	`50`	`database="dataplatform-" + environment + "-raw-zone-unrestricted-address-api",`
`51`		`- push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)",`
	`51`	`+ push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 30)",`
`52`	`52`	`table_name="unrestricted_address_api_dbo_hackney_address",`
`53`	`53`	`transformation_ctx="AmazonS3rawzoneunrestrictedaddressapiunrestricted_address_api_dbo_hackney_address_node1720617251032",`
`54`	`54`	`)`
Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra`
`32`	`32`	`# Script generated for node Amazon S3 - unrestricted_address_api_dbo_hackney_xref`
`33`	`33`	`AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724077428047 = glueContext.create_dynamic_frame.from_catalog(`
`34`	`34`	`database="dataplatform-" + environment + "-raw-zone-unrestricted-address-api",`
`35`		`- push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 7)",`
	`35`	`+ push_down_predicate="to_date(import_date, 'yyyyMMdd') >= date_sub(current_date, 30)",`
`36`	`36`	`table_name="unrestricted_address_api_dbo_hackney_xref",`
`37`	`37`	`transformation_ctx="AmazonS3unrestricted_address_api_dbo_hackney_xref_node1724077428047",`
`38`	`38`	`)`