reinstate pushdown predicates

timburke-hackit · timburke-hackit · commit 1f7c70a5f8e1 · 2025-05-01T11:25:01.000+01:00
diff --git a/scripts/jobs/parking/parking_cedar_fulling_total_summary.py b/scripts/jobs/parking/parking_cedar_fulling_total_summary.py
@@ -9,6 +9,7 @@
 
 from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var
 
+
 environment = get_glue_env_var("environment")
 
 
@@ -31,8 +32,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
     database="dataplatform-" + environment + "-liberator-refined-zone",
     table_name="parking_cedar_payments",
     transformation_ctx="AmazonS3_node1625732038443",
-    # teporarily removed while table partitions are fixed
-    # push_down_predicate=create_pushdown_predicate("import_date", 7),
+    push_down_predicate=create_pushdown_predicate("import_date", 7),
 )
 
 # Script generated for node Amazon S3
diff --git a/scripts/jobs/parking/parking_ceo_average_on_street_hrs_mins_secs.py b/scripts/jobs/parking/parking_ceo_average_on_street_hrs_mins_secs.py
@@ -9,6 +9,7 @@
 
 from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var
 
+
 environment = get_glue_env_var("environment")
 
 
@@ -31,17 +32,15 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
     database="dataplatform-" + environment + "-liberator-refined-zone",
     table_name="parking_ceo_on_street",
     transformation_ctx="AmazonS3_node1628173244776",
-    # teporarily removed while table partitions are fixed
-    # push_down_predicate=create_pushdown_predicate("import_date", 7),
+    push_down_predicate=create_pushdown_predicate("import_date", 7),
 )
 
 # Script generated for node Amazon S3
 AmazonS3_node1638273151502 = glueContext.create_dynamic_frame.from_catalog(
     database="dataplatform-" + environment + "-liberator-refined-zone",
     table_name="parking_ceo_summary",
     transformation_ctx="AmazonS3_node1638273151502",
-    # teporarily removed while table partitions are fixed
-    # push_down_predicate=create_pushdown_predicate("import_date", 7),
+    push_down_predicate=create_pushdown_predicate("import_date", 7),
 )
 
 # Script generated for node ApplyMapping
diff --git a/scripts/jobs/parking/parking_deployment_target_details.py b/scripts/jobs/parking/parking_deployment_target_details.py
@@ -9,6 +9,7 @@
 
 from scripts.helpers.helpers import create_pushdown_predicate, get_glue_env_var
 
+
 environment = get_glue_env_var("environment")
 
 
@@ -45,8 +46,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
     database="dataplatform-" + environment + "-liberator-refined-zone",
     table_name="parking_ceo_on_street",
     transformation_ctx="AmazonS3_node1633593851886",
-    # teporarily removed while table partitions are fixed
-    # push_down_predicate=create_pushdown_predicate("import_date", 7),
+    push_down_predicate=create_pushdown_predicate("import_date", 7),
 )
 
 # Script generated for node Amazon S3
diff --git a/scripts/jobs/parking/parking_permit_street_stress.py b/scripts/jobs/parking/parking_permit_street_stress.py
@@ -53,8 +53,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
     database="dataplatform-" + environment + "-liberator-refined-zone",
     table_name="parking_permit_denormalised_data",
     transformation_ctx="AmazonS3_node1681807784480",
-    # teporarily removed while table partitions are fixed
-    # push_down_predicate=create_pushdown_predicate("import_date", 7),
+    push_down_predicate=create_pushdown_predicate("import_date", 7),
 )
 
 # Script generated for node SQL
diff --git a/scripts/jobs/parking/parking_permit_street_stress_with_cpz.py b/scripts/jobs/parking/parking_permit_street_stress_with_cpz.py
@@ -53,8 +53,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
     database="dataplatform-" + environment + "-liberator-refined-zone",
     table_name="parking_permit_denormalised_data",
     transformation_ctx="AmazonS3_node1681807784480",
-    # teporarily removed while table partitions are fixed
-    # push_down_predicate=create_pushdown_predicate("import_date", 7),
+    push_down_predicate=create_pushdown_predicate("import_date", 7),
 )
 
 # Script generated for node SQL
diff --git a/scripts/jobs/parking/parking_suspensions_processed.py b/scripts/jobs/parking/parking_suspensions_processed.py
@@ -45,8 +45,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
     database="dataplatform-" + environment + "-liberator-refined-zone",
     table_name="parking_suspension_denormalised_data",
     transformation_ctx="AmazonS3_node1661350417347",
-    # teporarily removed while table partitions are fixed
-    # push_down_predicate=create_pushdown_predicate("import_date", 7),
+    push_down_predicate=create_pushdown_predicate("import_date", 7),
 )
 
 # Script generated for node SQL
diff --git a/scripts/jobs/parking/parking_suspensions_processed_with_finyear.py b/scripts/jobs/parking/parking_suspensions_processed_with_finyear.py
@@ -45,8 +45,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra
     database="dataplatform-" + environment + "-liberator-refined-zone",
     table_name="parking_suspension_denormalised_data",
     transformation_ctx="AmazonS3_node1661350417347",
-    # teporarily removed while table partitions are fixed
-    # push_down_predicate=create_pushdown_predicate("import_date", 7),
+    push_down_predicate=create_pushdown_predicate("import_date", 7),
 )
 
 # Script generated for node Amazon S3

Original file line number	Diff line number	Diff line change
`@@ -53,8 +53,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra`
`53`	`53`	`database="dataplatform-" + environment + "-liberator-refined-zone",`
`54`	`54`	`table_name="parking_permit_denormalised_data",`
`55`	`55`	`transformation_ctx="AmazonS3_node1681807784480",`
`56`		`- # teporarily removed while table partitions are fixed`
`57`		`- # push_down_predicate=create_pushdown_predicate("import_date", 7),`
	`56`	`+ push_down_predicate=create_pushdown_predicate("import_date", 7),`
`58`	`57`	`)`
`59`	`58`
`60`	`59`	`# Script generated for node SQL`
Original file line number	Diff line number	Diff line change
`@@ -45,8 +45,7 @@ def sparkSqlQuery(glueContext, query, mapping, transformation_ctx) -> DynamicFra`
`45`	`45`	`database="dataplatform-" + environment + "-liberator-refined-zone",`
`46`	`46`	`table_name="parking_suspension_denormalised_data",`
`47`	`47`	`transformation_ctx="AmazonS3_node1661350417347",`
`48`		`- # teporarily removed while table partitions are fixed`
`49`		`- # push_down_predicate=create_pushdown_predicate("import_date", 7),`
	`48`	`+ push_down_predicate=create_pushdown_predicate("import_date", 7),`
`50`	`49`	`)`
`51`	`50`
`52`	`51`	`# Script generated for node SQL`