reduce buffer in pushdown predicate

timburke-hackit · timburke-hackit · commit 4ecd71fcfc6c · 2025-05-14T15:12:08.000+01:00
diff --git a/scripts/jobs/planning/tascomi_create_daily_snapshot.py b/scripts/jobs/planning/tascomi_create_daily_snapshot.py
@@ -244,7 +244,7 @@ def purge_today_partition(
             # snapshot table in glue catalogue
             else:
                 pushDownPredicate = create_pushdown_predicate(
-                    partitionDateColumn="snapshot_date", daysBuffer=60
+                    partitionDateColumn="snapshot_date", daysBuffer=14
                 )
                 #   load latest snpashot
                 snapshot_ddf = glueContext.create_dynamic_frame.from_catalog(

Original file line number	Diff line number	Diff line change
`@@ -244,7 +244,7 @@ def purge_today_partition(`
`244`	`244`	`# snapshot table in glue catalogue`
`245`	`245`	`else:`
`246`	`246`	`pushDownPredicate = create_pushdown_predicate(`
`247`		`- partitionDateColumn="snapshot_date", daysBuffer=60`
	`247`	`+ partitionDateColumn="snapshot_date", daysBuffer=14`
`248`	`248`	`)`
`249`	`249`	`# load latest snpashot`
`250`	`250`	`snapshot_ddf = glueContext.create_dynamic_frame.from_catalog(`