Skip to content

Commit 8e625c8

Browse files
authored
fix(sqlmesh): attempt to optimize stg_github__events to use a smaller window for lookbacks (#5474)
* fix(sqlmesh): attempt to optimize stg_github__events to use a smaller window for lookbacks Also makes it so the github events model tries to use days instead of months for the partitions (mostly to reduce size of query in trino) * Update warehouse/oso_sqlmesh/models/staging/github/stg_github__events.py
1 parent acbeeaa commit 8e625c8

File tree

1 file changed

+7
-4
lines changed

1 file changed

+7
-4
lines changed

warehouse/oso_sqlmesh/models/staging/github/stg_github__events.py

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,17 @@
3030
"time_column": "created_at",
3131
"batch_size": 90,
3232
"batch_concurrency": 3,
33-
"lookback": 31,
33+
"lookback": 10,
3434
"forward_only": True,
3535
},
3636
partitioned_by=("day(created_at)",),
3737
physical_properties={"max_commit_retry": 15},
3838
audits=[
3939
("has_at_least_n_rows", {"threshold": 0}),
40-
("no_gaps", {"time_column": exp.to_column("created_at"), "no_gap_date_part": "day"}),
40+
(
41+
"no_gaps",
42+
{"time_column": exp.to_column("created_at"), "no_gap_date_part": "day"},
43+
),
4144
],
4245
)
4346
def github_events(
@@ -55,7 +58,7 @@ def github_events(
5558

5659
import arrow
5760

58-
runtime_stage = context.var('runtime_stage')
61+
runtime_stage = context.var("runtime_stage")
5962

6063
if runtime_stage == "testing" or context.gateway != "trino":
6164
data = {
@@ -115,7 +118,7 @@ def github_events(
115118

116119
difference = end_arrow - start_arrow
117120
selects: t.List[exp.Select] = []
118-
if difference.days < 7:
121+
if difference.days <= 10:
119122
unit = "day"
120123
format = "YYYYMMDD"
121124
else:

0 commit comments

Comments
 (0)