Skip to content

Commit 2cc9a51

Browse files
authored
Adding additional relational simplification patterns for datetime comparisons (#405)
Patterns include: - Switch comparisons between `QUARTER(x)` and a literal use `MONTH(x)` instead, e.g. `QUARTER(x) == 2` becomes `ISIN(MONTH(x), [4, 5, 6])` - Having patterns such as `MONTH(x) < 0` return False and `HOUR(x) <= 60` return True - Simplify `DATETIME(DATETIME(u, v, w), x, y, z)` -> `DATETIME(u, v, w, x, y, z)` - Simplify a chain of `DATETIME` information on a literal to produce the final literal (e.g. `DATETIME('2024-13-05', '+25 days', 'start of month')` -> `'2024-02-01`) - Simplify datetime part extraction on literals, e.g. `YEAR('2023-01-01')` ->` 2023` - Infer that a column `x` is non-null if it is used in a filter / join condition (for an inner/semi join) where the condition would be false if `x` were NULL - Simplify more functions to a NULL literal if any of their arguments are a NULL literal.
1 parent ebc0e40 commit 2cc9a51

File tree

91 files changed

+1777
-628
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+1777
-628
lines changed

pydough/conversion/filter_pushdown.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77

88
import pydough.pydough_operators as pydop
9+
from pydough.configs import PyDoughConfigs
910
from pydough.relational import (
1011
Aggregate,
1112
CallExpression,
@@ -65,7 +66,7 @@ class FilterPushdownShuttle(RelationalShuttle):
6566
cannot be pushed further.
6667
"""
6768

68-
def __init__(self):
69+
def __init__(self, configs: PyDoughConfigs):
6970
# The set of filters that are currently being pushed down. When
7071
# visit_xxx is called, it is presumed that the set of conditions in
7172
# self.filters are the conditions that can be pushed down as far as the
@@ -75,7 +76,7 @@ def __init__(self):
7576
# simplification logic to aid in advanced filter predicate inference,
7677
# such as determining that a left join is redundant because if the RHS
7778
# column is null then the filter will always be false.
78-
self.simplifier: SimplificationShuttle = SimplificationShuttle()
79+
self.simplifier: SimplificationShuttle = SimplificationShuttle(configs)
7980

8081
def reset(self):
8182
self.filters = set()
@@ -299,18 +300,19 @@ def visit_empty_singleton(self, empty_singleton: EmptySingleton) -> RelationalNo
299300
return self.flush_remaining_filters(empty_singleton, self.filters, set())
300301

301302

302-
def push_filters(node: RelationalNode) -> RelationalNode:
303+
def push_filters(node: RelationalNode, configs: PyDoughConfigs) -> RelationalNode:
303304
"""
304305
Transpose filter conditions down as far as possible.
305306
306307
Args:
307308
`node`: The current node of the relational tree.
309+
`configs`: The PyDough configuration settings.
308310
309311
Returns:
310312
The transformed version of `node` and all of its descendants with
311313
filters pushed down as far as possible, either materializing them above
312314
the node or into one of its inputs, or possibly both if there are
313315
multiple filters.
314316
"""
315-
pusher: FilterPushdownShuttle = FilterPushdownShuttle()
317+
pusher: FilterPushdownShuttle = FilterPushdownShuttle(configs)
316318
return node.accept_shuttle(pusher)

pydough/conversion/relational_converter.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1451,7 +1451,7 @@ def optimize_relational_tree(
14511451
root = ColumnPruner().prune_unused_columns(root)
14521452

14531453
# Step 1: push filters down as far as possible
1454-
root = confirm_root(push_filters(root))
1454+
root = confirm_root(push_filters(root, configs))
14551455

14561456
# Step 2: merge adjacent projections, unless it would result in excessive
14571457
# duplicate subexpression computations.
@@ -1492,8 +1492,8 @@ def optimize_relational_tree(
14921492
# pullup and pushdown and so on.
14931493
for _ in range(2):
14941494
root = confirm_root(pullup_projections(root))
1495-
simplify_expressions(root, additional_shuttles)
1496-
root = confirm_root(push_filters(root))
1495+
simplify_expressions(root, configs, additional_shuttles)
1496+
root = confirm_root(push_filters(root, configs))
14971497
root = ColumnPruner().prune_unused_columns(root)
14981498

14991499
# Step 9: re-run projection merging, without pushing into joins. This

0 commit comments

Comments
 (0)