Skip to content

Commit 278877a

Browse files
julian-elasticjulian-elastic
committed
Add additional checks for right pushable filters
1 parent e73996f commit 278877a

File tree

2 files changed

+143
-6
lines changed

2 files changed

+143
-6
lines changed

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,10 @@
1313
import org.elasticsearch.xpack.esql.core.expression.AttributeSet;
1414
import org.elasticsearch.xpack.esql.core.expression.Expression;
1515
import org.elasticsearch.xpack.esql.core.expression.Expressions;
16+
import org.elasticsearch.xpack.esql.core.expression.FoldContext;
17+
import org.elasticsearch.xpack.esql.core.expression.Literal;
1618
import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute;
19+
import org.elasticsearch.xpack.esql.core.type.DataType;
1720
import org.elasticsearch.xpack.esql.core.util.CollectionUtils;
1821
import org.elasticsearch.xpack.esql.expression.predicate.Predicates;
1922
import org.elasticsearch.xpack.esql.plan.logical.Enrich;
@@ -152,11 +155,15 @@ private static LogicalPlan pushDownPastJoin(Filter filter, Join join) {
152155
optimizationApplied = true;
153156
}
154157
// push the right scoped filter down to the right child
155-
if (scoped.rightFilters().isEmpty() == false) {
158+
if (scoped.rightFilters().isEmpty() == false && (join.right() instanceof Filter == false)) {
156159
// push the filter down to the right child
157-
right = new Filter(right.source(), right, Predicates.combineAnd(scoped.rightFilters()));
158-
// update the join with the new right child
159-
join = (Join) join.replaceRight(right);
160+
List<Expression> rightPushableFilters = buildRightPushableFilters(scoped.rightFilters());
161+
if (rightPushableFilters.isEmpty() == false) {
162+
right = new Filter(right.source(), right, Predicates.combineAnd(rightPushableFilters));
163+
// update the join with the new right child
164+
join = (Join) join.replaceRight(right);
165+
optimizationApplied = true;
166+
}
160167
// We still want to reapply the filters that we just applied to the right child,
161168
// so we do NOT update scoped, and we do NOT mark optimizationApplied as true.
162169
// This is because by pushing them on the right side, we filter what rows we get from the right side
@@ -187,6 +194,32 @@ private static LogicalPlan pushDownPastJoin(Filter filter, Join join) {
187194
return plan;
188195
}
189196

197+
/**
198+
* Builds the right pushable filters for the given expressions.
199+
*/
200+
private static List<Expression> buildRightPushableFilters(List<Expression> expressions) {
201+
return expressions.stream().filter(x -> isRightPushableFilter(x)).toList();
202+
}
203+
204+
/**
205+
* Determines if the given expression can be pushed down to the right side of a join.
206+
* A filter is right pushable if the filter's predicate evaluates to false or null when all fields are set to null
207+
*/
208+
private static boolean isRightPushableFilter(Expression filter) {
209+
// traverse the filter tree
210+
// replace any reference to an attribute with a null literal
211+
Expression nullifiedFilter = filter.transformUp(Attribute.class, r -> new Literal(r.source(), null, DataType.NULL));
212+
// try to fold the filter
213+
// check if the folded filter evaluates to false or null, if yes return true
214+
// pushable WHERE field > 1 (evaluates to null), WHERE field is NOT NULL (evaluates to false)
215+
// not pushable WHERE field is NULL (evaluates to true), WHERE coalesce(field, 10) = 10 (evaluates to true)
216+
if (nullifiedFilter.foldable()) {
217+
Object folded = nullifiedFilter.fold(FoldContext.small());
218+
return folded == null || Boolean.FALSE.equals(folded);
219+
}
220+
return false;
221+
}
222+
190223
private static Function<Expression, Expression> NO_OP = expression -> expression;
191224

192225
private static LogicalPlan maybePushDownPastUnary(

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java‎

Lines changed: 106 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1810,6 +1810,106 @@ public void testCombineOrderByThroughFilter() {
18101810
as(filter.child(), EsRelation.class);
18111811
}
18121812

1813+
/**
1814+
* Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang
1815+
* uages{f}#10 AS language_code#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]]
1816+
* \_Limit[1000[INTEGER],false]
1817+
* \_Filter[ISNULL(language_name{f}#19)]
1818+
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18]]
1819+
* |_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
1820+
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
1821+
*/
1822+
public void testDoNotPushDownIsNullFilterPastLookupJoin() {
1823+
var plan = plan("""
1824+
FROM test
1825+
| RENAME languages AS language_code
1826+
| LOOKUP JOIN languages_lookup ON language_code
1827+
| WHERE language_name IS NULL
1828+
""");
1829+
1830+
var project = as(plan, Project.class);
1831+
var limit = as(project.child(), Limit.class);
1832+
var filter = as(limit.child(), Filter.class);
1833+
var join = as(filter.child(), Join.class);
1834+
assertThat(join.right(), instanceOf(EsRelation.class));
1835+
}
1836+
1837+
/**
1838+
* Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang
1839+
* uages{f}#10 AS language_code#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]]
1840+
* \_Limit[1000[INTEGER],false]
1841+
* \_Filter[language_name{f}#19 > a[KEYWORD]]
1842+
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18]]
1843+
* |_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
1844+
* \_Filter[language_name{f}#19 > a[KEYWORD]]
1845+
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
1846+
*/
1847+
public void testPushDownGreaterThanFilterPastLookupJoin() {
1848+
var plan = plan("""
1849+
FROM test
1850+
| RENAME languages AS language_code
1851+
| LOOKUP JOIN languages_lookup ON language_code
1852+
| WHERE language_name > "a"
1853+
""");
1854+
1855+
var project = as(plan, Project.class);
1856+
var limit = as(project.child(), Limit.class);
1857+
var filter = as(limit.child(), Filter.class);
1858+
var join = as(filter.child(), Join.class);
1859+
var right = as(join.right(), Filter.class);
1860+
assertThat(right.condition().toString(), is("language_name > \"a\""));
1861+
}
1862+
1863+
/**
1864+
* Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang
1865+
* uages{f}#10 AS language_code#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]]
1866+
* \_Limit[1000[INTEGER],false]
1867+
* \_Filter[COALESCE(language_name{f}#19,a[KEYWORD]) == a[KEYWORD]]
1868+
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18]]
1869+
* |_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
1870+
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
1871+
*/
1872+
public void testDoNotPushDownCoalesceFilterPastLookupJoin() {
1873+
var plan = plan("""
1874+
FROM test
1875+
| RENAME languages AS language_code
1876+
| LOOKUP JOIN languages_lookup ON language_code
1877+
| WHERE COALESCE(language_name, "a") == "a"
1878+
""");
1879+
1880+
var project = as(plan, Project.class);
1881+
var limit = as(project.child(), Limit.class);
1882+
var filter = as(limit.child(), Filter.class);
1883+
var join = as(filter.child(), Join.class);
1884+
assertThat(join.right(), instanceOf(EsRelation.class));
1885+
}
1886+
1887+
/**
1888+
* Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, lang
1889+
* uages{f}#10 AS language_code#4, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_name{f}#19]]
1890+
* \_Limit[1000[INTEGER],false]
1891+
* \_Filter[ISNOTNULL(language_name{f}#19)]
1892+
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18]]
1893+
* |_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
1894+
* \_Filter[ISNOTNULL(language_name{f}#19)]
1895+
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
1896+
*/
1897+
public void testPushDownIsNotNullFilterPastLookupJoin() {
1898+
var plan = plan("""
1899+
FROM test
1900+
| RENAME languages AS language_code
1901+
| LOOKUP JOIN languages_lookup ON language_code
1902+
| WHERE language_name IS NOT NULL
1903+
""");
1904+
1905+
var project = as(plan, Project.class);
1906+
var limit = as(project.child(), Limit.class);
1907+
var filter = as(limit.child(), Filter.class);
1908+
var join = as(filter.child(), Join.class);
1909+
var right = as(join.right(), Filter.class);
1910+
assertThat(right.condition().toString(), is("language_name IS NOT NULL"));
1911+
}
1912+
18131913
/**
18141914
* Expected
18151915
* <pre>{@code
@@ -6973,7 +7073,8 @@ public void testLookupJoinPushDownFilterOnLeftSideField() {
69737073
* \_Filter[language_name{f}#19 == [45 6e 67 6c 69 73 68][KEYWORD]]
69747074
* \_Join[LEFT,[languages{f}#10],[languages{f}#10],[language_code{f}#18]]
69757075
* |_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..]
6976-
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
7076+
* \_Filter[language_name{f}#19 == English[KEYWORD]]
7077+
* \_EsRelation[languages_lookup][LOOKUP][language_code{f}#18, language_name{f}#19]
69777078
* }</pre>
69787079
*/
69797080
public void testLookupJoinPushDownDisabledForLookupField() {
@@ -7002,7 +7103,10 @@ public void testLookupJoinPushDownDisabledForLookupField() {
70027103
assertThat(join.config().type(), equalTo(JoinTypes.LEFT));
70037104

70047105
var leftRel = as(join.left(), EsRelation.class);
7005-
var rightRel = as(join.right(), EsRelation.class);
7106+
var filterRight = as(join.right(), Filter.class);
7107+
assertEquals("language_name == \"English\"", filterRight.condition().toString());
7108+
var joinRightEsRelation = as(filterRight.child(), EsRelation.class);
7109+
70067110
}
70077111

70087112
/**

0 commit comments

Comments
 (0)