diff --git a/docs/changelog/127383.yaml b/docs/changelog/127383.yaml new file mode 100644 index 0000000000000..056ee187a257e --- /dev/null +++ b/docs/changelog/127383.yaml @@ -0,0 +1,5 @@ +pr: 127383 +summary: Don't push down filters on the right hand side of an inlinejoin +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java index 5610db793abe5..911d02a8e15ce 100644 --- a/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java @@ -48,7 +48,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V2; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V5; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.INLINESTATS_V6; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_LOOKUP_V12; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.JOIN_PLANNING_V1; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METADATA_FIELDS_REMOTE_TEST; @@ -126,7 +126,7 @@ protected void shouldSkipTest(String testName) throws IOException { assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V2.capabilityName())); assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_PLANNING_V1.capabilityName())); - assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V5.capabilityName())); + assumeFalse("INLINESTATS not yet supported in CCS", testCase.requiredCapabilities.contains(INLINESTATS_V6.capabilityName())); assumeFalse("LOOKUP JOIN not yet supported in CCS", testCase.requiredCapabilities.contains(JOIN_LOOKUP_V12.capabilityName())); // Unmapped fields require a coorect capability response from every cluster, which isn't currently implemented. assumeFalse("UNMAPPED FIELDS not yet supported in CCS", testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName())); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index 051173b7b94b7..f523e7ff7ab0a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -3,7 +3,7 @@ // maxOfInt -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 // tag::max-languages[] FROM employees | KEEP emp_no, languages @@ -25,7 +25,7 @@ emp_no:integer | languages:integer | max_lang:integer ; maxOfIntByKeyword -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, languages, gender @@ -43,7 +43,7 @@ emp_no:integer | languages:integer | max_lang:integer | gender:keyword ; maxOfLongByKeyword -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, avg_worked_seconds, gender @@ -58,7 +58,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | gender: ; maxOfLong -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, avg_worked_seconds, gender @@ -71,7 +71,7 @@ emp_no:integer | avg_worked_seconds:long | gender:keyword | max_avg_worked_secon ; maxOfLongByCalculatedKeyword -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 // tag::longest-tenured-by-first[] FROM employees @@ -94,7 +94,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se ; maxOfLongByCalculatedNamedKeyword -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, avg_worked_seconds, last_name @@ -113,7 +113,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se ; maxOfLongByCalculatedDroppedKeyword -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | INLINESTATS max_avg_worked_seconds = MAX(avg_worked_seconds) BY l = SUBSTRING(last_name, 0, 1) @@ -132,7 +132,7 @@ emp_no:integer | avg_worked_seconds:long | last_name:keyword | max_avg_worked_se ; maxOfLongByEvaledKeyword -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | EVAL l = SUBSTRING(last_name, 0, 1) @@ -152,7 +152,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | l:keywo ; maxOfLongByInt -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, avg_worked_seconds, languages @@ -170,7 +170,7 @@ emp_no:integer | avg_worked_seconds:long | max_avg_worked_seconds:long | languag ; maxOfLongByIntDouble -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, avg_worked_seconds, languages, height @@ -205,8 +205,8 @@ emp_no:integer | languages:integer | avg_worked_seconds:long | gender:keyword | 10007 | 4 | 393084805 | F | 2.863684210555556E8 | 5 ; -byMultivaluedSimple-Ignore -required_capability: join_planning_v1 +byMultivaluedSimple +required_capability: inlinestats_v6 // tag::mv-group[] FROM airports @@ -223,8 +223,8 @@ abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer // end::mv-group-result[] ; -byMultivaluedMvExpand-Ignore -required_capability: join_planning_v1 +byMultivaluedMvExpand +required_capability: inlinestats_v6 // tag::mv-expand[] FROM airports @@ -237,14 +237,14 @@ FROM airports ; // tag::mv-expand-result[] -abbrev:keyword | type:keyword | scalerank:integer | min_scalerank:integer - GWL | mid | 9 | 2 - GWL | military | 9 | 4 +abbrev:keyword | scalerank:integer | min_scalerank:integer | type:keyword +GWL |9 |2 |mid +GWL |9 |4 |military // end::mv-expand-result[] ; byMvExpand -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 // tag::extreme-airports[] FROM airports @@ -308,7 +308,7 @@ count:long | country:keyword | avg:double ; afterWhere -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM airports | WHERE country != "United States" @@ -367,7 +367,7 @@ abbrev:keyword | city:keyword | region:text | "COUNT(*)":long ; beforeStats -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM airports | EVAL lat = ST_Y(location) @@ -380,7 +380,7 @@ northern:long | southern:long ; beforeKeepSort -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | INLINESTATS max_salary = MAX(salary) by languages @@ -394,8 +394,8 @@ emp_no:integer | languages:integer | max_salary:integer 10003 | 4 | 74572 ; -beforeKeepWhere-Ignore -required_capability: join_planning_v1 +beforeKeepWhere +required_capability: inlinestats_v6 FROM employees | INLINESTATS max_salary = MAX(salary) by languages @@ -407,8 +407,10 @@ emp_no:integer | languages:integer | max_salary:integer 10003 | 4 | 74572 ; -beforeEnrich-Ignore +beforeEnrich required_capability: join_planning_v1 +required_capability: inlinestats_v6 +required_capability: enrich_load FROM airports | KEEP abbrev, type, city @@ -419,10 +421,10 @@ FROM airports | LIMIT 3 ; -abbrev:keyword | type:keyword | city:keyword | "COUNT(*)":long | region:text - ABJ | mid | Abidjan | 499 | Abidjan - ABV | major | Abuja | 385 | Municipal Area Council - ACA | major | Acapulco de Juárez | 385 | Acapulco de Juárez +abbrev:keyword |city:keyword |"COUNT(*)":long|type:keyword | region:text +ABJ |Abidjan |499 |mid |Abidjan +ABV |Abuja |385 |major |Municipal Area Council +ACA |Acapulco de Juárez|385 |major |Acapulco de Juárez ; beforeAndAfterEnrich-Ignore @@ -501,7 +503,7 @@ Zürich | Zürich ; byConstant -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, languages @@ -520,7 +522,7 @@ emp_no:integer | languages:integer | max_lang:integer | y:integer ; aggConstant -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no @@ -538,7 +540,7 @@ one:integer | emp_no:integer ; percentile -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, salary @@ -557,7 +559,7 @@ emp_no:integer | salary:integer | ninety_fifth_salary:double ; byTwoCalculated -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM airports | WHERE abbrev IS NOT NULL @@ -642,7 +644,7 @@ abbrev:keyword | scalerank:integer | location:geo_point ; groupShadowsField -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, salary, hire_date @@ -661,7 +663,7 @@ emp_no:integer | salary:integer | avg_salary:double | hire_date:datetime ; groupByExpression_And_ExistentField -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, languages, gender | EVAL x = "ABC" @@ -679,7 +681,7 @@ emp_no:integer | languages:integer | x:keyword | max_lang:integer | y:keyword | ; groupByRenamedColumn -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, languages, gender | INLINESTATS max_lang = MAX(languages) BY y = gender @@ -787,7 +789,7 @@ emp_no:integer | languages:integer | gender:keyword|max_lang:integer| y:keyword ; twoAggregatesGroupedBy_AField_And_AnExpression -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, languages, gender, last_name @@ -809,7 +811,7 @@ emp_no:integer |languages:integer|last_name:keyword|max_lang:integer|min_lang:in ; groupByMultipleRenamedColumns_InversedOrder -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, languages, still_hired, gender @@ -827,7 +829,7 @@ emp_no:integer |languages:integer|still_hired:boolean| gender:keyword|max_lang:i ; groupByMultipleRenamedColumns_InversedOrder_ComplexEval -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, languages, still_hired, gender @@ -846,7 +848,7 @@ emp_no:integer |languages:integer|still_hired:boolean| gender:keyword|multilingu ; groupByMultipleRenamedColumns_AndComplexEval -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, languages, still_hired, gender @@ -888,7 +890,7 @@ emp_no:integer |languages:integer|gender:keyword |first_name:keyword | x:keyw ; groupByRenamedExpression -required_capability: inlinestats_v5 +required_capability: inlinestats_v6 FROM employees | KEEP emp_no, languages, gender, last_name @@ -908,4 +910,62 @@ emp_no:integer |languages:integer|last_name:keyword|max_lang:integer|min_lang:in 10049 |5 |Tramer |5 |5 |T |F 10028 |null |Tempesti |1 |1 |T |M ; + +doubleFilterOnLeftAndRight_InlineStats_Sides +required_capability: inlinestats_v6 +FROM employees +| INLINESTATS max_salary = MAX(salary), min_salary = MIN(salary) by languages +| KEEP emp_no, languages, *salary +| WHERE salary > 65000 and languages > 2 +| SORT emp_no +; + +emp_no:integer |languages:integer|salary:integer |max_salary:integer|min_salary:integer +10007 |4 |74572 |74572 |27215 +10030 |3 |67492 |74970 |26436 +10045 |3 |74970 |74970 |26436 +10054 |4 |65367 |74572 |27215 +10062 |3 |65030 |74970 |26436 +10094 |5 |66817 |66817 |25324 +10097 |3 |71165 |74970 |26436 +10100 |4 |68431 |74572 |27215 +; + +filterOnInlineStatsAggs +required_capability: inlinestats_v6 + +FROM employees +| INLINESTATS max_salary = MAX(salary), min_salary = MIN(salary) by languages +| KEEP emp_no, languages, *salary +| WHERE min_salary > 27000 or max_salary < 70000 +| sort salary desc +| limit 5 +; + +emp_no:integer |languages:integer|salary:integer |max_salary:integer|min_salary:integer +10029 |null |74999 |74999 |28336 +10007 |4 |74572 |74572 |27215 +10027 |null |73851 |74999 |28336 +10099 |2 |73578 |73578 |29175 +10078 |2 |69904 |73578 |29175 +; + +filterOnInlineStatsAggsValues_And_Groupings +required_capability: inlinestats_v6 + +FROM employees +| INLINESTATS max_salary = MAX(salary), min_salary = MIN(salary) by languages +| KEEP emp_no, languages, *salary +| WHERE (min_salary > 27000 or max_salary < 70000) and languages > 3 +| sort salary +| limit 5 +; + +emp_no:integer |languages:integer|salary:integer |max_salary:integer|min_salary:integer +10015 |5 |25324 |66817 |25324 +10035 |5 |25945 |66817 |25324 +10057 |4 |27215 |74572 |27215 +10011 |5 |31120 |66817 |25324 +10066 |5 |31897 |66817 |25324 +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 65e1237ef6b9e..cd6e6d4093883 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -845,7 +845,7 @@ public enum Cap { * Fixes a series of issues with inlinestats which had an incomplete implementation after lookup and inlinestats * were refactored. */ - INLINESTATS_V5(EsqlPlugin.INLINESTATS_FEATURE_FLAG), + INLINESTATS_V6(EsqlPlugin.INLINESTATS_FEATURE_FLAG), /** * Allow mixed numeric types in conditional functions - case, greatest and least diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java index f1f139bc2b0f2..e6f6d743b6ed7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineFilters.java @@ -24,6 +24,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.RegexExtract; import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; +import org.elasticsearch.xpack.esql.plan.logical.join.InlineJoin; import org.elasticsearch.xpack.esql.plan.logical.join.Join; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; @@ -79,7 +80,10 @@ protected LogicalPlan rule(Filter filter) { } else if (child instanceof OrderBy orderBy) { // swap the filter with its child plan = orderBy.replaceChild(filter.with(orderBy.child(), condition)); - } else if (child instanceof Join join) { + } else if (child instanceof Join join && child instanceof InlineJoin == false) { + // TODO: could we do better here about pushing down filters for inlinestats? + // See also https://github.com/elastic/elasticsearch/issues/127497 + // Push down past INLINESTATS if the condition is on the groupings return pushDownPastJoin(filter, join); } // cannot push past a Limit, this could change the tailing result set returned diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java index 8f2063146cabd..e3adf45b61e41 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PropagateInlineEvalsTests.java @@ -81,7 +81,7 @@ public static void init() { * \_StubRelation[[emp_no{f}#11, languages{f}#14, gender{f}#13, y{r}#10]] */ public void testGroupingAliasingMoved_To_LeftSideOfJoin() { - assumeTrue("Requires INLINESTATS", EsqlCapabilities.Cap.INLINESTATS_V5.isEnabled()); + assumeTrue("Requires INLINESTATS", EsqlCapabilities.Cap.INLINESTATS_V6.isEnabled()); var plan = plan(""" from test | keep emp_no, languages, gender @@ -124,7 +124,7 @@ public void testGroupingAliasingMoved_To_LeftSideOfJoin() { * {r}#21]] */ public void testGroupingAliasingMoved_To_LeftSideOfJoin_WithExpression() { - assumeTrue("Requires INLINESTATS", EsqlCapabilities.Cap.INLINESTATS_V5.isEnabled()); + assumeTrue("Requires INLINESTATS", EsqlCapabilities.Cap.INLINESTATS_V6.isEnabled()); var plan = plan(""" from test | keep emp_no, languages, gender, last_name, first_name