diff --git a/docs/changelog/125479.yaml b/docs/changelog/125479.yaml new file mode 100644 index 0000000000000..efc31441254c5 --- /dev/null +++ b/docs/changelog/125479.yaml @@ -0,0 +1,6 @@ +pr: 125479 +summary: ES|QL - Allow full text functions to be used in STATS +area: ES|QL +type: enhancement +issues: + - 125481 diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 98164298acd3a..4c2edd77ddc46 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -211,6 +211,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task -> task.skipTest("esql/61_enrich_ip/Invalid IP strings", "We switched from exceptions to null+warnings for ENRICH runtime errors") task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields") task.skipTest("esql/180_match_operator/match with functions", "Error message changed") + task.skipTest("esql/180_match_operator/match within eval", "Error message changed") task.skipTest("esql/40_unsupported_types/semantic_text declared in mapping", "The semantic text field format changed") task.skipTest("esql/190_lookup_join/Alias as lookup index", "LOOKUP JOIN does not support index aliases for now") task.skipTest("esql/190_lookup_join/alias-repeated-alias", "LOOKUP JOIN does not support index aliases for now") diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec index 884981d785689..f985973a8706b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec @@ -191,3 +191,82 @@ book_no:keyword 7140 2714 ; + +testKqlInStatsNonPushable +required_capability: kql_function +required_capability: full_text_functions_in_stats_where + +from books +| where length(title) > 40 +| stats c = count(*) where kql("title:Lord") +; + +c:long +3 +; + + +testMatchInStatsPushableAndNonPushable +required_capability: kql_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where (kql("title: lord") and ratings > 4.5) or (kql("author: dostoevsky") and length(title) > 50) +; + +c:long +6 +; + +testKqlInStatsPushable +required_capability: kql_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where kql("author:tolkien") +; + +c:long +22 +; + +testKqlInStatsWithNonPushableDisjunctions +required_capability: kql_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where kql("title: lord") or length(title) > 130 +; + +c:long +5 +; + +testKqlInStatsWithMultipleAggs +required_capability: kql_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where kql("title: lord"), m = max(book_no::integer) where kql("author: tolkien"), n = min(book_no::integer) where kql("author: dostoevsky") +; + +c:long | m:integer | n:integer +4 | 9607 | 1211 +; + + +testKqlInStatsWithGrouping +required_capability: kql_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS r = AVG(ratings) where kql("title: Lord AND Rings") by author | WHERE r is not null +; +ignoreOrder: true + +r:double | author: text +4.75 | Alan Lee +4.674999952316284 | J. R. R. Tolkien +4.670000076293945 | John Ronald Reuel Tolkien +4.670000076293945 | Agnes Perkins +4.670000076293945 | Charles Adolph Huttar +4.670000076293945 | Walter Scheps +4.559999942779541 | J.R.R. Tolkien +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec index 444c73414f8ba..05fc09b582ee4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec @@ -755,3 +755,94 @@ book_no:keyword 7140 2714 ; + +testMatchInStatsNonPushable +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +from books +| where length(title) > 40 +| stats c = count(*) where match(title, "Lord") +; + +c:long +3 +; + +testMatchInStatsPushableAndNonPushable +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where (match(title, "lord") and ratings > 4.5) or (match(author, "dostoevsky") and length(title) > 50) +; + +c:long +6 +; + +testMatchInStatsPushable +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where match(author, "tolkien") +; + +c:long +22 +; + +testMatchInStatsWithOptions +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +FROM books +| STATS c = count(*) where match(title, "Hobbit Back Again", {"operator": "AND"}) +; + +c:long +1 +; + +testMatchInStatsWithNonPushableDisjunctions +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +FROM books +| STATS c = count(*) where match(title, "lord") or length(title) > 130 +; + +c:long +5 +; + +testMatchInStatsWithMultipleAggs +required_capability: match_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where match(title, "lord"), m = max(book_no::integer) where match(author, "tolkien"), n = min(book_no::integer) where match(author, "dostoevsky") +; + +c:long | m:integer | n:integer +4 | 9607 | 1211 +; + + +testMatchInStatsWithGrouping +required_capability: match_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS r = AVG(ratings) where match(title, "Lord Rings", {"operator": "AND"}) by author | WHERE r is not null +; +ignoreOrder: true + +r:double | author: text +4.75 | Alan Lee +4.674999952316284 | J. R. R. Tolkien +4.670000076293945 | John Ronald Reuel Tolkien +4.670000076293945 | Agnes Perkins +4.670000076293945 | Charles Adolph Huttar +4.670000076293945 | Walter Scheps +4.559999942779541 | J.R.R. Tolkien +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec index 420225407d917..ee9203bf08b1b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec @@ -757,3 +757,29 @@ from semantic_text host:keyword | semantic_text_field:text | language_name:keyword | language_code:integer "host1" | live long and prosper | English | 1 ; + + +testMatchInStatsNonPushable +required_capability: match_operator_colon +required_capability: full_text_functions_in_stats_where + +from books +| where length(title) > 40 +| stats c = count(*) where title:"Lord" +; + +c:long +3 +; + +testMatchInStatsPushable +required_capability: match_operator_colon +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where author:"tolkien" +; + +c:long +22 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec index 08ac3d585a5b5..3458efdc872bc 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec @@ -211,3 +211,92 @@ book_no:keyword | title:text 7480 | The Hobbit // end::qstr-with-options-result[] ; + +testQstrInStatsNonPushable +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where + +from books +| where length(title) > 40 +| stats c = count(*) where qstr("title:Lord") +; + +c:long +3 +; + +testMatchInStatsPushableAndNonPushable +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where (qstr("title: lord") and ratings > 4.5) or (qstr("author: dostoevsky") and length(title) > 50) +; + +c:long +6 +; + +testQstrInStatsPushable +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where qstr("author:tolkien") +; + +c:long +22 +; + +testQstrInStatsWithOptions +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where + +FROM books +| STATS c = count(*) where qstr("title: Hobbit Back Again", {"default_operator": "AND"}) +; + +c:long +1 +; + +testQstrInStatsWithNonPushableDisjunctions +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where qstr("title: lord") or length(title) > 130 +; + +c:long +5 +; + +testQstrInStatsWithMultipleAggs +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where qstr("title: lord"), m = max(book_no::integer) where qstr("author: tolkien"), n = min(book_no::integer) where qstr("author: dostoevsky") +; + +c:long | m:integer | n:integer +4 | 9607 | 1211 +; + +testQstrInStatsWithGrouping +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS r = AVG(ratings) where qstr("title: Lord Rings", {"default_operator": "AND"}) by author | WHERE r is not null +; +ignoreOrder: true + +r:double | author: text +4.75 | Alan Lee +4.674999952316284 | J. R. R. Tolkien +4.670000076293945 | John Ronald Reuel Tolkien +4.670000076293945 | Agnes Perkins +4.670000076293945 | Charles Adolph Huttar +4.670000076293945 | Walter Scheps +4.559999942779541 | J.R.R. Tolkien +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec index ff89c4e14d4bb..09d0644d412e6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec @@ -454,7 +454,6 @@ book_no:keyword | _score:double 8678 | 0.0 ; - disjunctionScoresMultipleClauses required_capability: metadata_score @@ -477,3 +476,18 @@ book_no:keyword | _score:double 4023 | 1.5062403678894043 2924 | 1.2732219696044922 ; + +statsScores + +required_capability: metadata_score +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +from books metadata _score +| where match(title, "Lord Rings", {"operator": "AND"}) +| stats avg_score = avg(_score), max_score = max(_score), min_score = min(_score) +; + +avg_score:double | max_score:double | min_score:double +3.869828939437866 | 5.123856544494629 | 3.0124807357788086 +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java index 39c963e938ac5..8e0a5ed2e8ce9 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java @@ -66,7 +66,7 @@ public void testKqlQueryWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE and STATS commands")); } public void testInvalidKqlQueryEof() { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java index 2da9bee3701d7..0a204f9d79768 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java @@ -14,11 +14,13 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; +import org.hamcrest.Matchers; import org.junit.Before; import java.util.List; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; import static org.hamcrest.CoreMatchers.containsString; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug") @@ -250,6 +252,42 @@ public void testWhereMatchWithRow() { ); } + public void testMatchWithStats() { + var errorQuery = """ + FROM test + | STATS c = count(*) BY match(content, "fox") + """; + + var error = expectThrows(ElasticsearchException.class, () -> run(errorQuery)); + assertThat(error.getMessage(), containsString("[MATCH] function is only supported in WHERE and STATS commands")); + + var query = """ + FROM test + | STATS c = count(*) WHERE match(content, "fox"), d = count(*) WHERE match(content, "dog") + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("c", "d")); + assertColumnTypes(resp.columns(), List.of("long", "long")); + assertValues(resp.values(), List.of(List.of(2L, 4L))); + } + + query = """ + FROM test METADATA _score + | WHERE match(content, "fox") + | STATS m = max(_score), n = min(_score) + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("m", "n")); + assertColumnTypes(resp.columns(), List.of("double", "double")); + List> valuesList = getValuesList(resp.values()); + assertEquals(1, valuesList.size()); + assertThat((double) valuesList.get(0).get(0), Matchers.greaterThan(1.0)); + assertThat((double) valuesList.get(0).get(1), Matchers.greaterThan(0.0)); + } + } + public void testMatchWithinEval() { var query = """ FROM test @@ -257,7 +295,7 @@ public void testMatchWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[MATCH] function is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[MATCH] function is only supported in WHERE and STATS commands")); } private void createAndPopulateIndex() { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java index 17af92dabb867..8f6ac432fdd99 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java @@ -358,7 +358,7 @@ public void testMatchWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[:] operator is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[:] operator is only supported in WHERE and STATS commands")); } public void testMatchWithNonTextField() { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java index a3d1ac931528c..dcb3cc122c1c7 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java @@ -64,7 +64,7 @@ public void testQueryStringWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[QSTR] function is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[QSTR] function is only supported in WHERE and STATS commands")); } public void testInvalidQueryStringEof() { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java index 4bb4897c9db5f..75bdc701703e2 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java @@ -57,7 +57,7 @@ public void testTermWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[Term] function is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[Term] function is only supported in WHERE and STATS commands")); } public void testMultipleTerm() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index fe9da1a85cff4..103c8a452a71d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -867,7 +867,12 @@ public enum Cap { * see ES|QL: Grok only supports KEYWORD or TEXT values, * found expression [type] type [INTEGER] #127468 */ - KEEP_REGEX_EXTRACT_ATTRIBUTES; + KEEP_REGEX_EXTRACT_ATTRIBUTES, + + /** + * Full text functions in STATS + */ + FULL_TEXT_FUNCTIONS_IN_STATS_WHERE; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 913db82189e69..b1f3bdd937f6c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -209,13 +209,27 @@ private static void checkFullTextQueryFunctions(LogicalPlan plan, Failures failu failures ); checkFullTextFunctionsParents(condition, failures); + } else if (plan instanceof Aggregate agg) { + checkFullTextFunctionsInAggs(agg, failures); } else { plan.forEachExpression(FullTextFunction.class, ftf -> { - failures.add(fail(ftf, "[{}] {} is only supported in WHERE commands", ftf.functionName(), ftf.functionType())); + failures.add(fail(ftf, "[{}] {} is only supported in WHERE and STATS commands", ftf.functionName(), ftf.functionType())); }); } } + private static void checkFullTextFunctionsInAggs(Aggregate agg, Failures failures) { + agg.groupings().forEach(exp -> { + exp.forEachDown(e -> { + if (e instanceof FullTextFunction ftf) { + failures.add( + fail(ftf, "[{}] {} is only supported in WHERE and STATS commands", ftf.functionName(), ftf.functionType()) + ); + } + }); + }); + } + /** * Checks all commands that exist before a specific type satisfy conditions. * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java index 4a48810c041fb..f4d396ab1736f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java @@ -21,12 +21,15 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; @@ -262,7 +265,22 @@ public void postAnalysisVerification(Failures failures) { ); } checkCategorizeGrouping(failures); + checkMultipleScoreAggregations(failures); + } + private void checkMultipleScoreAggregations(Failures failures) { + Holder hasScoringAggs = new Holder<>(); + forEachExpression(FilteredExpression.class, fe -> { + if (fe.delegate() instanceof AggregateFunction aggregateFunction) { + if (aggregateFunction.field() instanceof MetadataAttribute metadataAttribute) { + if (MetadataAttribute.SCORE.equals(metadataAttribute.name())) { + if (fe.filter().anyMatch(e -> e instanceof FullTextFunction)) { + failures.add(fail(fe, "cannot use _score aggregations with a WHERE filter in a STATS command")); + } + } + } + } + }); } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java index 8fb51457b6a8a..c4e372c62d6d6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java @@ -95,7 +95,8 @@ public final PhysicalOperation groupingPhysicalOperation( aggregatorMode, sourceLayout, false, // non-grouping - s -> aggregatorFactories.add(s.supplier.aggregatorFactory(s.mode, s.channels)) + s -> aggregatorFactories.add(s.supplier.aggregatorFactory(s.mode, s.channels)), + context ); if (aggregatorFactories.isEmpty() == false) { @@ -169,7 +170,8 @@ else if (aggregatorMode.isOutputPartial()) { aggregatorMode, sourceLayout, true, // grouping - s -> aggregatorFactories.add(s.supplier.groupingAggregatorFactory(s.mode, s.channels)) + s -> aggregatorFactories.add(s.supplier.groupingAggregatorFactory(s.mode, s.channels)), + context ); if (groupSpecs.size() == 1 && groupSpecs.get(0).channel == null) { @@ -259,7 +261,8 @@ private void aggregatesToFactory( AggregatorMode mode, Layout layout, boolean grouping, - Consumer consumer + Consumer consumer, + LocalExecutionPlannerContext context ) { // extract filtering channels - and wrap the aggregation with the new evaluator expression only during the init phase for (NamedExpression ne : aggregates) { @@ -319,7 +322,8 @@ else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) { EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator( foldContext, aggregateFunction.filter(), - layout + layout, + context.shardContexts() ); aggSupplier = new FilteredAggregatorFunctionSupplier(aggSupplier, evalFactory); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 0e0b7907d83f7..b1013e2ccfe31 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -182,7 +182,8 @@ public LocalExecutionPlan plan(String taskDescription, FoldContext foldCtx, Phys bigArrays, blockFactory, foldCtx, - settings + settings, + shardContexts ); // workaround for https://github.com/elastic/elasticsearch/issues/99782 @@ -846,7 +847,8 @@ public record LocalExecutionPlannerContext( BigArrays bigArrays, BlockFactory blockFactory, FoldContext foldCtx, - Settings settings + Settings settings, + List shardContexts ) { void addDriverFactory(DriverFactory driverFactory) { driverFactories.add(driverFactory); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index d71040ca1aed6..b50fdb199d660 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1210,9 +1210,8 @@ public void testWeightedAvg() { } public void testMatchInsideEval() throws Exception { - assumeTrue("Match operator is available just for snapshots", Build.current().isSnapshot()); assertEquals( - "1:36: [:] operator is only supported in WHERE commands\n" + "1:36: [:] operator is only supported in WHERE and STATS commands\n" + "line 1:36: [:] operator cannot operate on [title], which is not a field from an index mapping", error("row title = \"brown fox\" | eval x = title:\"fox\" ") ); @@ -1372,12 +1371,12 @@ public void testKqlFunctionsNotAllowedAfterCommands() throws Exception { } public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception { - assertEquals("1:9: [QSTR] function is only supported in WHERE commands", error("row a = qstr(\"Anna\")")); + assertEquals("1:9: [QSTR] function is only supported in WHERE and STATS commands", error("row a = qstr(\"Anna\")")); checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")", "function"); } public void testKqlFunctionOnlyAllowedInWhere() throws Exception { - assertEquals("1:9: [KQL] function is only supported in WHERE commands", error("row a = kql(\"Anna\")")); + assertEquals("1:9: [KQL] function is only supported in WHERE and STATS commands", error("row a = kql(\"Anna\")")); checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function"); } @@ -1397,23 +1396,15 @@ public void testMatchOperatornOnlyAllowedInWhere() throws Exception { private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation, String functionType) throws Exception { assertEquals( - "1:22: [" + functionName + "] " + functionType + " is only supported in WHERE commands", + "1:22: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", error("from test | eval y = " + functionInvocation) ); assertEquals( - "1:18: [" + functionName + "] " + functionType + " is only supported in WHERE commands", + "1:18: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", error("from test | sort " + functionInvocation + " asc") ); assertEquals( - "1:23: [" + functionName + "] " + functionType + " is only supported in WHERE commands", - error("from test | STATS c = " + functionInvocation + " BY first_name") - ); - assertEquals( - "1:50: [" + functionName + "] " + functionType + " is only supported in WHERE commands", - error("from test | stats max_salary = max(salary) where " + functionInvocation) - ); - assertEquals( - "1:47: [" + functionName + "] " + functionType + " is only supported in WHERE commands", + "1:47: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", error("from test | stats max_salary = max(salary) by " + functionInvocation) ); } @@ -2326,6 +2317,26 @@ public void testQueryStringOptions() { ); } + public void testFullTextFunctionsInStats() { + checkFullTextFunctionsInStats("match(last_name, \"Smith\")"); + checkFullTextFunctionsInStats("last_name : \"Smith\""); + checkFullTextFunctionsInStats("qstr(\"last_name: Smith\")"); + checkFullTextFunctionsInStats("kql(\"last_name: Smith\")"); + } + + private void checkFullTextFunctionsInStats(String functionInvocation) { + + query("from test | stats c = max(salary) where " + functionInvocation); + query("from test | stats c = max(salary) where " + functionInvocation + " or length(first_name) > 10"); + query("from test metadata _score | where " + functionInvocation + " | stats c = max(_score)"); + query("from test metadata _score | where " + functionInvocation + " or length(first_name) > 10 | stats c = max(_score)"); + + assertThat( + error("from test metadata _score | stats c = max(_score) where " + functionInvocation), + containsString("cannot use _score aggregations with a WHERE filter in a STATS command") + ); + } + private void query(String query) { query(query, defaultAnalyzer); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 5e19c2e32e6b6..ed3541ca064db 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -39,12 +39,14 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; @@ -1857,6 +1859,64 @@ public void testConstantKeywordDissectFilter() { assertNull(query.query()); } + public void testMatchFunctionWithStatsWherePushable() { + String query = """ + from test + | stats c = count(*) where match(last_name, "Smith") + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var exchange = as(agg.child(), ExchangeExec.class); + var stats = as(exchange.child(), EsStatsQueryExec.class); + QueryBuilder expected = new MatchQueryBuilder("last_name", "Smith").lenient(true); + assertThat(stats.query().toString(), equalTo(expected.toString())); + } + + public void testMatchFunctionWithStatsPushableAndNonPushableCondition() { + String query = """ + from test + | where length(first_name) > 10 + | stats c = count(*) where match(last_name, "Smith") + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var exchange = as(agg.child(), ExchangeExec.class); + var aggExec = as(exchange.child(), AggregateExec.class); + var filter = as(aggExec.child(), FilterExec.class); + assertTrue(filter.condition() instanceof GreaterThan); + var fieldExtract = as(filter.child(), FieldExtractExec.class); + var esQuery = as(fieldExtract.child(), EsQueryExec.class); + QueryBuilder expected = new MatchQueryBuilder("last_name", "Smith").lenient(true); + assertThat(esQuery.query().toString(), equalTo(expected.toString())); + } + + public void testMatchFunctionStatisWithNonPushableCondition() { + String query = """ + from test + | stats c = count(*) where match(last_name, "Smith"), d = count(*) where match(first_name, "Anna") + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var aggregates = agg.aggregates(); + assertThat(aggregates.size(), is(2)); + for (NamedExpression aggregate : aggregates) { + var alias = as(aggregate, Alias.class); + var count = as(alias.child(), Count.class); + var match = as(count.filter(), Match.class); + } + var exchange = as(agg.child(), ExchangeExec.class); + var aggExec = as(exchange.child(), AggregateExec.class); + var fieldExtract = as(aggExec.child(), FieldExtractExec.class); + var esQuery = as(fieldExtract.child(), EsQueryExec.class); + assertNull(esQuery.query()); + } + private QueryBuilder wrapWithSingleQuery(String query, QueryBuilder inner, String fieldName, Source source) { return FilterTests.singleValueQuery(query, inner, fieldName, source); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml index 96a9d1f925d7e..5c326bcbb2f4a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml @@ -125,7 +125,7 @@ setup: - match: { status: 400 } - match: { error.type: verification_exception } - - match: { error.reason: "Found 1 problem\nline 1:19: Unknown column [something]" } + - match: { error.reason: "/Unknown.column.\\[something\\]/" } --- "match on eval column": @@ -139,7 +139,7 @@ setup: - match: { status: 400 } - match: { error.type: verification_exception } - - match: { error.reason: "Found 1 problem\nline 1:60: [:] operator cannot operate on [upper_content], which is not a field from an index mapping" } + - match: { error.reason: "/operator.cannot.operate.on.\\[upper_content\\],.which.is.not.a.field.from.an.index.mapping/" } --- "match on overwritten column": @@ -153,7 +153,7 @@ setup: - match: { status: 400 } - match: { error.type: verification_exception } - - match: { error.reason: "Found 1 problem\nline 1:78: [:] operator cannot operate on [content], which is not a field from an index mapping" } + - match: { error.reason: "/operator.cannot.operate.on.\\[content\\],.which.is.not.a.field.from.an.index.mapping/" } --- "match after stats": @@ -167,7 +167,7 @@ setup: - match: { status: 400 } - match: { error.type: verification_exception } - - match: { error.reason: "Found 1 problem\nline 1:36: Unknown column [content], did you mean [count(*)]?" } + - match: { error.reason: "/Unknown.column.\\[content\\]/" } --- "match with disjunctions": @@ -199,4 +199,4 @@ setup: - match: { status: 400 } - match: { error.type: verification_exception } - - match: { error.reason: "Found 1 problem\nline 1:34: [:] operator is only supported in WHERE commands" } + - match: { error.reason: "/.*operator.is.only.supported.in.WHERE.*/" }