diff --git a/docs/changelog/125479.yaml b/docs/changelog/125479.yaml new file mode 100644 index 0000000000000..efc31441254c5 --- /dev/null +++ b/docs/changelog/125479.yaml @@ -0,0 +1,6 @@ +pr: 125479 +summary: ES|QL - Allow full text functions to be used in STATS +area: ES|QL +type: enhancement +issues: + - 125481 diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 7605c8060f873..f2f77e626c004 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -95,6 +95,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("esql/61_enrich_ip/Invalid IP strings", "We switched from exceptions to null+warnings for ENRICH runtime errors") task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields") task.skipTest("esql/180_match_operator/match with functions", "Error message changed") + task.skipTest("esql/180_match_operator/match within eval", "Error message changed") task.skipTest("esql/40_unsupported_types/semantic_text declared in mapping", "The semantic text field format changed") task.skipTest("esql/190_lookup_join/Alias as lookup index", "LOOKUP JOIN does not support index aliases for now") task.skipTest("esql/190_lookup_join/alias-repeated-alias", "LOOKUP JOIN does not support index aliases for now") diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec index bcb32daa34660..410da34d42cbb 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec @@ -189,3 +189,82 @@ book_no:keyword 7140 2714 ; + +testKqlInStatsNonPushable +required_capability: kql_function +required_capability: full_text_functions_in_stats_where + +from books +| where length(title) > 40 +| stats c = count(*) where kql("title:Lord") +; + +c:long +3 +; + + +testMatchInStatsPushableAndNonPushable +required_capability: kql_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where (kql("title: lord") and ratings > 4.5) or (kql("author: dostoevsky") and length(title) > 50) +; + +c:long +6 +; + +testKqlInStatsPushable +required_capability: kql_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where kql("author:tolkien") +; + +c:long +22 +; + +testKqlInStatsWithNonPushableDisjunctions +required_capability: kql_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where kql("title: lord") or length(title) > 130 +; + +c:long +5 +; + +testKqlInStatsWithMultipleAggs +required_capability: kql_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where kql("title: lord"), m = max(book_no::integer) where kql("author: tolkien"), n = min(book_no::integer) where kql("author: dostoevsky") +; + +c:long | m:integer | n:integer +4 | 9607 | 1211 +; + + +testKqlInStatsWithGrouping +required_capability: kql_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS r = AVG(ratings) where kql("title: Lord AND Rings") by author | WHERE r is not null +; +ignoreOrder: true + +r:double | author: text +4.75 | Alan Lee +4.674999952316284 | J. R. R. Tolkien +4.670000076293945 | John Ronald Reuel Tolkien +4.670000076293945 | Agnes Perkins +4.670000076293945 | Charles Adolph Huttar +4.670000076293945 | Walter Scheps +4.559999942779541 | J.R.R. Tolkien +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec index b04f90e986a7e..04532d2a9693d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec @@ -750,3 +750,94 @@ book_no:keyword 7140 2714 ; + +testMatchInStatsNonPushable +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +from books +| where length(title) > 40 +| stats c = count(*) where match(title, "Lord") +; + +c:long +3 +; + +testMatchInStatsPushableAndNonPushable +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where (match(title, "lord") and ratings > 4.5) or (match(author, "dostoevsky") and length(title) > 50) +; + +c:long +6 +; + +testMatchInStatsPushable +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where match(author, "tolkien") +; + +c:long +22 +; + +testMatchInStatsWithOptions +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +FROM books +| STATS c = count(*) where match(title, "Hobbit Back Again", {"operator": "AND"}) +; + +c:long +1 +; + +testMatchInStatsWithNonPushableDisjunctions +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +FROM books +| STATS c = count(*) where match(title, "lord") or length(title) > 130 +; + +c:long +5 +; + +testMatchInStatsWithMultipleAggs +required_capability: match_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where match(title, "lord"), m = max(book_no::integer) where match(author, "tolkien"), n = min(book_no::integer) where match(author, "dostoevsky") +; + +c:long | m:integer | n:integer +4 | 9607 | 1211 +; + + +testMatchInStatsWithGrouping +required_capability: match_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS r = AVG(ratings) where match(title, "Lord Rings", {"operator": "AND"}) by author | WHERE r is not null +; +ignoreOrder: true + +r:double | author: text +4.75 | Alan Lee +4.674999952316284 | J. R. R. Tolkien +4.670000076293945 | John Ronald Reuel Tolkien +4.670000076293945 | Agnes Perkins +4.670000076293945 | Charles Adolph Huttar +4.670000076293945 | Walter Scheps +4.559999942779541 | J.R.R. Tolkien +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec index 1a56c79066f74..d93c6b3884d22 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec @@ -751,3 +751,29 @@ from semantic_text host:keyword | semantic_text_field:text | language_name:keyword | language_code:integer "host1" | live long and prosper | English | 1 ; + + +testMatchInStatsNonPushable +required_capability: match_operator_colon +required_capability: full_text_functions_in_stats_where + +from books +| where length(title) > 40 +| stats c = count(*) where title:"Lord" +; + +c:long +3 +; + +testMatchInStatsPushable +required_capability: match_operator_colon +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where author:"tolkien" +; + +c:long +22 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec index 1bce0d7bfcbc6..9819662a8af13 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec @@ -210,3 +210,92 @@ book_no:keyword | title:text 7480 | The Hobbit // end::qstr-with-options-result[] ; + +testQstrInStatsNonPushable +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where + +from books +| where length(title) > 40 +| stats c = count(*) where qstr("title:Lord") +; + +c:long +3 +; + +testMatchInStatsPushableAndNonPushable +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where (qstr("title: lord") and ratings > 4.5) or (qstr("author: dostoevsky") and length(title) > 50) +; + +c:long +6 +; + +testQstrInStatsPushable +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where + +from books +| stats c = count(*) where qstr("author:tolkien") +; + +c:long +22 +; + +testQstrInStatsWithOptions +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where + +FROM books +| STATS c = count(*) where qstr("title: Hobbit Back Again", {"default_operator": "AND"}) +; + +c:long +1 +; + +testQstrInStatsWithNonPushableDisjunctions +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where qstr("title: lord") or length(title) > 130 +; + +c:long +5 +; + +testQstrInStatsWithMultipleAggs +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS c = count(*) where qstr("title: lord"), m = max(book_no::integer) where qstr("author: tolkien"), n = min(book_no::integer) where qstr("author: dostoevsky") +; + +c:long | m:integer | n:integer +4 | 9607 | 1211 +; + +testQstrInStatsWithGrouping +required_capability: qstr_function +required_capability: full_text_functions_in_stats_where +FROM books +| STATS r = AVG(ratings) where qstr("title: Lord Rings", {"default_operator": "AND"}) by author | WHERE r is not null +; +ignoreOrder: true + +r:double | author: text +4.75 | Alan Lee +4.674999952316284 | J. R. R. Tolkien +4.670000076293945 | John Ronald Reuel Tolkien +4.670000076293945 | Agnes Perkins +4.670000076293945 | Charles Adolph Huttar +4.670000076293945 | Walter Scheps +4.559999942779541 | J.R.R. Tolkien +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec index f6f3352b4e90c..401d946334c45 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec @@ -521,7 +521,6 @@ book_no:keyword | _score:double 8678 | 0.0 ; - disjunctionScoresMultipleClauses required_capability: metadata_score @@ -544,3 +543,18 @@ book_no:keyword | _score:double 4023 | 1.5062403678894043 2924 | 1.2732219696044922 ; + +statsScores + +required_capability: metadata_score +required_capability: match_function +required_capability: full_text_functions_in_stats_where + +from books metadata _score +| where match(title, "Lord Rings", {"operator": "AND"}) +| stats avg_score = avg(_score), max_score = max(_score), min_score = min(_score) +; + +avg_score:double | max_score:double | min_score:double +3.869828939437866 | 5.123856544494629 | 3.0124807357788086 +; diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java index 39c963e938ac5..8e0a5ed2e8ce9 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java @@ -66,7 +66,7 @@ public void testKqlQueryWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE and STATS commands")); } public void testInvalidKqlQueryEof() { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java index 4ec309ff05cee..23958fcd35f30 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java @@ -13,11 +13,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; +import org.hamcrest.Matchers; import org.junit.Before; import java.util.List; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.getValuesList; import static org.hamcrest.CoreMatchers.containsString; //@TestLogging(value = "org.elasticsearch.xpack.esql:TRACE,org.elasticsearch.compute:TRACE", reason = "debug") @@ -246,6 +248,42 @@ public void testWhereMatchWithRow() { ); } + public void testMatchWithStats() { + var errorQuery = """ + FROM test + | STATS c = count(*) BY match(content, "fox") + """; + + var error = expectThrows(ElasticsearchException.class, () -> run(errorQuery)); + assertThat(error.getMessage(), containsString("[MATCH] function is only supported in WHERE and STATS commands")); + + var query = """ + FROM test + | STATS c = count(*) WHERE match(content, "fox"), d = count(*) WHERE match(content, "dog") + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("c", "d")); + assertColumnTypes(resp.columns(), List.of("long", "long")); + assertValues(resp.values(), List.of(List.of(2L, 4L))); + } + + query = """ + FROM test METADATA _score + | WHERE match(content, "fox") + | STATS m = max(_score), n = min(_score) + """; + + try (var resp = run(query)) { + assertColumnNames(resp.columns(), List.of("m", "n")); + assertColumnTypes(resp.columns(), List.of("double", "double")); + List> valuesList = getValuesList(resp.values()); + assertEquals(1, valuesList.size()); + assertThat((double) valuesList.get(0).get(0), Matchers.greaterThan(1.0)); + assertThat((double) valuesList.get(0).get(1), Matchers.greaterThan(0.0)); + } + } + public void testMatchWithinEval() { var query = """ FROM test @@ -253,7 +291,7 @@ public void testMatchWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[MATCH] function is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[MATCH] function is only supported in WHERE and STATS commands")); } private void createAndPopulateIndex() { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java index 392b17909ff79..28df30e6d0841 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchOperatorIT.java @@ -354,7 +354,7 @@ public void testMatchWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[:] operator is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[:] operator is only supported in WHERE and STATS commands")); } public void testMatchWithNonTextField() { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java index 97a98da6e8291..aba0a5f4a5b97 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/QueryStringIT.java @@ -63,7 +63,7 @@ public void testQueryStringWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[QSTR] function is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[QSTR] function is only supported in WHERE and STATS commands")); } public void testInvalidQueryStringEof() { diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java index 4bb4897c9db5f..75bdc701703e2 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/TermIT.java @@ -57,7 +57,7 @@ public void testTermWithinEval() { """; var error = expectThrows(VerificationException.class, () -> run(query)); - assertThat(error.getMessage(), containsString("[Term] function is only supported in WHERE commands")); + assertThat(error.getMessage(), containsString("[Term] function is only supported in WHERE and STATS commands")); } public void testMultipleTerm() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 07990a72e99cc..defa70737cef8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -1074,7 +1074,12 @@ public enum Cap { /** * Support for the SAMPLE aggregation function */ - AGG_SAMPLE; + AGG_SAMPLE, + + /** + * Full text functions in STATS + */ + FULL_TEXT_FUNCTIONS_IN_STATS_WHERE; private final boolean enabled; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java index 8fdb305804c04..3abdebf63c1fc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/FullTextFunction.java @@ -226,13 +226,27 @@ private static void checkFullTextQueryFunctions(LogicalPlan plan, Failures failu failures ); checkFullTextFunctionsParents(condition, failures); + } else if (plan instanceof Aggregate agg) { + checkFullTextFunctionsInAggs(agg, failures); } else { plan.forEachExpression(FullTextFunction.class, ftf -> { - failures.add(fail(ftf, "[{}] {} is only supported in WHERE commands", ftf.functionName(), ftf.functionType())); + failures.add(fail(ftf, "[{}] {} is only supported in WHERE and STATS commands", ftf.functionName(), ftf.functionType())); }); } } + private static void checkFullTextFunctionsInAggs(Aggregate agg, Failures failures) { + agg.groupings().forEach(exp -> { + exp.forEachDown(e -> { + if (e instanceof FullTextFunction ftf) { + failures.add( + fail(ftf, "[{}] {} is only supported in WHERE and STATS commands", ftf.functionName(), ftf.functionType()) + ); + } + }); + }); + } + /** * Checks all commands that exist before a specific type satisfy conditions. * diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java index 593a31228fc9a..78503785ea7f2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Aggregate.java @@ -22,13 +22,16 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; +import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateFunction; import org.elasticsearch.xpack.esql.expression.function.aggregate.FilteredExpression; import org.elasticsearch.xpack.esql.expression.function.aggregate.Rate; import org.elasticsearch.xpack.esql.expression.function.aggregate.TimeSeriesAggregateFunction; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.Categorize; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; @@ -229,7 +232,22 @@ public void postAnalysisVerification(Failures failures) { ); } checkCategorizeGrouping(failures); + checkMultipleScoreAggregations(failures); + } + private void checkMultipleScoreAggregations(Failures failures) { + Holder hasScoringAggs = new Holder<>(); + forEachExpression(FilteredExpression.class, fe -> { + if (fe.delegate() instanceof AggregateFunction aggregateFunction) { + if (aggregateFunction.field() instanceof MetadataAttribute metadataAttribute) { + if (MetadataAttribute.SCORE.equals(metadataAttribute.name())) { + if (fe.filter().anyMatch(e -> e instanceof FullTextFunction)) { + failures.add(fail(fe, "cannot use _score aggregations with a WHERE filter in a STATS command")); + } + } + } + } + }); } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java index 93c8b54d81eb4..7567f5be736a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/AbstractPhysicalOperationProviders.java @@ -96,7 +96,8 @@ public final PhysicalOperation groupingPhysicalOperation( aggregatorMode, sourceLayout, false, // non-grouping - s -> aggregatorFactories.add(s.supplier.aggregatorFactory(s.mode, s.channels)) + s -> aggregatorFactories.add(s.supplier.aggregatorFactory(s.mode, s.channels)), + context ); if (aggregatorFactories.isEmpty() == false) { @@ -170,7 +171,8 @@ else if (aggregatorMode.isOutputPartial()) { aggregatorMode, sourceLayout, true, // grouping - s -> aggregatorFactories.add(s.supplier.groupingAggregatorFactory(s.mode, s.channels)) + s -> aggregatorFactories.add(s.supplier.groupingAggregatorFactory(s.mode, s.channels)), + context ); // time-series aggregation if (aggregateExec instanceof TimeSeriesAggregateExec ts) { @@ -269,7 +271,8 @@ private void aggregatesToFactory( AggregatorMode mode, Layout layout, boolean grouping, - Consumer consumer + Consumer consumer, + LocalExecutionPlannerContext context ) { // extract filtering channels - and wrap the aggregation with the new evaluator expression only during the init phase for (NamedExpression ne : aggregates) { @@ -329,7 +332,8 @@ else if (mode == AggregatorMode.FINAL || mode == AggregatorMode.INTERMEDIATE) { EvalOperator.ExpressionEvaluator.Factory evalFactory = EvalMapper.toEvaluator( foldContext, aggregateFunction.filter(), - layout + layout, + context.shardContexts() ); aggSupplier = new FilteredAggregatorFunctionSupplier(aggSupplier, evalFactory); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index abcd0ec1318ed..c89327e6a6aa4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -201,7 +201,8 @@ public LocalExecutionPlan plan(String description, FoldContext foldCtx, Physical bigArrays, blockFactory, foldCtx, - settings + settings, + shardContexts ); // workaround for https://github.com/elastic/elasticsearch/issues/99782 @@ -928,7 +929,8 @@ public record LocalExecutionPlannerContext( BigArrays bigArrays, BlockFactory blockFactory, FoldContext foldCtx, - Settings settings + Settings settings, + List shardContexts ) { void addDriverFactory(DriverFactory driverFactory) { driverFactories.add(driverFactory); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index e75cd5ea05a6c..d76a355a6c9a9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1208,9 +1208,8 @@ public void testWeightedAvg() { } public void testMatchInsideEval() throws Exception { - assumeTrue("Match operator is available just for snapshots", Build.current().isSnapshot()); assertEquals( - "1:36: [:] operator is only supported in WHERE commands\n" + "1:36: [:] operator is only supported in WHERE and STATS commands\n" + "line 1:36: [:] operator cannot operate on [title], which is not a field from an index mapping", error("row title = \"brown fox\" | eval x = title:\"fox\" ") ); @@ -1370,12 +1369,12 @@ public void testKqlFunctionsNotAllowedAfterCommands() throws Exception { } public void testQueryStringFunctionOnlyAllowedInWhere() throws Exception { - assertEquals("1:9: [QSTR] function is only supported in WHERE commands", error("row a = qstr(\"Anna\")")); + assertEquals("1:9: [QSTR] function is only supported in WHERE and STATS commands", error("row a = qstr(\"Anna\")")); checkFullTextFunctionsOnlyAllowedInWhere("QSTR", "qstr(\"Anna\")", "function"); } public void testKqlFunctionOnlyAllowedInWhere() throws Exception { - assertEquals("1:9: [KQL] function is only supported in WHERE commands", error("row a = kql(\"Anna\")")); + assertEquals("1:9: [KQL] function is only supported in WHERE and STATS commands", error("row a = kql(\"Anna\")")); checkFullTextFunctionsOnlyAllowedInWhere("KQL", "kql(\"Anna\")", "function"); } @@ -1395,23 +1394,15 @@ public void testMatchOperatornOnlyAllowedInWhere() throws Exception { private void checkFullTextFunctionsOnlyAllowedInWhere(String functionName, String functionInvocation, String functionType) throws Exception { assertEquals( - "1:22: [" + functionName + "] " + functionType + " is only supported in WHERE commands", + "1:22: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", error("from test | eval y = " + functionInvocation) ); assertEquals( - "1:18: [" + functionName + "] " + functionType + " is only supported in WHERE commands", + "1:18: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", error("from test | sort " + functionInvocation + " asc") ); assertEquals( - "1:23: [" + functionName + "] " + functionType + " is only supported in WHERE commands", - error("from test | STATS c = " + functionInvocation + " BY first_name") - ); - assertEquals( - "1:50: [" + functionName + "] " + functionType + " is only supported in WHERE commands", - error("from test | stats max_salary = max(salary) where " + functionInvocation) - ); - assertEquals( - "1:47: [" + functionName + "] " + functionType + " is only supported in WHERE commands", + "1:47: [" + functionName + "] " + functionType + " is only supported in WHERE and STATS commands", error("from test | stats max_salary = max(salary) by " + functionInvocation) ); } @@ -2487,7 +2478,7 @@ public void testMultiMatchTargetsExistingField() throws Exception { public void testMultiMatchInsideEval() throws Exception { assumeTrue("MultiMatch operator is available just for snapshots", Build.current().isSnapshot()); assertEquals( - "1:36: [MultiMatch] function is only supported in WHERE commands\n" + "1:36: [MultiMatch] function is only supported in WHERE and STATS commands\n" + "line 1:55: [MultiMatch] function cannot operate on [title], which is not a field from an index mapping", error("row title = \"brown fox\" | eval x = multi_match(\"fox\", title)") ); @@ -2502,6 +2493,27 @@ public void testInsistNotOnTopOfFrom() { ); } + public void testFullTextFunctionsInStats() { + checkFullTextFunctionsInStats("match(last_name, \"Smith\")"); + checkFullTextFunctionsInStats("multi_match(\"Smith\", first_name, last_name)"); + checkFullTextFunctionsInStats("last_name : \"Smith\""); + checkFullTextFunctionsInStats("qstr(\"last_name: Smith\")"); + checkFullTextFunctionsInStats("kql(\"last_name: Smith\")"); + } + + private void checkFullTextFunctionsInStats(String functionInvocation) { + + query("from test | stats c = max(salary) where " + functionInvocation); + query("from test | stats c = max(salary) where " + functionInvocation + " or length(first_name) > 10"); + query("from test metadata _score | where " + functionInvocation + " | stats c = max(_score)"); + query("from test metadata _score | where " + functionInvocation + " or length(first_name) > 10 | stats c = max(_score)"); + + assertThat( + error("from test metadata _score | stats c = max(_score) where " + functionInvocation), + containsString("cannot use _score aggregations with a WHERE filter in a STATS command") + ); + } + private void query(String query) { query(query, defaultAnalyzer); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index e540649e8c602..0cc8c670895e9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -42,6 +42,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; @@ -49,6 +50,7 @@ import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; @@ -1856,6 +1858,64 @@ public void testPushDownFieldExtractToTimeSeriesSource() { assertTrue(timeSeriesSource.attrs().stream().noneMatch(EsQueryExec::isSourceAttribute)); } + public void testMatchFunctionWithStatsWherePushable() { + String query = """ + from test + | stats c = count(*) where match(last_name, "Smith") + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var exchange = as(agg.child(), ExchangeExec.class); + var stats = as(exchange.child(), EsStatsQueryExec.class); + QueryBuilder expected = new MatchQueryBuilder("last_name", "Smith").lenient(true); + assertThat(stats.query().toString(), equalTo(expected.toString())); + } + + public void testMatchFunctionWithStatsPushableAndNonPushableCondition() { + String query = """ + from test + | where length(first_name) > 10 + | stats c = count(*) where match(last_name, "Smith") + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var exchange = as(agg.child(), ExchangeExec.class); + var aggExec = as(exchange.child(), AggregateExec.class); + var filter = as(aggExec.child(), FilterExec.class); + assertTrue(filter.condition() instanceof GreaterThan); + var fieldExtract = as(filter.child(), FieldExtractExec.class); + var esQuery = as(fieldExtract.child(), EsQueryExec.class); + QueryBuilder expected = new MatchQueryBuilder("last_name", "Smith").lenient(true); + assertThat(esQuery.query().toString(), equalTo(expected.toString())); + } + + public void testMatchFunctionStatisWithNonPushableCondition() { + String query = """ + from test + | stats c = count(*) where match(last_name, "Smith"), d = count(*) where match(first_name, "Anna") + """; + var plan = plannerOptimizer.plan(query); + + var limit = as(plan, LimitExec.class); + var agg = as(limit.child(), AggregateExec.class); + var aggregates = agg.aggregates(); + assertThat(aggregates.size(), is(2)); + for (NamedExpression aggregate : aggregates) { + var alias = as(aggregate, Alias.class); + var count = as(alias.child(), Count.class); + var match = as(count.filter(), Match.class); + } + var exchange = as(agg.child(), ExchangeExec.class); + var aggExec = as(exchange.child(), AggregateExec.class); + var fieldExtract = as(aggExec.child(), FieldExtractExec.class); + var esQuery = as(fieldExtract.child(), EsQueryExec.class); + assertNull(esQuery.query()); + } + private QueryBuilder wrapWithSingleQuery(String query, QueryBuilder inner, String fieldName, Source source) { return FilterTests.singleValueQuery(query, inner, fieldName, source); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml index e2b870aca9f1e..5c326bcbb2f4a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/180_match_operator.yml @@ -199,4 +199,4 @@ setup: - match: { status: 400 } - match: { error.type: verification_exception } - - match: { error.reason: "/operator.is.only.supported.in.WHERE.commands/" } + - match: { error.reason: "/.*operator.is.only.supported.in.WHERE.*/" }