[8.19] ES|QL - Allow full text functions to be used in STATS ... WHERE (#125479)

docs/changelog/125479.yaml

@@ -0,0 +1,6 @@
+pr: 125479
+summary: ES|QL - Allow full text functions to be used in STATS
+area: ES|QL
+type: enhancement
+issues:
+ - 125481

x-pack/plugin/build.gradle

@@ -211,6 +211,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
   task.skipTest("esql/61_enrich_ip/Invalid IP strings", "We switched from exceptions to null+warnings for ENRICH runtime errors")
   task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields")
   task.skipTest("esql/180_match_operator/match with functions", "Error message changed")
+  task.skipTest("esql/180_match_operator/match within eval", "Error message changed")
   task.skipTest("esql/40_unsupported_types/semantic_text declared in mapping", "The semantic text field format changed")
   task.skipTest("esql/190_lookup_join/Alias as lookup index", "LOOKUP JOIN does not support index aliases for now")
   task.skipTest("esql/190_lookup_join/alias-repeated-alias", "LOOKUP JOIN does not support index aliases for now")

x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec

@@ -191,3 +191,82 @@ book_no:keyword
 7140
 2714
 ;
+
+testKqlInStatsNonPushable
+required_capability: kql_function
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| where length(title) > 40 
+| stats c = count(*) where kql("title:Lord")
+;
+
+c:long
+3
+;
+
+
+testMatchInStatsPushableAndNonPushable
+required_capability: kql_function
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| stats c = count(*) where (kql("title: lord") and ratings > 4.5) or (kql("author: dostoevsky") and length(title) > 50)
+;
+
+c:long
+6
+;
+
+testKqlInStatsPushable
+required_capability: kql_function
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| stats c = count(*) where kql("author:tolkien")
+;
+
+c:long
+22
+;
+
+testKqlInStatsWithNonPushableDisjunctions
+required_capability: kql_function
+required_capability: full_text_functions_in_stats_where
+FROM books
+| STATS c = count(*) where kql("title: lord") or length(title) > 130 
+;
+
+c:long
+5
+;
+
+testKqlInStatsWithMultipleAggs
+required_capability: kql_function
+required_capability: full_text_functions_in_stats_where
+FROM books
+| STATS c = count(*) where kql("title: lord"), m = max(book_no::integer) where kql("author: tolkien"), n = min(book_no::integer) where kql("author: dostoevsky") 
+;
+
+c:long | m:integer | n:integer
+4      | 9607      | 1211
+;
+
+
+testKqlInStatsWithGrouping
+required_capability: kql_function
+required_capability: full_text_functions_in_stats_where
+FROM books
+| STATS r = AVG(ratings) where kql("title: Lord AND Rings") by author | WHERE r is not null
+;
+ignoreOrder: true
+
+r:double           | author: text
+4.75               | Alan Lee                 
+4.674999952316284  | J. R. R. Tolkien         
+4.670000076293945  | John Ronald Reuel Tolkien
+4.670000076293945  | Agnes Perkins            
+4.670000076293945  | Charles Adolph Huttar    
+4.670000076293945  | Walter Scheps            
+4.559999942779541  | J.R.R. Tolkien           
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec

@@ -755,3 +755,94 @@ book_no:keyword
 7140
 2714
 ;
+
+testMatchInStatsNonPushable
+required_capability: match_function
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| where length(title) > 40 
+| stats c = count(*) where match(title, "Lord")
+;
+
+c:long
+3
+;
+
+testMatchInStatsPushableAndNonPushable
+required_capability: match_function
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| stats c = count(*) where (match(title, "lord") and ratings > 4.5) or (match(author, "dostoevsky") and length(title) > 50)
+;
+
+c:long
+6
+;
+
+testMatchInStatsPushable
+required_capability: match_function
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| stats c = count(*) where match(author, "tolkien")
+;
+
+c:long
+22
+;
+
+testMatchInStatsWithOptions
+required_capability: match_function
+required_capability: full_text_functions_in_stats_where
+
+FROM books
+| STATS c = count(*) where match(title, "Hobbit Back Again", {"operator": "AND"})
+;
+
+c:long
+1
+;
+
+testMatchInStatsWithNonPushableDisjunctions
+required_capability: match_function
+required_capability: full_text_functions_in_stats_where
+
+FROM books
+| STATS c = count(*) where match(title, "lord") or length(title) > 130 
+;
+
+c:long
+5
+;
+
+testMatchInStatsWithMultipleAggs
+required_capability: match_function
+required_capability: full_text_functions_in_stats_where
+FROM books
+| STATS c = count(*) where match(title, "lord"), m = max(book_no::integer) where match(author, "tolkien"), n = min(book_no::integer) where match(author, "dostoevsky") 
+;
+
+c:long | m:integer | n:integer
+4      | 9607      | 1211
+;
+
+
+testMatchInStatsWithGrouping
+required_capability: match_function
+required_capability: full_text_functions_in_stats_where
+FROM books
+| STATS r = AVG(ratings) where match(title, "Lord Rings", {"operator": "AND"}) by author | WHERE r is not null
+;
+ignoreOrder: true
+
+r:double           | author: text
+4.75               | Alan Lee                 
+4.674999952316284  | J. R. R. Tolkien         
+4.670000076293945  | John Ronald Reuel Tolkien
+4.670000076293945  | Agnes Perkins            
+4.670000076293945  | Charles Adolph Huttar    
+4.670000076293945  | Walter Scheps            
+4.559999942779541  | J.R.R. Tolkien           
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec

@@ -757,3 +757,29 @@ from semantic_text
 host:keyword | semantic_text_field:text | language_name:keyword | language_code:integer
 "host1"      | live long and prosper    | English               | 1
 ;
+
+
+testMatchInStatsNonPushable
+required_capability: match_operator_colon
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| where length(title) > 40 
+| stats c = count(*) where title:"Lord"
+;
+
+c:long
+3
+;
+
+testMatchInStatsPushable
+required_capability: match_operator_colon
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| stats c = count(*) where author:"tolkien"
+;
+
+c:long
+22
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec

@@ -211,3 +211,92 @@ book_no:keyword | title:text
 7480            | The Hobbit                                     
 // end::qstr-with-options-result[]
 ;
+
+testQstrInStatsNonPushable
+required_capability: qstr_function
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| where length(title) > 40 
+| stats c = count(*) where qstr("title:Lord")
+;
+
+c:long
+3
+;
+
+testMatchInStatsPushableAndNonPushable
+required_capability: qstr_function
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| stats c = count(*) where (qstr("title: lord") and ratings > 4.5) or (qstr("author: dostoevsky") and length(title) > 50)
+;
+
+c:long
+6
+;
+
+testQstrInStatsPushable
+required_capability: qstr_function
+required_capability: full_text_functions_in_stats_where
+
+from books 
+| stats c = count(*) where qstr("author:tolkien")
+;
+
+c:long
+22
+;
+
+testQstrInStatsWithOptions
+required_capability: qstr_function
+required_capability: full_text_functions_in_stats_where
+
+FROM books
+| STATS c = count(*) where qstr("title: Hobbit Back Again", {"default_operator": "AND"})
+;
+
+c:long
+1
+;
+
+testQstrInStatsWithNonPushableDisjunctions
+required_capability: qstr_function
+required_capability: full_text_functions_in_stats_where
+FROM books
+| STATS c = count(*) where qstr("title: lord") or length(title) > 130 
+;
+
+c:long
+5
+;
+
+testQstrInStatsWithMultipleAggs
+required_capability: qstr_function
+required_capability: full_text_functions_in_stats_where
+FROM books
+| STATS c = count(*) where qstr("title: lord"), m = max(book_no::integer) where qstr("author: tolkien"), n = min(book_no::integer) where qstr("author: dostoevsky") 
+;
+
+c:long | m:integer | n:integer
+4      | 9607      | 1211
+;
+
+testQstrInStatsWithGrouping
+required_capability: qstr_function
+required_capability: full_text_functions_in_stats_where
+FROM books
+| STATS r = AVG(ratings) where qstr("title: Lord Rings", {"default_operator": "AND"}) by author | WHERE r is not null
+;
+ignoreOrder: true
+
+r:double           | author: text
+4.75               | Alan Lee                 
+4.674999952316284  | J. R. R. Tolkien         
+4.670000076293945  | John Ronald Reuel Tolkien
+4.670000076293945  | Agnes Perkins            
+4.670000076293945  | Charles Adolph Huttar    
+4.670000076293945  | Walter Scheps            
+4.559999942779541  | J.R.R. Tolkien           
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec

@@ -454,7 +454,6 @@ book_no:keyword |  _score:double
 8678            | 0.0
 ;
 
-
 disjunctionScoresMultipleClauses
 
 required_capability: metadata_score
@@ -477,3 +476,18 @@ book_no:keyword |  _score:double
 4023            | 1.5062403678894043
 2924            | 1.2732219696044922
 ;
+
+statsScores
+
+required_capability: metadata_score
+required_capability: match_function
+required_capability: full_text_functions_in_stats_where
+
+from books metadata _score 
+| where match(title, "Lord Rings", {"operator": "AND"})
+| stats avg_score = avg(_score), max_score = max(_score), min_score = min(_score)
+;
+
+avg_score:double   | max_score:double   | min_score:double
+3.869828939437866  | 5.123856544494629  | 3.0124807357788086
+;

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java

@@ -66,7 +66,7 @@ public void testKqlQueryWithinEval() {
             """;
 
         var error = expectThrows(VerificationException.class, () -> run(query));
-        assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE commands"));
+        assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE and STATS commands"));
     }
 
     public void testInvalidKqlQueryEof() {