ESQL - Allow full text functions disjunctions for non-full text functions (#120291)

Author: carlosdelest

docs/changelog/120291.yaml

@@ -0,0 +1,5 @@
+pr: 120291
+summary: ESQL - Allow full text functions disjunctions for non-full text functions
+area: ES|QL
+type: feature
+issues: []

docs/reference/esql/esql-limitations.asciidoc

@@ -112,7 +112,9 @@ it is necessary to use the search function, like <<esql-match>>, in a <<esql-whe
 directly after the <<esql-from>> source command, or close enough to it.
 Otherwise, the query will fail with a validation error.
 Another limitation is that any <<esql-where>> command containing a full-text search function
-cannot also use disjunctions (`OR`) unless all functions used in the OR clauses are full-text functions themselves.
+cannot use disjunctions (`OR`), unless:
+
+* All functions used in the OR clauses are full-text functions themselves, or scoring is not used
 
 For example, this query is valid:
 
@@ -131,19 +133,27 @@ FROM books
 | WHERE MATCH(author, "Faulkner")
 ----
 
-And this query will fail due to the disjunction:
+And this query that uses a disjunction will succeed:
 
 [source,esql]
 ----
 FROM books
+| WHERE MATCH(author, "Faulkner") OR QSTR("author: Hemingway")
+----
+
+However using scoring will fail because it uses a non full text function as part of the disjunction:
+
+[source,esql]
+----
+FROM books METADATA _score
 | WHERE MATCH(author, "Faulkner") OR author LIKE "Hemingway"
 ----
 
-However this query will succeed because it uses full text functions on both `OR` clauses:
+Scoring will work in the following query, as it uses full text functions on both `OR` clauses:
 
 [source,esql]
 ----
-FROM books
+FROM books METADATA _score
 | WHERE MATCH(author, "Faulkner") OR QSTR("author: Hemingway")
 ----
 

x-pack/plugin/build.gradle

@@ -101,7 +101,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("esql/190_lookup_join/alias-repeated-index", "LOOKUP JOIN does not support index aliases for now")
   task.skipTest("esql/190_lookup_join/alias-pattern-multiple", "LOOKUP JOIN does not support index aliases for now")
   task.skipTest("esql/190_lookup_join/alias-pattern-single", "LOOKUP JOIN does not support index aliases for now")
-
+  task.skipTest("esql/180_match_operator/match with disjunctions", "Disjunctions in full text functions work now")
 })
 
 tasks.named('yamlRestCompatTest').configure {

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneQueryExpressionEvaluator.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.compute.data.DocVector;
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
@@ -44,19 +45,20 @@ public record ShardConfig(Query query, IndexSearcher searcher) {}
 
     private final BlockFactory blockFactory;
     private final ShardConfig[] shards;
-    private final int docChannel;
 
     private ShardState[] perShardState = EMPTY_SHARD_STATES;
 
-    public LuceneQueryExpressionEvaluator(BlockFactory blockFactory, ShardConfig[] shards, int docChannel) {
+    public LuceneQueryExpressionEvaluator(BlockFactory blockFactory, ShardConfig[] shards) {
         this.blockFactory = blockFactory;
         this.shards = shards;
-        this.docChannel = docChannel;
     }
 
     @Override
     public Block eval(Page page) {
-        DocVector docs = page.<DocBlock>getBlock(docChannel).asVector();
+        // Lucene based operators retrieve DocVectors as first block
+        Block block = page.getBlock(0);
+        assert block instanceof DocBlock : "LuceneQueryExpressionEvaluator expects DocBlock as input";
+        DocVector docs = (DocVector) block.asVector();
         try {
             if (docs.singleSegmentNonDecreasing()) {
                 return evalSingleSegmentNonDecreasing(docs).asBlock();
@@ -341,4 +343,17 @@ public void close() {
             Releasables.closeExpectNoException(builder);
         }
     }
+
+    public static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+        private final ShardConfig[] shardConfigs;
+
+        public Factory(ShardConfig[] shardConfigs) {
+            this.shardConfigs = shardConfigs;
+        }
+
+        @Override
+        public EvalOperator.ExpressionEvaluator get(DriverContext context) {
+            return new LuceneQueryExpressionEvaluator(context.blockFactory(), shardConfigs);
+        }
+    }
 }

x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneQueryExpressionEvaluatorTests.java

@@ -183,8 +183,8 @@ private List<Page> runQuery(Set<String> values, Query query, boolean shuffleDocs
             );
             LuceneQueryExpressionEvaluator luceneQueryEvaluator = new LuceneQueryExpressionEvaluator(
                 blockFactory,
-                new LuceneQueryExpressionEvaluator.ShardConfig[] { shard },
-                0
+                new LuceneQueryExpressionEvaluator.ShardConfig[] { shard }
+
             );
 
             List<Operator> operators = new ArrayList<>();

x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec

@@ -152,3 +152,40 @@ emp_no:integer | first_name:keyword | last_name:keyword
 10053          | Sanjiv             | Zschoche
 10069          | Margareta          | Bierman
 ;
+
+testKqlWithNonPushableDisjunctions
+required_capability: kql_function
+required_capability: full_text_functions_disjunctions_compute_engine
+
+from books 
+| where kql("title:lord") or length(title) > 130 
+| keep book_no
+;
+ignoreOrder: true
+
+book_no:keyword
+2675   
+2714   
+4023   
+7140   
+8678
+;
+
+testKqlWithNonPushableDisjunctionsOnComplexExpressions
+required_capability: kql_function
+required_capability: full_text_functions_disjunctions_compute_engine
+
+from books 
+| where (kql("title:lord") and ratings > 4.5) or (kql("author:dostoevsky") and length(title) > 50)
+| keep book_no
+;
+ignoreOrder: true
+
+book_no:keyword
+2675
+2924
+4023
+1937
+7140
+2714
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec

@@ -718,3 +718,40 @@ from books
 title:text
 The Hobbit or There and Back Again
 ;
+
+testMatchWithNonPushableDisjunctions
+required_capability: match_function
+required_capability: full_text_functions_disjunctions_compute_engine
+
+from books 
+| where match(title, "lord") or length(title) > 130 
+| keep book_no
+;
+ignoreOrder: true
+
+book_no:keyword
+2675   
+2714   
+4023   
+7140   
+8678
+;
+
+testMatchWithNonPushableDisjunctionsOnComplexExpressions
+required_capability: match_function
+required_capability: full_text_functions_disjunctions_compute_engine
+
+from books 
+| where (match(title, "lord") and ratings > 4.5) or (match(author, "dostoevsky") and length(title) > 50)
+| keep book_no
+;
+ignoreOrder: true
+
+book_no:keyword
+2675
+2924
+4023
+1937
+7140
+2714
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec

@@ -684,3 +684,40 @@ from semantic_text
 host:keyword | semantic_text_field:text
 "host1"      | live long and prosper
 ;
+
+testMatchWithNonPushableDisjunctions
+required_capability: match_operator_colon
+required_capability: full_text_functions_disjunctions_compute_engine
+
+from books 
+| where title:"lord" or length(title) > 130 
+| keep book_no
+;
+ignoreOrder: true
+
+book_no:keyword
+2675   
+2714   
+4023   
+7140   
+8678
+;
+
+testMatchWithNonPushableDisjunctionsOnComplexExpressions
+required_capability: match_operator_colon
+required_capability: full_text_functions_disjunctions_compute_engine
+
+from books 
+| where (title:"lord" and ratings > 4.5) or (author:"dostoevsky" and length(title) > 50)
+| keep book_no
+;
+ignoreOrder: true
+
+book_no:keyword
+2675
+2924
+4023
+1937
+7140
+2714
+;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec

@@ -151,3 +151,40 @@ emp_no:integer | first_name:keyword | last_name:keyword
 10053          | Sanjiv             | Zschoche
 10069          | Margareta          | Bierman
 ;
+
+testQstrWithNonPushableDisjunctions
+required_capability: qstr_function
+required_capability: full_text_functions_disjunctions_compute_engine
+
+from books 
+| where qstr("title:lord") or length(title) > 130 
+| keep book_no
+;
+ignoreOrder: true
+
+book_no:keyword
+2675   
+2714   
+4023   
+7140   
+8678
+;
+
+testQstrWithNonPushableDisjunctionsOnComplexExpressions
+required_capability: qstr_function
+required_capability: full_text_functions_disjunctions_compute_engine
+
+from books 
+| where (qstr("title:lord") and ratings > 4.5) or (qstr("author:dostoevsky") and length(title) > 50)
+| keep book_no
+;
+ignoreOrder: true
+
+book_no:keyword
+2675
+2924
+4023
+1937
+7140
+2714
+;

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/MatchFunctionIT.java

@@ -14,8 +14,6 @@
 import org.elasticsearch.xpack.esql.VerificationException;
 import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase;
 import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
-import org.elasticsearch.xpack.esql.action.EsqlQueryRequest;
-import org.elasticsearch.xpack.esql.action.EsqlQueryResponse;
 import org.junit.Before;
 
 import java.util.List;
@@ -31,12 +29,6 @@ public void setupIndex() {
         createAndPopulateIndex();
     }
 
-    @Override
-    protected EsqlQueryResponse run(EsqlQueryRequest request) {
-        assumeTrue("match function capability not available", EsqlCapabilities.Cap.MATCH_FUNCTION.isEnabled());
-        return super.run(request);
-    }
-
     public void testSimpleWhereMatch() {
         var query = """
             FROM test
@@ -230,20 +222,19 @@ public void testWhereMatchAfterStats() {
         assertThat(error.getMessage(), containsString("Unknown column [content]"));
     }
 
-    public void testWhereMatchWithFunctions() {
+    public void testWhereMatchNotPushedDown() {
         var query = """
             FROM test
-            | WHERE match(content, "fox") OR to_upper(content) == "FOX"
+            | WHERE match(content, "fox") OR length(content) < 20
+            | KEEP id
+            | SORT id
             """;
-        var error = expectThrows(ElasticsearchException.class, () -> run(query));
-        assertThat(
-            error.getMessage(),
-            containsString(
-                "Invalid condition [match(content, \"fox\") OR to_upper(content) == \"FOX\"]. "
-                    + "Full text functions can be used in an OR condition,"
-                    + " but only if just full text functions are used in the OR condition"
-            )
-        );
+
+        try (var resp = run(query)) {
+            assertColumnNames(resp.columns(), List.of("id"));
+            assertColumnTypes(resp.columns(), List.of("integer"));
+            assertValues(resp.values(), List.of(List.of(1), List.of(2), List.of(6)));
+        }
     }
 
     public void testWhereMatchWithRow() {