Merge branch 'main' into lucene_snapshot_10_1

Author: ChrisHegarty

docs/README.asciidoc

@@ -157,16 +157,15 @@ used for its modifiers:
   * `// TESTRESPONSE[skip:reason]`: Skip the assertions specified by this
   response.
 * `// TESTSETUP`: Marks this snippet as the "setup" for all other snippets in
-  this file. This is a somewhat natural way of structuring documentation. You
-  say "this is the data we use to explain this feature" then you add the
-  snippet that you mark `// TESTSETUP` and then every snippet will turn into
-  a test that runs the setup snippet first. See the "painless" docs for a file
-  that puts this to good use. This is fairly similar to `// TEST[setup:name]`
-  but rather than the setup defined in `docs/build.gradle` the setup is defined
-  right in the documentation file. In general, we should prefer `// TESTSETUP`
-  over `// TEST[setup:name]` because it makes it more clear what steps have to
-  be taken before the examples will work. Tip: `// TESTSETUP` can only be used
-  on the first snippet of a document.
+  this file. In order to enhance clarity and simplify understanding for readers,
+  a straightforward approach involves marking the first snippet in the documentation file with the 
+  `// TESTSETUP` marker. By doing so, it clearly indicates that this particular snippet serves as the setup 
+  or preparation step for all subsequent snippets in the file. 
+  This helps in explaining the necessary steps that need to be executed before running the examples.
+  Unlike the alternative convention `// TEST[setup:name]`, which relies on a setup defined in a separate file,
+  this convention brings the setup directly into the documentation file, making it more self-contained and reducing ambiguity.
+  By adopting this convention, users can easily identify and follow the correct sequence 
+  of steps to ensure that the examples provided in the documentation work as intended.
 * `// TEARDOWN`: Ends and cleans up a test series started with `// TESTSETUP` or
   `// TEST[setup:name]`. You can use `// TEARDOWN` to set up multiple tests in
   the same file.

docs/changelog/120291.yaml

@@ -0,0 +1,5 @@
+pr: 120291
+summary: ESQL - Allow full text functions disjunctions for non-full text functions
+area: ES|QL
+type: feature
+issues: []

docs/reference/esql/esql-limitations.asciidoc

@@ -112,7 +112,9 @@ it is necessary to use the search function, like <<esql-match>>, in a <<esql-whe
 directly after the <<esql-from>> source command, or close enough to it.
 Otherwise, the query will fail with a validation error.
 Another limitation is that any <<esql-where>> command containing a full-text search function
-cannot also use disjunctions (`OR`) unless all functions used in the OR clauses are full-text functions themselves.
+cannot use disjunctions (`OR`), unless:
+
+* All functions used in the OR clauses are full-text functions themselves, or scoring is not used
 
 For example, this query is valid:
 
@@ -131,19 +133,27 @@ FROM books
 | WHERE MATCH(author, "Faulkner")
 ----
 
-And this query will fail due to the disjunction:
+And this query that uses a disjunction will succeed:
 
 [source,esql]
 ----
 FROM books
+| WHERE MATCH(author, "Faulkner") OR QSTR("author: Hemingway")
+----
+
+However using scoring will fail because it uses a non full text function as part of the disjunction:
+
+[source,esql]
+----
+FROM books METADATA _score
 | WHERE MATCH(author, "Faulkner") OR author LIKE "Hemingway"
 ----
 
-However this query will succeed because it uses full text functions on both `OR` clauses:
+Scoring will work in the following query, as it uses full text functions on both `OR` clauses:
 
 [source,esql]
 ----
-FROM books
+FROM books METADATA _score
 | WHERE MATCH(author, "Faulkner") OR QSTR("author: Hemingway")
 ----
 

muted-tests.yml

@@ -289,6 +289,12 @@ tests:
 - class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityEsqlIT
   method: testCrossClusterAsyncQueryStop
   issue: https://github.com/elastic/elasticsearch/issues/121021
+- class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
+  method: testSuggestProfilesWithName
+  issue: https://github.com/elastic/elasticsearch/issues/121022
+- class: org.elasticsearch.xpack.inference.action.filter.ShardBulkInferenceActionFilterIT
+  method: testBulkOperations {p0=true}
+  issue: https://github.com/elastic/elasticsearch/issues/120969
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -718,7 +718,8 @@ public Iterator<Setting<?>> settings() {
         "index.mapping.source.mode",
         value -> {},
         Setting.Property.Final,
-        Setting.Property.IndexScope
+        Setting.Property.IndexScope,
+        Setting.Property.ServerlessPublic
     );
 
     public static final Setting<Boolean> RECOVERY_USE_SYNTHETIC_SOURCE_SETTING = Setting.boolSetting(

x-pack/plugin/build.gradle

@@ -101,7 +101,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
   task.skipTest("esql/190_lookup_join/alias-repeated-index", "LOOKUP JOIN does not support index aliases for now")
   task.skipTest("esql/190_lookup_join/alias-pattern-multiple", "LOOKUP JOIN does not support index aliases for now")
   task.skipTest("esql/190_lookup_join/alias-pattern-single", "LOOKUP JOIN does not support index aliases for now")
-
+  task.skipTest("esql/180_match_operator/match with disjunctions", "Disjunctions in full text functions work now")
 })
 
 tasks.named('yamlRestCompatTest').configure {

x-pack/plugin/esql/compute/src/main/java/module-info.java

@@ -35,4 +35,5 @@
     exports org.elasticsearch.compute.operator.mvdedupe;
     exports org.elasticsearch.compute.aggregation.table;
     exports org.elasticsearch.compute.data.sort;
+    exports org.elasticsearch.compute.querydsl.query;
 }

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneQueryExpressionEvaluator.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.compute.data.DocVector;
 import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.Page;
+import org.elasticsearch.compute.operator.DriverContext;
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.core.Releasable;
 import org.elasticsearch.core.Releasables;
@@ -44,19 +45,20 @@ public record ShardConfig(Query query, IndexSearcher searcher) {}
 
     private final BlockFactory blockFactory;
     private final ShardConfig[] shards;
-    private final int docChannel;
 
     private ShardState[] perShardState = EMPTY_SHARD_STATES;
 
-    public LuceneQueryExpressionEvaluator(BlockFactory blockFactory, ShardConfig[] shards, int docChannel) {
+    public LuceneQueryExpressionEvaluator(BlockFactory blockFactory, ShardConfig[] shards) {
         this.blockFactory = blockFactory;
         this.shards = shards;
-        this.docChannel = docChannel;
     }
 
     @Override
     public Block eval(Page page) {
-        DocVector docs = page.<DocBlock>getBlock(docChannel).asVector();
+        // Lucene based operators retrieve DocVectors as first block
+        Block block = page.getBlock(0);
+        assert block instanceof DocBlock : "LuceneQueryExpressionEvaluator expects DocBlock as input";
+        DocVector docs = (DocVector) block.asVector();
         try {
             if (docs.singleSegmentNonDecreasing()) {
                 return evalSingleSegmentNonDecreasing(docs).asBlock();
@@ -341,4 +343,17 @@ public void close() {
             Releasables.closeExpectNoException(builder);
         }
     }
+
+    public static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
+        private final ShardConfig[] shardConfigs;
+
+        public Factory(ShardConfig[] shardConfigs) {
+            this.shardConfigs = shardConfigs;
+        }
+
+        @Override
+        public EvalOperator.ExpressionEvaluator get(DriverContext context) {
+            return new LuceneQueryExpressionEvaluator(context.blockFactory(), shardConfigs);
+        }
+    }
 }

x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java

@@ -9,6 +9,9 @@
 
 import org.apache.lucene.document.InetAddressPoint;
 import org.apache.lucene.geo.GeoEncodingUtils;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.geo.ShapeRelation;
@@ -20,6 +23,8 @@
 import org.elasticsearch.compute.data.FloatBlock;
 import org.elasticsearch.compute.data.IntBlock;
 import org.elasticsearch.compute.data.LongBlock;
+import org.elasticsearch.compute.operator.Warnings;
+import org.elasticsearch.compute.querydsl.query.SingleValueMatchQuery;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.geometry.Point;
@@ -30,6 +35,7 @@
 import org.elasticsearch.index.mapper.RangeFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.function.IntFunction;
@@ -38,10 +44,14 @@
  * Generates a list of Lucene queries based on the input block.
  */
 public abstract class QueryList {
+    protected final SearchExecutionContext searchExecutionContext;
+    protected final MappedFieldType field;
     protected final Block block;
     protected final boolean onlySingleValues;
 
-    protected QueryList(Block block, boolean onlySingleValues) {
+    protected QueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, Block block, boolean onlySingleValues) {
+        this.searchExecutionContext = searchExecutionContext;
+        this.field = field;
         this.block = block;
         this.onlySingleValues = onlySingleValues;
     }
@@ -59,11 +69,52 @@ int getPositionCount() {
      */
     public abstract QueryList onlySingleValues();
 
+    final Query getQuery(int position) {
+        final int valueCount = block.getValueCount(position);
+        if (onlySingleValues && valueCount != 1) {
+            return null;
+        }
+        final int firstValueIndex = block.getFirstValueIndex(position);
+
+        Query query = doGetQuery(position, firstValueIndex, valueCount);
+
+        if (onlySingleValues) {
+            query = wrapSingleValueQuery(query);
+        }
+
+        return query;
+    }
+
     /**
      * Returns the query at the given position.
      */
     @Nullable
-    abstract Query getQuery(int position);
+    abstract Query doGetQuery(int position, int firstValueIndex, int valueCount);
+
+    private Query wrapSingleValueQuery(Query query) {
+        SingleValueMatchQuery singleValueQuery = new SingleValueMatchQuery(
+            searchExecutionContext.getForField(field, MappedFieldType.FielddataOperation.SEARCH),
+            // Not emitting warnings for multivalued fields not matching
+            Warnings.NOOP_WARNINGS
+        );
+
+        Query rewrite = singleValueQuery;
+        try {
+            rewrite = singleValueQuery.rewrite(searchExecutionContext.searcher());
+            if (rewrite instanceof MatchAllDocsQuery) {
+                // nothing to filter
+                return query;
+            }
+        } catch (IOException e) {
+            // ignore
+            // TODO: Should we do something with the exception?
+        }
+
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        builder.add(query, BooleanClause.Occur.FILTER);
+        builder.add(rewrite, BooleanClause.Occur.FILTER);
+        return builder.build();
+    }
 
     /**
      * Returns a list of term queries for the given field and the input block
@@ -146,8 +197,6 @@ public static QueryList geoShapeQueryList(MappedFieldType field, SearchExecution
     }
 
     private static class TermQueryList extends QueryList {
-        private final MappedFieldType field;
-        private final SearchExecutionContext searchExecutionContext;
         private final IntFunction<Object> blockValueReader;
 
         private TermQueryList(
@@ -157,9 +206,7 @@ private TermQueryList(
             boolean onlySingleValues,
             IntFunction<Object> blockValueReader
         ) {
-            super(block, onlySingleValues);
-            this.field = field;
-            this.searchExecutionContext = searchExecutionContext;
+            super(field, searchExecutionContext, block, onlySingleValues);
             this.blockValueReader = blockValueReader;
         }
 
@@ -169,19 +216,14 @@ public TermQueryList onlySingleValues() {
         }
 
         @Override
-        Query getQuery(int position) {
-            final int count = block.getValueCount(position);
-            if (onlySingleValues && count != 1) {
-                return null;
-            }
-            final int first = block.getFirstValueIndex(position);
-            return switch (count) {
+        Query doGetQuery(int position, int firstValueIndex, int valueCount) {
+            return switch (valueCount) {
                 case 0 -> null;
-                case 1 -> field.termQuery(blockValueReader.apply(first), searchExecutionContext);
+                case 1 -> field.termQuery(blockValueReader.apply(firstValueIndex), searchExecutionContext);
                 default -> {
-                    final List<Object> terms = new ArrayList<>(count);
-                    for (int i = 0; i < count; i++) {
-                        final Object value = blockValueReader.apply(first + i);
+                    final List<Object> terms = new ArrayList<>(valueCount);
+                    for (int i = 0; i < valueCount; i++) {
+                        final Object value = blockValueReader.apply(firstValueIndex + i);
                         terms.add(value);
                     }
                     yield field.termsQuery(terms, searchExecutionContext);
@@ -192,8 +234,6 @@ Query getQuery(int position) {
 
     private static class GeoShapeQueryList extends QueryList {
         private final BytesRef scratch = new BytesRef();
-        private final MappedFieldType field;
-        private final SearchExecutionContext searchExecutionContext;
         private final IntFunction<Geometry> blockValueReader;
         private final IntFunction<Query> shapeQuery;
 
@@ -203,10 +243,8 @@ private GeoShapeQueryList(
             Block block,
             boolean onlySingleValues
         ) {
-            super(block, onlySingleValues);
+            super(field, searchExecutionContext, block, onlySingleValues);
 
-            this.field = field;
-            this.searchExecutionContext = searchExecutionContext;
             this.blockValueReader = blockToGeometry(block);
             this.shapeQuery = shapeQuery();
         }
@@ -217,15 +255,10 @@ public GeoShapeQueryList onlySingleValues() {
         }
 
         @Override
-        Query getQuery(int position) {
-            final int count = block.getValueCount(position);
-            if (onlySingleValues && count != 1) {
-                return null;
-            }
-            final int first = block.getFirstValueIndex(position);
-            return switch (count) {
+        Query doGetQuery(int position, int firstValueIndex, int valueCount) {
+            return switch (valueCount) {
                 case 0 -> null;
-                case 1 -> shapeQuery.apply(first);
+                case 1 -> shapeQuery.apply(firstValueIndex);
                 // TODO: support multiple values
                 default -> throw new IllegalArgumentException("can't read multiple Geometry values from a single position");
             };

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/querydsl/query/SingleValueMatchQuery.java renamed to x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/querydsl/query/SingleValueMatchQuery.java

@@ -5,7 +5,7 @@
  * 2.0.
  */
 
-package org.elasticsearch.xpack.esql.querydsl.query;
+package org.elasticsearch.compute.querydsl.query;
 
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
@@ -39,7 +39,7 @@
 /**
  * Finds all fields with a single-value. If a field has a multi-value, it emits a {@link Warnings}.
  */
-final class SingleValueMatchQuery extends Query {
+public final class SingleValueMatchQuery extends Query {
 
     /**
      * Choose a big enough value so this approximation never drives the iteration.
@@ -52,7 +52,7 @@ final class SingleValueMatchQuery extends Query {
     private final IndexFieldData<?> fieldData;
     private final Warnings warnings;
 
-    SingleValueMatchQuery(IndexFieldData<?> fieldData, Warnings warnings) {
+    public SingleValueMatchQuery(IndexFieldData<?> fieldData, Warnings warnings) {
         this.fieldData = fieldData;
         this.warnings = warnings;
     }