merging main

Author: masseyke

docs/changelog/126641.yaml

@@ -0,0 +1,5 @@
+pr: 126641
+summary: Push more `==`s on text fields to lucene
+area: ES|QL
+type: enhancement
+issues: []

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorMatchedSlotSubFetchPhase.java

@@ -100,7 +100,7 @@ public void process(HitContext hitContext) throws IOException {
                     IntStream slots = convertTopDocsToSlots(topDocs, pc.rootDocsBySlot);
                     // _percolator_document_slot fields are document fields and should be under "fields" section in a hit
                     List<Object> docSlots = slots.boxed().collect(Collectors.toList());
-                    hitContext.hit().setDocumentField(fieldName, new DocumentField(fieldName, docSlots));
+                    hitContext.hit().setDocumentField(new DocumentField(fieldName, docSlots));
 
                     // Add info what sub-queries of percolator query matched this each percolated document
                     if (fetchContext.getSearchExecutionContext().hasNamedQueries()) {
@@ -120,7 +120,7 @@ public void process(HitContext hitContext) throws IOException {
                                 matchedQueries.add(match.getName());
                             }
                             String matchedFieldName = fieldName + "_" + docSlots.get(i) + "_matched_queries";
-                            hitContext.hit().setDocumentField(matchedFieldName, new DocumentField(matchedFieldName, matchedQueries));
+                            hitContext.hit().setDocumentField(new DocumentField(matchedFieldName, matchedQueries));
                         }
                     }
                 }

muted-tests.yml

@@ -438,6 +438,12 @@ tests:
 - class: org.elasticsearch.xpack.esql.plugin.DataNodeRequestSenderTests
   method: testRetryOnlyMovedShards
   issue: https://github.com/elastic/elasticsearch/issues/127168
+- class: org.elasticsearch.xpack.esql.plugin.DataNodeRequestSenderIT
+  method: testSearchWhileRelocating
+  issue: https://github.com/elastic/elasticsearch/issues/127188
+- class: org.elasticsearch.synonyms.SynonymsManagementAPIServiceIT
+  method: testUpdateRuleWithMaxSynonyms
+  issue: https://github.com/elastic/elasticsearch/issues/127195
 
 # Examples:
 #

server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java

@@ -1327,8 +1327,7 @@ public void setNextReader(LeafReaderContext ctx) {
                         public void process(FetchSubPhase.HitContext hitContext) {
                             leafSearchLookup.setDocument(hitContext.docId());
                             FieldLookup fieldLookup = leafSearchLookup.fields().get("text");
-                            hitContext.hit()
-                                .setDocumentField("text_stored_lookup", new DocumentField("text_stored_lookup", fieldLookup.getValues()));
+                            hitContext.hit().setDocumentField(new DocumentField("text_stored_lookup", fieldLookup.getValues()));
                         }
 
                         @Override

server/src/internalClusterTest/java/org/elasticsearch/search/fetch/FetchSubPhasePluginIT.java

@@ -137,7 +137,7 @@ private void hitExecute(FetchContext context, HitContext hitContext) throws IOEx
             DocumentField hitField = hitContext.hit().getFields().get(NAME);
             if (hitField == null) {
                 hitField = new DocumentField(NAME, new ArrayList<>(1));
-                hitContext.hit().setDocumentField(NAME, hitField);
+                hitContext.hit().setDocumentField(hitField);
             }
             Terms terms = hitContext.reader().termVectors().get(hitContext.docId(), field);
             if (terms != null) {

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -233,7 +233,8 @@ static TransportVersion def(int id) {
     public static final TransportVersion SEARCH_INCREMENTAL_TOP_DOCS_NULL = def(9_058_0_00);
     public static final TransportVersion COMPRESS_DELAYABLE_WRITEABLE = def(9_059_0_00);
     public static final TransportVersion SYNONYMS_REFRESH_PARAM = def(9_060_0_00);
-    public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_061_00_0);
+    public static final TransportVersion DOC_FIELDS_AS_LIST = def(9_061_0_00);
+    public static final TransportVersion SETTINGS_IN_DATA_STREAMS = def(9_062_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/common/document/DocumentField.java

@@ -24,6 +24,7 @@
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
@@ -70,6 +71,15 @@ public DocumentField(String name, List<Object> values, List<Object> ignoredValue
             : "DocumentField can't have both lookup fields and values";
     }
 
+    /**
+     * Read map of document fields written via {@link StreamOutput#writeMapValues(Map)}.
+     * @param in stream input
+     * @return map of {@link DocumentField} keyed by {@link DocumentField#getName()}
+     */
+    public static Map<String, DocumentField> readFieldsFromMapValues(StreamInput in) throws IOException {
+        return in.readMapValues(DocumentField::new, DocumentField::getName);
+    }
+
     /**
      * The name of the field.
      */

server/src/main/java/org/elasticsearch/index/get/GetResult.java

@@ -74,8 +74,8 @@ public GetResult(StreamInput in) throws IOException {
             if (source.length() == 0) {
                 source = null;
             }
-            documentFields = in.readMapValues(DocumentField::new, DocumentField::getName);
-            metaFields = in.readMapValues(DocumentField::new, DocumentField::getName);
+            documentFields = DocumentField.readFieldsFromMapValues(in);
+            metaFields = DocumentField.readFieldsFromMapValues(in);
         } else {
             metaFields = Collections.emptyMap();
             documentFields = Collections.emptyMap();

server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java

@@ -983,6 +983,21 @@ public boolean canUseSyntheticSourceDelegateForQuerying() {
                 && syntheticSourceDelegate.isIndexed();
         }
 
+        /**
+         * Returns true if the delegate sub-field can be used for querying only (ie. isIndexed must be true)
+         */
+        public boolean canUseSyntheticSourceDelegateForQueryingEquality(String str) {
+            if (syntheticSourceDelegate == null
+                // Can't push equality to an index if there isn't an index
+                || syntheticSourceDelegate.isIndexed() == false
+                // ESQL needs docs values to push equality
+                || syntheticSourceDelegate.hasDocValues() == false) {
+                return false;
+            }
+            // Can't push equality if the field we're checking for is so big we'd ignore it.
+            return str.length() <= syntheticSourceDelegate.ignoreAbove();
+        }
+
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (canUseSyntheticSourceDelegateForLoading()) {

server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java

@@ -132,6 +132,11 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
         public Query termQuery(Object value, SearchExecutionContext context) {
             throw new IllegalArgumentException("[" + NAME + "] is not searchable");
         }
+
+        @Override
+        public BlockLoader blockLoader(BlockLoaderContext blContext) {
+            return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name());
+        }
     }
 
     private final boolean useDocValuesSkipper;