Merge branch 'main' into 2025/10/24/new-id-format

Author: tlrx

docs/changelog/137536.yaml

@@ -0,0 +1,5 @@
+pr: 137536
+summary: Use DV rewrites where possible in Keyword queries
+area: Search
+type: enhancement
+issues: []

muted-tests.yml

@@ -426,27 +426,12 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.BasicDistributedJobsIT
   method: testFailOverBasics
   issue: https://github.com/elastic/elasticsearch/issues/136778
-- class: org.elasticsearch.xpack.inference.qa.mixed.CohereServiceMixedIT
-  method: testRerank
-  issue: https://github.com/elastic/elasticsearch/issues/136872
-- class: org.elasticsearch.xpack.inference.qa.mixed.CohereServiceMixedIT
-  method: testCohereEmbeddings
-  issue: https://github.com/elastic/elasticsearch/issues/136779
 - class: org.elasticsearch.xpack.search.CrossClusterAsyncSearchIT
   method: testRemoteClusterOnlyCCSWithFailuresOnAllShards
   issue: https://github.com/elastic/elasticsearch/issues/136894
-- class: org.elasticsearch.xpack.esql.qa.mixed.FieldExtractorIT
-  method: testTextFieldWithIpSubfieldMalformed {STORED}
-  issue: https://github.com/elastic/elasticsearch/issues/136917
-- class: org.elasticsearch.xpack.esql.qa.mixed.FieldExtractorIT
-  method: testTextFieldWithKeywordSubfield {STORED}
-  issue: https://github.com/elastic/elasticsearch/issues/136918
 - class: org.elasticsearch.xpack.esql.optimizer.rules.logical.HoistRemoteEnrichTopNTests
   method: testTopNSortExpressionWithinRemoteEnrichAliasing
   issue: https://github.com/elastic/elasticsearch/issues/136957
-- class: org.elasticsearch.xpack.esql.qa.mixed.FieldExtractorIT
-  method: testByteFieldWithIntSubfieldTooBig {NONE}
-  issue: https://github.com/elastic/elasticsearch/issues/137034
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeMetricsIT
   method: test
   issue: https://github.com/elastic/elasticsearch/issues/137071
@@ -481,6 +466,9 @@ tests:
   method: testSimilarityWithOneDimVector {functionName=v_cosine
     similarityFunction=org.elasticsearch.xpack.esql.expression.function.vector.CosineSimilarity$1@6f152006 elementType=byte}
   issue: https://github.com/elastic/elasticsearch/issues/137552
+- class: org.elasticsearch.xpack.ilm.CCRIndexLifecycleIT
+  method: testTsdbLeaderIndexRolloverAndSyncAfterWaitUntilEndTime {targetCluster=FOLLOWER}
+  issue: https://github.com/elastic/elasticsearch/issues/137565
 
 # Examples:
 #

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -27,8 +27,12 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.FuzzyQuery;
 import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
+import org.apache.lucene.search.RegexpQuery;
+import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
@@ -67,9 +71,7 @@
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
-import org.elasticsearch.search.runtime.StringScriptFieldFuzzyQuery;
 import org.elasticsearch.search.runtime.StringScriptFieldPrefixQuery;
-import org.elasticsearch.search.runtime.StringScriptFieldRegexpQuery;
 import org.elasticsearch.search.runtime.StringScriptFieldTermQuery;
 import org.elasticsearch.search.runtime.StringScriptFieldWildcardQuery;
 import org.elasticsearch.xcontent.Text;
@@ -720,14 +722,13 @@ public Query fuzzyQuery(
             if (indexType.hasTerms()) {
                 return super.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context, rewriteMethod);
             } else {
-                return StringScriptFieldFuzzyQuery.build(
-                    new Script(""),
-                    ctx -> new SortedSetDocValuesStringFieldScript(name(), context.lookup(), ctx),
-                    name(),
-                    indexedValueForSearch(value).utf8ToString(),
+                return new FuzzyQuery(
+                    new Term(name(), indexedValueForSearch(value)),
                     fuzziness.asDistance(BytesRefs.toString(value)),
                     prefixLength,
-                    transpositions
+                    maxExpansions,
+                    transpositions,
+                    MultiTermQuery.DOC_VALUES_REWRITE
                 );
             }
         }
@@ -743,6 +744,10 @@ public Query prefixQuery(
             if (indexType.hasTerms()) {
                 return super.prefixQuery(value, method, caseInsensitive, context);
             } else {
+                if (caseInsensitive == false) {
+                    Term prefix = new Term(name(), indexedValueForSearch(value));
+                    return new PrefixQuery(prefix, MultiTermQuery.DOC_VALUES_REWRITE);
+                }
                 return new StringScriptFieldPrefixQuery(
                     new Script(""),
                     ctx -> new SortedSetDocValuesStringFieldScript(name(), context.lookup(), ctx),
@@ -1017,6 +1022,10 @@ public Query wildcardQuery(
                 } else {
                     value = indexedValueForSearch(value).utf8ToString();
                 }
+                if (caseInsensitive == false) {
+                    Term term = new Term(name(), value);
+                    return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE);
+                }
                 return new StringScriptFieldWildcardQuery(
                     new Script(""),
                     ctx -> new SortedSetDocValuesStringFieldScript(name(), context.lookup(), ctx),
@@ -1038,13 +1047,8 @@ public Query normalizedWildcardQuery(String value, MultiTermQuery.RewriteMethod
                 } else {
                     value = indexedValueForSearch(value).utf8ToString();
                 }
-                return new StringScriptFieldWildcardQuery(
-                    new Script(""),
-                    ctx -> new SortedSetDocValuesStringFieldScript(name(), context.lookup(), ctx),
-                    name(),
-                    value,
-                    false
-                );
+                Term term = new Term(name(), value);
+                return new WildcardQuery(term, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT, MultiTermQuery.DOC_VALUES_REWRITE);
             }
         }
 
@@ -1064,14 +1068,13 @@ public Query regexpQuery(
                 if (matchFlags != 0) {
                     throw new IllegalArgumentException("Match flags not yet implemented [" + matchFlags + "]");
                 }
-                return new StringScriptFieldRegexpQuery(
-                    new Script(""),
-                    ctx -> new SortedSetDocValuesStringFieldScript(name(), context.lookup(), ctx),
-                    name(),
-                    indexedValueForSearch(value).utf8ToString(),
+                return new RegexpQuery(
+                    new Term(name(), indexedValueForSearch(value)),
                     syntaxFlags,
                     matchFlags,
-                    maxDeterminizedStates
+                    RegexpQuery.DEFAULT_PROVIDER,
+                    maxDeterminizedStates,
+                    MultiTermQuery.DOC_VALUES_REWRITE
                 );
             }
         }

x-pack/plugin/ml/src/main/java/module-info.java

@@ -32,6 +32,7 @@
     opens org.elasticsearch.xpack.ml to org.elasticsearch.painless.spi; // whitelist resource access
     opens org.elasticsearch.xpack.ml.utils; // for exact.properties access
 
+    provides org.elasticsearch.features.FeatureSpecification with org.elasticsearch.xpack.ml.MachineLearningFeatures;
     provides org.elasticsearch.painless.spi.PainlessExtension with org.elasticsearch.xpack.ml.MachineLearningPainlessExtension;
     provides org.elasticsearch.xpack.autoscaling.AutoscalingExtension with org.elasticsearch.xpack.ml.autoscaling.MlAutoscalingExtension;
 

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearningFeatures.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.ml;
+
+import org.elasticsearch.features.FeatureSpecification;
+import org.elasticsearch.features.NodeFeature;
+
+import java.util.Set;
+
+public class MachineLearningFeatures implements FeatureSpecification {
+
+    public static final NodeFeature COMPONENTS_RESET_ACTION = new NodeFeature("ml.components.reset");
+
+    public Set<NodeFeature> getFeatures() {
+        return Set.of(COMPONENTS_RESET_ACTION);
+    }
+}

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportResetMlComponentsAction.java

@@ -7,17 +7,20 @@
 
 package org.elasticsearch.xpack.ml.action;
 
+import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.FailedNodeException;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.nodes.TransportNodesAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 import org.elasticsearch.xpack.core.ml.action.ResetMlComponentsAction;
+import org.elasticsearch.xpack.ml.MachineLearningFeatures;
 import org.elasticsearch.xpack.ml.inference.TrainedModelStatsService;
 import org.elasticsearch.xpack.ml.notifications.AnomalyDetectionAuditor;
 import org.elasticsearch.xpack.ml.notifications.DataFrameAnalyticsAuditor;
@@ -37,6 +40,7 @@ public class TransportResetMlComponentsAction extends TransportNodesAction<
     private final DataFrameAnalyticsAuditor dfaAuditor;
     private final InferenceAuditor inferenceAuditor;
     private final TrainedModelStatsService trainedModelStatsService;
+    private final FeatureService featureService;
 
     @Inject
     public TransportResetMlComponentsAction(
@@ -47,7 +51,8 @@ public TransportResetMlComponentsAction(
         AnomalyDetectionAuditor anomalyDetectionAuditor,
         DataFrameAnalyticsAuditor dfaAuditor,
         InferenceAuditor inferenceAuditor,
-        TrainedModelStatsService trainedModelStatsService
+        TrainedModelStatsService trainedModelStatsService,
+        FeatureService featureService
     ) {
         super(
             ResetMlComponentsAction.NAME,
@@ -61,6 +66,20 @@ public TransportResetMlComponentsAction(
         this.dfaAuditor = dfaAuditor;
         this.inferenceAuditor = inferenceAuditor;
         this.trainedModelStatsService = trainedModelStatsService;
+        this.featureService = featureService;
+    }
+
+    @Override
+    protected void doExecute(
+        Task task,
+        ResetMlComponentsAction.Request request,
+        ActionListener<ResetMlComponentsAction.Response> listener
+    ) {
+        if (featureService.clusterHasFeature(clusterService.state(), MachineLearningFeatures.COMPONENTS_RESET_ACTION) == false) {
+            listener.onResponse(new ResetMlComponentsAction.Response(clusterService.getClusterName(), List.of(), List.of()));
+        } else {
+            super.doExecute(task, request, listener);
+        }
     }
 
     @Override