Merge branch 'main' into file-prefixes

Author: prdoyle

docs/changelog/121240.yaml

@@ -0,0 +1,5 @@
+pr: 121240
+summary: Implement runtime skip_unavailable=true
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/121256.yaml

@@ -0,0 +1,5 @@
+pr: 121256
+summary: Run `TransportEnrichStatsAction` on local node
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/121556.yaml

@@ -0,0 +1,5 @@
+pr: 121556
+summary: Enable New Semantic Text Format Only On Newly Created Indices
+area: Mapping
+type: bug
+issues: []

muted-tests.yml

@@ -122,15 +122,9 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
   method: test {p0=search.highlight/50_synthetic_source/text multi unified from vectors}
   issue: https://github.com/elastic/elasticsearch/issues/117815
-- class: org.elasticsearch.xpack.esql.plugin.ClusterRequestTests
-  method: testFallbackIndicesOptions
-  issue: https://github.com/elastic/elasticsearch/issues/117937
 - class: org.elasticsearch.xpack.ml.integration.RegressionIT
   method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet
   issue: https://github.com/elastic/elasticsearch/issues/117805
-- class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT
-  method: testCancelRequestWhenFailingFetchingPages
-  issue: https://github.com/elastic/elasticsearch/issues/118193
 - class: org.elasticsearch.packaging.test.ArchiveTests
   method: test44AutoConfigurationNotTriggeredOnNotWriteableConfDir
   issue: https://github.com/elastic/elasticsearch/issues/118208
@@ -384,9 +378,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT
   method: testDependentVariableIsAliasToNested
   issue: https://github.com/elastic/elasticsearch/issues/121415
-- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
-  method: testLookupExplosionBigStringManyMatches
-  issue: https://github.com/elastic/elasticsearch/issues/121465
 - class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
   method: testClientSecretRotation
   issue: https://github.com/elastic/elasticsearch/issues/120985
@@ -396,9 +387,6 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=cluster.health/10_basic/cluster health basic test}
   issue: https://github.com/elastic/elasticsearch/issues/121478
-- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
-  method: testLookupExplosionManyMatches
-  issue: https://github.com/elastic/elasticsearch/issues/121481
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
   method: testGetUsersWithProfileUid
   issue: https://github.com/elastic/elasticsearch/issues/121483
@@ -408,15 +396,6 @@ tests:
 - class: org.elasticsearch.xpack.transform.checkpoint.TransformCCSCanMatchIT
   method: testTransformLifecycle_RangeQueryThatMatchesNoShards
   issue: https://github.com/elastic/elasticsearch/issues/121480
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
-  method: testStopQueryLocal
-  issue: https://github.com/elastic/elasticsearch/issues/121487
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
-  method: testSuccessfulPathways
-  issue: https://github.com/elastic/elasticsearch/issues/121488
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
-  method: testAsyncQueriesWithLimit0
-  issue: https://github.com/elastic/elasticsearch/issues/121489
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
   method: testSuggestProfilesWithHint
   issue: https://github.com/elastic/elasticsearch/issues/121116
@@ -449,6 +428,9 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=snapshot.create/10_basic/Create a snapshot for missing index}
   issue: https://github.com/elastic/elasticsearch/issues/121536
+- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryUnavailableRemotesIT
+  method: testRemoteOnlyCCSAgainstDisconnectedRemoteWithSkipUnavailableTrue
+  issue: https://github.com/elastic/elasticsearch/issues/121578
 
 # Examples:
 #

rest-api-spec/src/main/resources/rest-api-spec/api/enrich.stats.json

@@ -20,7 +20,7 @@
     "params": {
       "master_timeout":{
         "type":"time",
-        "description":"Timeout for processing on master node"
+        "description":"Timeout for waiting for new cluster state in case it is blocked"
       }
     }
   }

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -172,6 +172,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion TIMEOUT_GET_PARAM_FOR_RESOLVE_CLUSTER = def(8_838_0_00);
     public static final TransportVersion INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING = def(8_839_0_00);
     public static final TransportVersion ML_INFERENCE_IBM_WATSONX_RERANK_ADDED = def(8_840_0_00);
+    public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED_BACKPORT_8_X = def(8_840_0_01);
     public static final TransportVersion ELASTICSEARCH_9_0 = def(9_000_0_00);
     public static final TransportVersion COHERE_BIT_EMBEDDING_TYPE_SUPPORT_ADDED = def(9_001_0_00);
     /*

server/src/main/java/org/elasticsearch/index/mapper/InferenceMetadataFieldsMapper.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.query.SearchExecutionContext;
 
@@ -41,6 +42,10 @@ public abstract class InferenceMetadataFieldsMapper extends MetadataFieldMapper
         Setting.Property.InternalIndex
     );
 
+    // Check index version SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP because that index version was added in the same serverless promotion
+    // where the new format was enabled by default
+    public static final IndexVersion USE_NEW_SEMANTIC_TEXT_FORMAT_BY_DEFAULT = IndexVersions.SOURCE_MAPPER_MODE_ATTRIBUTE_NOOP;
+
     public static final String NAME = "_inference_fields";
     public static final String CONTENT_TYPE = "_inference_fields";
 
@@ -86,10 +91,12 @@ public abstract ValueFetcher valueFetcher(
      */
     public static boolean isEnabled(Settings settings) {
         var version = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings);
-        if (version.before(IndexVersions.INFERENCE_METADATA_FIELDS)
-            && version.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) {
+        if ((version.before(IndexVersions.INFERENCE_METADATA_FIELDS)
+            && version.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false)
+            || (version.before(USE_NEW_SEMANTIC_TEXT_FORMAT_BY_DEFAULT) && USE_LEGACY_SEMANTIC_TEXT_FORMAT.exists(settings) == false)) {
             return false;
         }
+
         return USE_LEGACY_SEMANTIC_TEXT_FORMAT.get(settings) == false;
     }
 

test/external-modules/esql-heap-attack/src/javaRestTest/java/org/elasticsearch/xpack/esql/heap_attack/HeapAttackIT.java

@@ -615,7 +615,6 @@ public void testFetchMvLongs() throws IOException {
         assertMap(map, matchesMap().entry("columns", columns));
     }
 
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106683")
     public void testFetchTooManyMvLongs() throws IOException {
         initMvLongsIndex(500, 100, 1000);
         assertCircuitBreaks(() -> fetchMvLongs());
@@ -636,7 +635,8 @@ public void testLookupExplosion() throws IOException {
 
     public void testLookupExplosionManyMatches() throws IOException {
         assertCircuitBreaks(() -> {
-            Map<?, ?> result = lookupExplosion(1500, 10000);
+            // 1500, 10000 is enough locally, but some CI machines need more.
+            Map<?, ?> result = lookupExplosion(2000, 10000);
             logger.error("should have failed but got {}", result);
         });
     }
@@ -664,7 +664,8 @@ public void testLookupExplosionBigString() throws IOException {
 
     public void testLookupExplosionBigStringManyMatches() throws IOException {
         assertCircuitBreaks(() -> {
-            Map<?, ?> result = lookupExplosionBigString(500, 1);
+            // 500, 1 is enough to make it fail locally but some CI needs more
+            Map<?, ?> result = lookupExplosionBigString(800, 1);
             logger.error("should have failed but got {}", result);
         });
     }

test/framework/src/main/java/org/elasticsearch/test/FailingFieldPlugin.java

@@ -24,12 +24,14 @@
 
 public class FailingFieldPlugin extends Plugin implements ScriptPlugin {
 
+    public static final String FAILING_FIELD_LANG = "failing_field";
+
     @Override
     public ScriptEngine getScriptEngine(Settings settings, Collection<ScriptContext<?>> contexts) {
         return new ScriptEngine() {
             @Override
             public String getType() {
-                return "failing_field";
+                return FAILING_FIELD_LANG;
             }
 
             @Override

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/enrich/action/EnrichStatsAction.java

@@ -10,19 +10,24 @@
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.action.ActionType;
-import org.elasticsearch.action.support.master.MasterNodeRequest;
+import org.elasticsearch.action.support.local.LocalClusterStateRequest;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.UpdateForV10;
+import org.elasticsearch.tasks.CancellableTask;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.tasks.TaskId;
 import org.elasticsearch.tasks.TaskInfo;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 public class EnrichStatsAction extends ActionType<EnrichStatsAction.Response> {
@@ -34,12 +39,17 @@ private EnrichStatsAction() {
         super(NAME);
     }
 
-    public static class Request extends MasterNodeRequest<Request> {
+    public static class Request extends LocalClusterStateRequest {
 
         public Request(TimeValue masterNodeTimeout) {
             super(masterNodeTimeout);
         }
 
+        /**
+         * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to read these requests until
+         * we no longer need to support calling this action remotely.
+         */
+        @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
         public Request(StreamInput in) throws IOException {
             super(in);
         }
@@ -48,6 +58,11 @@ public Request(StreamInput in) throws IOException {
         public ActionRequestValidationException validate() {
             return null;
         }
+
+        @Override
+        public Task createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
+            return new CancellableTask(id, type, action, "", parentTaskId, headers);
+        }
     }
 
     public static class Response extends ActionResponse implements ToXContentObject {
@@ -62,13 +77,6 @@ public Response(List<ExecutingPolicy> executingPolicies, List<CoordinatorStats>
             this.cacheStats = cacheStats;
         }
 
-        public Response(StreamInput in) throws IOException {
-            super(in);
-            executingPolicies = in.readCollectionAsList(ExecutingPolicy::new);
-            coordinatorStats = in.readCollectionAsList(CoordinatorStats::new);
-            cacheStats = in.readCollectionAsList(CacheStats::new);
-        }
-
         public List<ExecutingPolicy> getExecutingPolicies() {
             return executingPolicies;
         }
@@ -81,6 +89,11 @@ public List<CacheStats> getCacheStats() {
             return cacheStats;
         }
 
+        /**
+         * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until
+         * we no longer need to support calling this action remotely.
+         */
+        @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeCollection(executingPolicies);
@@ -167,10 +180,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
 
         public record ExecutingPolicy(String name, TaskInfo taskInfo) implements Writeable, ToXContentFragment {
 
-            ExecutingPolicy(StreamInput in) throws IOException {
-                this(in.readString(), TaskInfo.from(in));
-            }
-
+            /**
+             * NB prior to 9.0 this was a TransportMasterNodeReadAction so for BwC we must remain able to write these responses until
+             * we no longer need to support calling this action remotely.
+             */
+            @UpdateForV10(owner = UpdateForV10.Owner.DATA_MANAGEMENT)
             @Override
             public void writeTo(StreamOutput out) throws IOException {
                 out.writeString(name);