Merge branch 'main' into failure-store-security-poc

Author: n1v0lg

docs/changelog/120807.yaml

@@ -0,0 +1,5 @@
+pr: 121240
+summary: Implement runtime skip_unavailable=true
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/121240.yaml

@@ -0,0 +1,5 @@
+pr: 121256
+summary: Run `TransportEnrichStatsAction` on local node
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/121256.yaml

@@ -0,0 +1,5 @@
+pr: 121556
+summary: Enable New Semantic Text Format Only On Newly Created Indices
+area: Mapping
+type: bug
+issues: []

docs/changelog/121556.yaml

@@ -52,9 +52,10 @@
  * <p>
  * A bit like Mockito but way more painful.
  */
-public class DummyImplementations {
+class DummyImplementations {
+
+    static class DummyLocaleServiceProvider extends LocaleServiceProvider {
 
-    public static class DummyLocaleServiceProvider extends LocaleServiceProvider {
         @Override
         public Locale[] getAvailableLocales() {
             throw unexpected();

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/DummyImplementations.java

@@ -96,9 +96,6 @@ static CheckAction alwaysDenied(CheckedRunnable<Exception> action) {
 
     private static final Map<String, CheckAction> checkActions = Stream.concat(
         Stream.<Entry<String, CheckAction>>of(
-            entry("static_reflection", deniedToPlugins(RestEntitlementsCheckAction::staticMethodNeverEntitledViaReflection)),
-            entry("nonstatic_reflection", deniedToPlugins(RestEntitlementsCheckAction::nonstaticMethodNeverEntitledViaReflection)),
-            entry("constructor_reflection", deniedToPlugins(RestEntitlementsCheckAction::constructorNeverEntitledViaReflection)),
             entry("runtime_exit", deniedToPlugins(RestEntitlementsCheckAction::runtimeExit)),
             entry("runtime_halt", deniedToPlugins(RestEntitlementsCheckAction::runtimeHalt)),
             entry("system_exit", deniedToPlugins(RestEntitlementsCheckAction::systemExit)),
@@ -341,11 +338,6 @@ private static void systemExit() {
         System.exit(123);
     }
 
-    private static void staticMethodNeverEntitledViaReflection() throws Exception {
-        Method systemExit = System.class.getMethod("exit", int.class);
-        systemExit.invoke(null, 123);
-    }
-
     private static void createClassLoader() throws IOException {
         try (var classLoader = new URLClassLoader("test", new URL[0], RestEntitlementsCheckAction.class.getClassLoader())) {
             logger.info("Created URLClassLoader [{}]", classLoader.getName());
@@ -356,11 +348,6 @@ private static void processBuilder_start() throws IOException {
         new ProcessBuilder("").start();
     }
 
-    private static void nonstaticMethodNeverEntitledViaReflection() throws Exception {
-        Method processBuilderStart = ProcessBuilder.class.getMethod("start");
-        processBuilderStart.invoke(new ProcessBuilder(""));
-    }
-
     private static void processBuilder_startPipeline() throws IOException {
         ProcessBuilder.startPipeline(List.of());
     }
@@ -399,10 +386,6 @@ private static void setHttpsConnectionProperties() {
         new DummyLocaleServiceProvider();
     }
 
-    private static void constructorNeverEntitledViaReflection() throws Exception {
-        DummyLocaleServiceProvider.class.getConstructor().newInstance();
-    }
-
     private static void breakIteratorProvider$() {
         new DummyBreakIteratorProvider();
     }

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/RestEntitlementsCheckAction.java

@@ -122,15 +122,9 @@ tests:
 - class: org.elasticsearch.test.rest.yaml.CcsCommonYamlTestSuiteIT
   method: test {p0=search.highlight/50_synthetic_source/text multi unified from vectors}
   issue: https://github.com/elastic/elasticsearch/issues/117815
-- class: org.elasticsearch.xpack.esql.plugin.ClusterRequestTests
-  method: testFallbackIndicesOptions
-  issue: https://github.com/elastic/elasticsearch/issues/117937
 - class: org.elasticsearch.xpack.ml.integration.RegressionIT
   method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet
   issue: https://github.com/elastic/elasticsearch/issues/117805
-- class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT
-  method: testCancelRequestWhenFailingFetchingPages
-  issue: https://github.com/elastic/elasticsearch/issues/118193
 - class: org.elasticsearch.packaging.test.ArchiveTests
   method: test44AutoConfigurationNotTriggeredOnNotWriteableConfDir
   issue: https://github.com/elastic/elasticsearch/issues/118208
@@ -215,8 +209,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncEnrichStopIT
   method: testEnrichAfterStop
   issue: https://github.com/elastic/elasticsearch/issues/120757
-- class: org.elasticsearch.search.fieldcaps.FieldCapabilitiesIT
-  issue: https://github.com/elastic/elasticsearch/issues/120772
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=ml/3rd_party_deployment/Test start deployment fails while model download in progress}
   issue: https://github.com/elastic/elasticsearch/issues/120810
@@ -384,9 +376,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT
   method: testDependentVariableIsAliasToNested
   issue: https://github.com/elastic/elasticsearch/issues/121415
-- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
-  method: testLookupExplosionBigStringManyMatches
-  issue: https://github.com/elastic/elasticsearch/issues/121465
 - class: org.elasticsearch.xpack.security.authc.jwt.JwtRealmSingleNodeTests
   method: testClientSecretRotation
   issue: https://github.com/elastic/elasticsearch/issues/120985
@@ -396,9 +385,6 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=cluster.health/10_basic/cluster health basic test}
   issue: https://github.com/elastic/elasticsearch/issues/121478
-- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
-  method: testLookupExplosionManyMatches
-  issue: https://github.com/elastic/elasticsearch/issues/121481
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
   method: testGetUsersWithProfileUid
   issue: https://github.com/elastic/elasticsearch/issues/121483
@@ -408,15 +394,6 @@ tests:
 - class: org.elasticsearch.xpack.transform.checkpoint.TransformCCSCanMatchIT
   method: testTransformLifecycle_RangeQueryThatMatchesNoShards
   issue: https://github.com/elastic/elasticsearch/issues/121480
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
-  method: testStopQueryLocal
-  issue: https://github.com/elastic/elasticsearch/issues/121487
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
-  method: testSuccessfulPathways
-  issue: https://github.com/elastic/elasticsearch/issues/121488
-- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryIT
-  method: testAsyncQueriesWithLimit0
-  issue: https://github.com/elastic/elasticsearch/issues/121489
 - class: org.elasticsearch.xpack.security.profile.ProfileIntegTests
   method: testSuggestProfilesWithHint
   issue: https://github.com/elastic/elasticsearch/issues/121116
@@ -449,6 +426,21 @@ tests:
 - class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
   method: test {yaml=snapshot.create/10_basic/Create a snapshot for missing index}
   issue: https://github.com/elastic/elasticsearch/issues/121536
+- class: org.elasticsearch.xpack.esql.action.CrossClusterQueryUnavailableRemotesIT
+  method: testRemoteOnlyCCSAgainstDisconnectedRemoteWithSkipUnavailableTrue
+  issue: https://github.com/elastic/elasticsearch/issues/121578
+- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
+  method: testTasks
+  issue: https://github.com/elastic/elasticsearch/issues/121626
+- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
+  method: testCloseSkipUnavailable
+  issue: https://github.com/elastic/elasticsearch/issues/121627
+- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
+  method: testCancel
+  issue: https://github.com/elastic/elasticsearch/issues/121632
+- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
+  method: testCancelSkipUnavailable
+  issue: https://github.com/elastic/elasticsearch/issues/121631
 
 # Examples:
 #

muted-tests.yml

@@ -18,7 +18,6 @@
 import java.util.List;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 
 public class SourceModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
@@ -83,20 +82,10 @@ public void testConfigureStoredSourceWhenIndexIsCreatedLegacy() throws IOExcepti
     private void assertDeprecationWarningForTemplate(String templateName) throws IOException {
         var request = new Request("GET", "/_migration/deprecations");
         var response = entityAsMap(client().performRequest(request));
-        if (response.containsKey("templates")) {
-            // Check the newer version of the deprecation API that contains the templates section
-            Map<?, ?> issuesByTemplate = (Map<?, ?>) response.get("templates");
-            assertThat(issuesByTemplate.containsKey(templateName), equalTo(true));
-            var templateIssues = (List<?>) issuesByTemplate.get(templateName);
-            assertThat(((Map<?, ?>) templateIssues.getFirst()).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
-        } else {
-            // Bwc version with 8.18 until https://github.com/elastic/elasticsearch/pull/120505/ gets backported, clean up after backport
-            var nodeSettings = (Map<?, ?>) ((List<?>) response.get("node_settings")).getFirst();
-            assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
-            assertThat(
-                (String) nodeSettings.get("details"),
-                containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]")
-            );
-        }
+        assertThat(response.containsKey("templates"), equalTo(true));
+        Map<?, ?> issuesByTemplate = (Map<?, ?>) response.get("templates");
+        assertThat(issuesByTemplate.containsKey(templateName), equalTo(true));
+        var templateIssues = (List<?>) issuesByTemplate.get(templateName);
+        assertThat(((Map<?, ?>) templateIssues.getFirst()).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
     }
 }

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java

@@ -20,7 +20,7 @@
     "params": {
       "master_timeout":{
         "type":"time",
-        "description":"Timeout for processing on master node"
+        "description":"Timeout for waiting for new cluster state in case it is blocked"
       }
     }
   }

rest-api-spec/src/main/resources/rest-api-spec/api/enrich.stats.json

@@ -28,6 +28,7 @@
 import org.elasticsearch.client.Cancellable;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
@@ -73,6 +74,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.CancellationException;
@@ -591,21 +593,31 @@ public void testNoActiveCopy() throws Exception {
 
     private void moveOrCloseShardsOnNodes(String nodeName) throws Exception {
         final IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeName);
+        final ClusterState clusterState = clusterService().state();
         for (IndexService indexService : indicesService) {
             for (IndexShard indexShard : indexService) {
                 if (randomBoolean()) {
                     closeShardNoCheck(indexShard, randomBoolean());
                 } else if (randomBoolean()) {
                     final ShardId shardId = indexShard.shardId();
-
+                    final var assignedNodes = new HashSet<>();
+                    clusterState.routingTable().shardRoutingTable(shardId).allShards().forEach(shr -> {
+                        if (shr.currentNodeId() != null) {
+                            assignedNodes.add(shr.currentNodeId());
+                        }
+                        if (shr.relocatingNodeId() != null) {
+                            assignedNodes.add(shr.relocatingNodeId());
+                        }
+                    });
                     final var targetNodes = new ArrayList<String>();
                     for (final var targetIndicesService : internalCluster().getInstances(IndicesService.class)) {
                         final var targetNode = targetIndicesService.clusterService().localNode();
-                        if (targetNode.canContainData() && targetIndicesService.getShardOrNull(shardId) == null) {
+                        if (targetNode.canContainData()
+                            && targetIndicesService.getShardOrNull(shardId) == null
+                            && assignedNodes.contains(targetNode.getId()) == false) {
                             targetNodes.add(targetNode.getId());
                         }
                     }
-
                     if (targetNodes.isEmpty()) {
                         continue;
                     }