Merge remote-tracking branch 'elastic/main' into batched-exec-short

Author: original-brownbear

docs/changelog/121843.yaml

@@ -0,0 +1,6 @@
+pr: 121843
+summary: Fix async stop sometimes not properly collecting result
+area: ES|QL
+type: bug
+issues:
+ - 121249

libs/entitlement/bridge/src/main/java/org/elasticsearch/entitlement/bridge/EntitlementChecker.java

@@ -50,6 +50,7 @@
 import java.nio.channels.SocketChannel;
 import java.nio.channels.spi.SelectorProvider;
 import java.nio.charset.Charset;
+import java.nio.file.LinkOption;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.attribute.UserPrincipal;
@@ -514,6 +515,8 @@ public interface EntitlementChecker {
     void check$java_util_Scanner$(Class<?> callerClass, File source, Charset charset);
 
     // nio
+    void check$java_nio_file_Files$$getOwner(Class<?> callerClass, Path path, LinkOption... options);
+
     void check$java_nio_file_Files$$probeContentType(Class<?> callerClass, Path path);
 
     void check$java_nio_file_Files$$setOwner(Class<?> callerClass, Path path, UserPrincipal principal);

libs/entitlement/qa/entitled-plugin/src/main/plugin-metadata/entitlement-policy.yaml

@@ -80,6 +80,11 @@ static void createFileOutputStreamFileWithAppend() throws IOException {
         new FileOutputStream(readWriteFile().toFile(), false).close();
     }
 
+    @EntitlementTest(expectedAccess = PLUGINS)
+    static void filesGetOwner() throws IOException {
+        Files.getOwner(readFile());
+    }
+
     @EntitlementTest(expectedAccess = PLUGINS)
     static void filesProbeContentType() throws IOException {
         Files.probeContentType(readFile());

libs/entitlement/qa/entitlement-test-plugin/src/main/java/org/elasticsearch/entitlement/qa/test/FileCheckActions.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.client.Request;
 import org.elasticsearch.client.Response;
+import org.elasticsearch.entitlement.qa.EntitlementsTestRule.PolicyBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
 
 import java.io.IOException;
@@ -22,7 +23,7 @@
 
 public abstract class AbstractEntitlementsIT extends ESRestTestCase {
 
-    static final EntitlementsTestRule.PolicyBuilder ALLOWED_TEST_ENTITLEMENTS = (builder, tempDir) -> {
+    static final PolicyBuilder ALLOWED_TEST_ENTITLEMENTS = (builder, tempDir) -> {
         builder.value("create_class_loader");
         builder.value("set_https_connection_properties");
         builder.value("inbound_network");

libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/AbstractEntitlementsIT.java

@@ -26,9 +26,27 @@
 import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
 
 class EntitlementsTestRule implements TestRule {
 
+    // entitlements that test methods may use, see EntitledActions
+    private static final PolicyBuilder ENTITLED_POLICY = (builder, tempDir) -> {
+        builder.value(Map.of("write_system_properties", Map.of("properties", List.of("org.elasticsearch.entitlement.qa.selfTest"))));
+        builder.value(
+            Map.of(
+                "files",
+                List.of(
+                    Map.of("path", tempDir.resolve("read_dir"), "mode", "read"),
+                    Map.of("path", tempDir.resolve("read_write_dir"), "mode", "read_write"),
+                    Map.of("path", tempDir.resolve("read_file"), "mode", "read"),
+                    Map.of("path", tempDir.resolve("read_write_file"), "mode", "read_write")
+                )
+            )
+        );
+    };
+
     interface PolicyBuilder {
         void build(XContentBuilder builder, Path tempDir) throws IOException;
     }
@@ -51,7 +69,7 @@ protected void before() throws Throwable {
             }
         };
         cluster = ElasticsearchCluster.local()
-            .module("entitled")
+            .module("entitled", spec -> buildEntitlements(spec, "org.elasticsearch.entitlement.qa.entitled", ENTITLED_POLICY))
             .module("entitlement-test-plugin", spec -> setupEntitlements(spec, modular, policyBuilder))
             .systemProperty("es.entitlements.enabled", "true")
             .systemProperty("es.entitlements.testdir", () -> testDir.getRoot().getAbsolutePath())
@@ -65,29 +83,30 @@ public Statement apply(Statement statement, Description description) {
         return ruleChain.apply(statement, description);
     }
 
-    private void setupEntitlements(PluginInstallSpec spec, boolean modular, PolicyBuilder policyBuilder) {
-        String moduleName = modular ? "org.elasticsearch.entitlement.qa.test" : "ALL-UNNAMED";
-        if (policyBuilder != null) {
-            spec.withEntitlementsOverride(old -> {
-                try {
-                    try (var builder = YamlXContent.contentBuilder()) {
-                        builder.startObject();
-                        builder.field(moduleName);
-                        builder.startArray();
+    private void buildEntitlements(PluginInstallSpec spec, String moduleName, PolicyBuilder policyBuilder) {
+        spec.withEntitlementsOverride(old -> {
+            try (var builder = YamlXContent.contentBuilder()) {
+                builder.startObject();
+                builder.field(moduleName);
+                builder.startArray();
 
-                        policyBuilder.build(builder, testDir.getRoot().toPath());
-                        builder.endArray();
-                        builder.endObject();
+                policyBuilder.build(builder, testDir.getRoot().toPath());
+                builder.endArray();
+                builder.endObject();
 
-                        String policy = Strings.toString(builder);
-                        System.out.println("Using entitlement policy:\n" + policy);
-                        return Resource.fromString(policy);
-                    }
+                String policy = Strings.toString(builder);
+                System.out.println("Using entitlement policy for module " + moduleName + ":\n" + policy);
+                return Resource.fromString(policy);
+            } catch (IOException e) {
+                throw new UncheckedIOException(e);
+            }
+        });
+    }
 
-                } catch (IOException e) {
-                    throw new UncheckedIOException(e);
-                }
-            });
+    private void setupEntitlements(PluginInstallSpec spec, boolean modular, PolicyBuilder policyBuilder) {
+        String moduleName = modular ? "org.elasticsearch.entitlement.qa.test" : "ALL-UNNAMED";
+        if (policyBuilder != null) {
+            buildEntitlements(spec, moduleName, policyBuilder);
         }
 
         if (modular == false) {

libs/entitlement/qa/src/javaRestTest/java/org/elasticsearch/entitlement/qa/EntitlementsTestRule.java

@@ -55,6 +55,7 @@
 import java.nio.channels.SocketChannel;
 import java.nio.channels.spi.SelectorProvider;
 import java.nio.charset.Charset;
+import java.nio.file.LinkOption;
 import java.nio.file.OpenOption;
 import java.nio.file.Path;
 import java.nio.file.attribute.UserPrincipal;
@@ -976,6 +977,11 @@ public void checkSelectorProviderInheritedChannel(Class<?> callerClass, Selector
 
     // nio
 
+    @Override
+    public void check$java_nio_file_Files$$getOwner(Class<?> callerClass, Path path, LinkOption... options) {
+        policyManager.checkFileRead(callerClass, path);
+    }
+
     @Override
     public void check$java_nio_file_Files$$probeContentType(Class<?> callerClass, Path path) {
         policyManager.checkFileRead(callerClass, path);

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/api/ElasticsearchEntitlementChecker.java

@@ -137,8 +137,6 @@ tests:
 - class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests
   method: testSearcherId
   issue: https://github.com/elastic/elasticsearch/issues/118374
-- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
-  issue: https://github.com/elastic/elasticsearch/issues/118238
 - class: org.elasticsearch.xpack.ccr.rest.ShardChangesRestIT
   method: testShardChangesNoOperation
   issue: https://github.com/elastic/elasticsearch/issues/118800
@@ -273,9 +271,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT
   method: testWithDatastreams
   issue: https://github.com/elastic/elasticsearch/issues/121236
-- class: org.elasticsearch.xpack.remotecluster.RemoteClusterSecurityEsqlIT
-  method: testCrossClusterAsyncQueryStop
-  issue: https://github.com/elastic/elasticsearch/issues/121249
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/*}
   issue: https://github.com/elastic/elasticsearch/issues/120816
@@ -358,18 +353,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.ClassificationIT
   method: testDependentVariableIsAliasToKeyword
   issue: https://github.com/elastic/elasticsearch/issues/121492
-- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
-  method: testTasks
-  issue: https://github.com/elastic/elasticsearch/issues/121626
-- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
-  method: testCloseSkipUnavailable
-  issue: https://github.com/elastic/elasticsearch/issues/121627
-- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
-  method: testCancel
-  issue: https://github.com/elastic/elasticsearch/issues/121632
-- class: org.elasticsearch.xpack.esql.action.CrossClustersCancellationIT
-  method: testCancelSkipUnavailable
-  issue: https://github.com/elastic/elasticsearch/issues/121631
 - class: org.elasticsearch.search.CrossClusterSearchUnavailableClusterIT
   method: testSearchSkipUnavailable
   issue: https://github.com/elastic/elasticsearch/issues/121497
@@ -384,35 +367,14 @@ tests:
 - class: org.elasticsearch.analysis.common.CommonAnalysisClientYamlTestSuiteIT
   method: test {yaml=analysis-common/40_token_filters/stemmer_override file access}
   issue: https://github.com/elastic/elasticsearch/issues/121625
-- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
-  method: test {yaml=update/100_synthetic_source/stored text}
-  issue: https://github.com/elastic/elasticsearch/issues/121964
-- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
-  method: test {yaml=update/100_synthetic_source/keyword}
-  issue: https://github.com/elastic/elasticsearch/issues/121965
 - class: org.elasticsearch.xpack.esql.plugin.DataNodeRequestSenderTests
   method: testDoNotRetryOnRequestLevelFailure
   issue: https://github.com/elastic/elasticsearch/issues/121966
 - class: org.elasticsearch.xpack.searchablesnapshots.hdfs.SecureHdfsSearchableSnapshotsIT
   issue: https://github.com/elastic/elasticsearch/issues/121967
-- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
-  method: test {yaml=update/100_synthetic_source/stored text}
-  issue: https://github.com/elastic/elasticsearch/issues/121991
-- class: org.elasticsearch.smoketest.SmokeTestMultiNodeClientYamlTestSuiteIT
-  method: test {yaml=update/100_synthetic_source/keyword}
-  issue: https://github.com/elastic/elasticsearch/issues/121992
 - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
   method: testBottomFieldSort
   issue: https://github.com/elastic/elasticsearch/issues/121503
-- class: org.elasticsearch.xpack.logsdb.LogsdbTestSuiteIT
-  method: test {yaml=/60_synthetic_source_recovery/synthetic recovery for synthetic source mode index}
-  issue: https://github.com/elastic/elasticsearch/issues/122026
-- class: org.elasticsearch.xpack.logsdb.LogsdbTestSuiteIT
-  method: test {yaml=/60_synthetic_source_recovery/synthetic recovery for time_series index}
-  issue: https://github.com/elastic/elasticsearch/issues/122027
-- class: org.elasticsearch.xpack.logsdb.LogsdbTestSuiteIT
-  method: test {yaml=/60_synthetic_source_recovery/synthetic recovery for logsdb index}
-  issue: https://github.com/elastic/elasticsearch/issues/122028
 - class: org.elasticsearch.xpack.application.CohereServiceUpgradeIT
   issue: https://github.com/elastic/elasticsearch/issues/121537
 - class: org.elasticsearch.xpack.migrate.action.ReindexDatastreamIndexTransportActionIT
@@ -423,6 +385,29 @@ tests:
 - class: org.elasticsearch.xpack.restart.FullClusterRestartIT
   method: testWatcherWithApiKey {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/122061
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=update/100_synthetic_source/stored text}
+  issue: https://github.com/elastic/elasticsearch/issues/122100
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=update/100_synthetic_source/keyword}
+  issue: https://github.com/elastic/elasticsearch/issues/122101
+- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
+  method: test {yaml=snapshot.delete/10_basic/Delete a snapshot asynchronously}
+  issue: https://github.com/elastic/elasticsearch/issues/122102
+- class: org.elasticsearch.test.rest.ClientYamlTestSuiteIT
+  method: test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}
+  issue: https://github.com/elastic/elasticsearch/issues/122103
+- class: org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT
+  method: test {yaml=snapshot.delete/10_basic/Delete a snapshot asynchronously}
+  issue: https://github.com/elastic/elasticsearch/issues/122104
+- class: org.elasticsearch.datastreams.TSDBPassthroughIndexingIT
+  issue: https://github.com/elastic/elasticsearch/issues/121716
+- class: org.elasticsearch.xpack.ml.integration.ClassificationIT
+  method: testWithOnlyTrainingRowsAndTrainingPercentIsFifty_DependentVariableIsBoolean
+  issue: https://github.com/elastic/elasticsearch/issues/121680
+- class: org.elasticsearch.xpack.esql.action.EsqlActionBreakerIT
+  method: testExtractFields
+  issue: https://github.com/elastic/elasticsearch/issues/122125
 
 # Examples:
 #

muted-tests.yml

@@ -36,6 +36,7 @@ public class SmokeTestMultiNodeClientYamlTestSuiteIT extends ESClientYamlSuiteTe
         .node(0, n -> n.setting("node.roles", "[master,data,ml,remote_cluster_client,transform]"))
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
+        .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE)
         .build();
 
     public SmokeTestMultiNodeClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {

qa/smoke-test-multinode/src/yamlRestTest/java/org/elasticsearch/smoketest/SmokeTestMultiNodeClientYamlTestSuiteIT.java

@@ -36,6 +36,7 @@ public class ClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .module("data-streams")
         .feature(FeatureFlag.TIME_SERIES_MODE)
         .feature(FeatureFlag.SUB_OBJECTS_AUTO_ENABLED)
+        .feature(FeatureFlag.INDEX_RECOVERY_USE_SYNTHETIC_SOURCE)
         .build();
 
     public ClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {