Add support for addition configuration files to test clusters framework (#92579) (#92581)

This adds the ability to supply arbitrary files to the config directory
of cluster nodes. Typically, this is used for security use cases, such
as providing for SSL certificates and trust stores.

This commit adds a few other features to enable more testing ues cases
as well, such as the ability to restart a cluster, as well as explicit
ordering of test cases withing a test class. This is needed for test
suites that need to execute some tests, restart the cluster, then
execute more in a particular order.

Author: mark-vieira

test/framework/src/main/java/org/elasticsearch/test/AnnotationTestOrdering.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.test;
+
+import com.carrotsearch.randomizedtesting.TestMethodAndParams;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Inherited;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+import java.util.Comparator;
+
+/**
+ * Test case ordering to be used in conjunction with {@link com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering}. Tests are
+ * ordered with respect to ordinals defined with {@link Order} annotations placed on individual test methods.
+ */
+public class AnnotationTestOrdering implements Comparator<TestMethodAndParams> {
+    @Override
+    public int compare(TestMethodAndParams o1, TestMethodAndParams o2) {
+        return Integer.compare(
+            o1.getTestMethod().getAnnotation(Order.class).value(),
+            o2.getTestMethod().getAnnotation(Order.class).value()
+        );
+    }
+
+    @Retention(RetentionPolicy.RUNTIME)
+    @Target(ElementType.METHOD)
+    @Inherited
+    public @interface Order {
+        int value();
+    }
+}

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/ClusterHandle.java

@@ -27,6 +27,13 @@ public interface ClusterHandle extends Closeable {
      */
     void stop(boolean forcibly);
 
+    /**
+     * Restarts the cluster. Effectively the same as calling {@link #stop(boolean)} followed by {@link #start()}
+     *
+     * @param forcibly whether to ficibly terminate the cluster
+     */
+    void restart(boolean forcibly);
+
     /**
      * Whether the cluster is started or not. This method makes no guarantees on cluster availability, only that the node processes have
      * been started.

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.SettingsProvider;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.elasticsearch.test.cluster.util.resource.Resource;
 
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -31,6 +32,7 @@ public abstract class AbstractLocalSpecBuilder<T extends LocalSpecBuilder<?>> im
     private final Set<String> plugins = new HashSet<>();
     private final Set<FeatureFlag> features = new HashSet<>();
     private final Map<String, String> keystoreSettings = new HashMap<>();
+    private final Map<String, Resource> extraConfigFiles = new HashMap<>();
     private DistributionType distributionType;
 
     protected AbstractLocalSpecBuilder(AbstractLocalSpecBuilder<?> parent) {
@@ -134,6 +136,16 @@ public Map<String, String> getKeystoreSettings() {
         return inherit(() -> parent.getKeystoreSettings(), keystoreSettings);
     }
 
+    @Override
+    public T configFile(String fileName, Resource configFile) {
+        this.extraConfigFiles.put(fileName, configFile);
+        return cast(this);
+    }
+
+    public Map<String, Resource> getExtraConfigFiles() {
+        return inherit(() -> parent.getExtraConfigFiles(), extraConfigFiles);
+    }
+
     private <T> List<T> inherit(Supplier<List<T>> parent, List<T> child) {
         List<T> combinedList = new ArrayList<>();
         if (this.parent != null) {

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java

@@ -13,7 +13,7 @@
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.local.model.User;
 import org.elasticsearch.test.cluster.util.Version;
-import org.elasticsearch.test.cluster.util.resource.TextResource;
+import org.elasticsearch.test.cluster.util.resource.Resource;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -24,13 +24,13 @@ public class DefaultLocalClusterSpecBuilder extends AbstractLocalSpecBuilder<Loc
     private String name = "test-cluster";
     private final List<DefaultLocalNodeSpecBuilder> nodeBuilders = new ArrayList<>();
     private final List<User> users = new ArrayList<>();
-    private final List<TextResource> roleFiles = new ArrayList<>();
+    private final List<Resource> roleFiles = new ArrayList<>();
 
     public DefaultLocalClusterSpecBuilder() {
         super(null);
         this.settings(new DefaultSettingsProvider());
         this.environment(new DefaultEnvironmentProvider());
-        this.rolesFile(TextResource.fromClasspath("default_test_roles.yml"));
+        this.rolesFile(Resource.fromClasspath("default_test_roles.yml"));
     }
 
     @Override
@@ -95,7 +95,7 @@ public DefaultLocalClusterSpecBuilder user(String username, String password, Str
     }
 
     @Override
-    public DefaultLocalClusterSpecBuilder rolesFile(TextResource rolesFile) {
+    public DefaultLocalClusterSpecBuilder rolesFile(Resource rolesFile) {
         this.roleFiles.add(rolesFile);
         return this;
     }
@@ -146,7 +146,8 @@ private LocalNodeSpec build(LocalClusterSpec cluster) {
                 getPlugins(),
                 Optional.ofNullable(getDistributionType()).orElse(DistributionType.INTEG_TEST),
                 getFeatures(),
-                getKeystoreSettings()
+                getKeystoreSettings(),
+                getExtraConfigFiles()
             );
         }
     }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java

@@ -25,11 +25,15 @@
 import org.elasticsearch.test.cluster.util.Version;
 
 import java.io.BufferedInputStream;
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
 import java.io.UncheckedIOException;
+import java.io.Writer;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -100,17 +104,25 @@ public synchronized void start() {
                 distributionDescriptor = resolveDistribution();
                 LOGGER.info("Distribution for node '{}': {}", spec.getName(), distributionDescriptor);
                 initializeWorkingDirectory();
-                writeConfiguration();
-                createKeystore();
-                addKeystoreSettings();
-                configureSecurity();
                 installPlugins();
                 if (spec.getDistributionType() == DistributionType.INTEG_TEST) {
                     installModules();
                 }
                 initialized = true;
             }
 
+            try {
+                IOUtils.deleteWithRetry(configDir);
+                Files.createDirectories(configDir);
+            } catch (IOException e) {
+                throw new UncheckedIOException("An error occurred creating config directory", e);
+            }
+            writeConfiguration();
+            createKeystore();
+            addKeystoreSettings();
+            configureSecurity();
+            copyExtraConfigFiles();
+
             startElasticsearch();
         }
 
@@ -192,7 +204,6 @@ private void initializeWorkingDirectory() {
                     IOUtils.deleteWithRetry(distributionDir);
                     IOUtils.syncWithCopy(distributionDescriptor.getDistributionDir(), distributionDir);
                 }
-                Files.createDirectories(configDir);
                 Files.createDirectories(snapshotsDir);
                 Files.createDirectories(dataDir);
                 Files.createDirectories(logsDir);
@@ -254,6 +265,10 @@ private void writeConfiguration() {
             }
         }
 
+        private void copyExtraConfigFiles() {
+            spec.getExtraConfigFiles().forEach((fileName, resource) -> resource.writeTo(configDir.resolve(fileName)));
+        }
+
         private void createKeystore() {
             try {
                 ProcessUtils.exec(
@@ -300,13 +315,11 @@ private void configureSecurity() {
 
                     Path destination = workingDir.resolve("config").resolve("roles.yml");
                     spec.getRolesFiles().forEach(rolesFile -> {
-                        try {
-                            Files.writeString(
-                                destination,
-                                rolesFile.getText() + System.lineSeparator(),
-                                StandardCharsets.UTF_8,
-                                StandardOpenOption.APPEND
-                            );
+                        try (
+                            Writer writer = Files.newBufferedWriter(destination, StandardOpenOption.APPEND);
+                            Reader reader = new BufferedReader(new InputStreamReader(rolesFile.asStream()))
+                        ) {
+                            reader.transferTo(writer);
                         } catch (IOException e) {
                             throw new UncheckedIOException("Failed to append roles file " + rolesFile + " to " + destination, e);
                         }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java

@@ -18,6 +18,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
@@ -75,12 +76,19 @@ public void stop(boolean forcibly) {
         if (started.getAndSet(false)) {
             LOGGER.info("Stopping Elasticsearch test cluster '{}', forcibly: {}", name, forcibly);
             execute(() -> nodes.forEach(n -> n.stop(forcibly)));
+            deletePortFiles();
         } else {
             // Make sure the process is stopped, otherwise wait
             execute(() -> nodes.forEach(n -> n.waitForExit()));
         }
     }
 
+    @Override
+    public void restart(boolean forcibly) {
+        stop(forcibly);
+        start();
+    }
+
     @Override
     public boolean isStarted() {
         return started.get();
@@ -124,18 +132,7 @@ private void waitUntilReady() {
         writeUnicastHostsFile();
         try {
             Retry.retryUntilTrue(CLUSTER_UP_TIMEOUT, Duration.ZERO, () -> {
-                Node node = nodes.get(0);
-                boolean securityEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.enabled", "true"));
-                boolean sslEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.http.ssl.enabled", "false"));
-                boolean securityAutoConfigured = isSecurityAutoConfigured(node);
-                String scheme = securityEnabled && (sslEnabled || securityAutoConfigured) ? "https" : "http";
-                WaitForHttpResource wait = new WaitForHttpResource(scheme, node.getHttpAddress(), nodes.size());
-                User credentials = node.getSpec().getUsers().get(0);
-                wait.setUsername(credentials.getUsername());
-                wait.setPassword(credentials.getPassword());
-                if (securityAutoConfigured) {
-                    wait.setCertificateAuthorities(node.getWorkingDir().resolve("config/certs/http_ca.crt").toFile());
-                }
+                WaitForHttpResource wait = configureWaitForReady();
                 return wait.wait(500);
             });
         } catch (TimeoutException e) {
@@ -145,6 +142,23 @@ private void waitUntilReady() {
         }
     }
 
+    private WaitForHttpResource configureWaitForReady() throws MalformedURLException {
+        Node node = nodes.get(0);
+        boolean securityEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.enabled", "true"));
+        boolean sslEnabled = Boolean.parseBoolean(node.getSpec().getSetting("xpack.security.http.ssl.enabled", "false"));
+        boolean securityAutoConfigured = isSecurityAutoConfigured(node);
+        String scheme = securityEnabled && (sslEnabled || securityAutoConfigured) ? "https" : "http";
+        WaitForHttpResource wait = new WaitForHttpResource(scheme, node.getHttpAddress(), nodes.size());
+        User credentials = node.getSpec().getUsers().get(0);
+        wait.setUsername(credentials.getUsername());
+        wait.setPassword(credentials.getPassword());
+        if (securityAutoConfigured) {
+            wait.setCertificateAuthorities(node.getWorkingDir().resolve("config/certs/http_ca.crt").toFile());
+        }
+
+        return wait;
+    }
+
     private boolean isSecurityAutoConfigured(Node node) {
         Path configFile = node.getWorkingDir().resolve("config").resolve("elasticsearch.yml");
         try (Stream<String> lines = Files.lines(configFile)) {
@@ -168,6 +182,22 @@ private void writeUnicastHostsFile() {
         });
     }
 
+    private void deletePortFiles() {
+        nodes.forEach(node -> {
+            try {
+                Path hostsFile = node.getWorkingDir().resolve("config").resolve("unicast_hosts.txt");
+                Path httpPortsFile = node.getWorkingDir().resolve("logs").resolve("http.ports");
+                Path tranportPortsFile = node.getWorkingDir().resolve("logs").resolve("transport.ports");
+
+                Files.deleteIfExists(hostsFile);
+                Files.deleteIfExists(httpPortsFile);
+                Files.deleteIfExists(tranportPortsFile);
+            } catch (IOException e) {
+                throw new UncheckedIOException("Failed to write unicast_hosts for: " + node, e);
+            }
+        });
+    }
+
     private <T> T execute(Callable<T> task) {
         try {
             return executor.submit(task).get();

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java

@@ -15,7 +15,7 @@
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.cluster.local.model.User;
 import org.elasticsearch.test.cluster.util.Version;
-import org.elasticsearch.test.cluster.util.resource.TextResource;
+import org.elasticsearch.test.cluster.util.resource.Resource;
 
 import java.util.HashMap;
 import java.util.List;
@@ -26,10 +26,10 @@
 public class LocalClusterSpec implements ClusterSpec {
     private final String name;
     private final List<User> users;
-    private final List<TextResource> roleFiles;
+    private final List<Resource> roleFiles;
     private List<LocalNodeSpec> nodes;
 
-    public LocalClusterSpec(String name, List<User> users, List<TextResource> roleFiles) {
+    public LocalClusterSpec(String name, List<User> users, List<Resource> roleFiles) {
         this.name = name;
         this.users = users;
         this.roleFiles = roleFiles;
@@ -43,7 +43,7 @@ public List<User> getUsers() {
         return users;
     }
 
-    public List<TextResource> getRoleFiles() {
+    public List<Resource> getRoleFiles() {
         return roleFiles;
     }
 
@@ -79,6 +79,7 @@ public static class LocalNodeSpec {
         private final DistributionType distributionType;
         private final Set<FeatureFlag> features;
         private final Map<String, String> keystoreSettings;
+        private final Map<String, Resource> extraConfigFiles;
 
         public LocalNodeSpec(
             LocalClusterSpec cluster,
@@ -92,7 +93,8 @@ public LocalNodeSpec(
             Set<String> plugins,
             DistributionType distributionType,
             Set<FeatureFlag> features,
-            Map<String, String> keystoreSettings
+            Map<String, String> keystoreSettings,
+            Map<String, Resource> extraConfigFiles
         ) {
             this.cluster = cluster;
             this.name = name;
@@ -106,6 +108,7 @@ public LocalNodeSpec(
             this.distributionType = distributionType;
             this.features = features;
             this.keystoreSettings = keystoreSettings;
+            this.extraConfigFiles = extraConfigFiles;
         }
 
         public LocalClusterSpec getCluster() {
@@ -124,7 +127,7 @@ public List<User> getUsers() {
             return cluster.getUsers();
         }
 
-        public List<TextResource> getRolesFiles() {
+        public List<Resource> getRolesFiles() {
             return cluster.getRoleFiles();
         }
 
@@ -148,6 +151,10 @@ public Map<String, String> getKeystoreSettings() {
             return keystoreSettings;
         }
 
+        public Map<String, Resource> getExtraConfigFiles() {
+            return extraConfigFiles;
+        }
+
         public boolean isSecurityEnabled() {
             return Boolean.parseBoolean(
                 resolveSettings().getOrDefault("xpack.security.enabled", getVersion().onOrAfter("8.0.0") ? "true" : "false")