From 280d89dd1e638ff67f6eb41fa6b8bc6686951e62 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 19 Mar 2025 00:18:42 +1100 Subject: [PATCH 01/10] [Test] Allow add and remove files on Java test cluster For creating and deleting projects in multi-project tests, we need create and delete settings and secrets files on the fly. This PR adds such feature to the Java test cluster. --- .../local/AbstractLocalClusterFactory.java | 26 +++++++++++++++++++ .../local/DefaultLocalClusterHandle.java | 10 +++++++ .../DefaultLocalElasticsearchCluster.java | 10 +++++++ .../cluster/local/LocalClusterHandle.java | 10 +++++++ 4 files changed, 56 insertions(+) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index 680acdccf2f39..f5c9a3e8eb37c 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -286,6 +286,32 @@ public InputStream getLog(LogType logType) { throw new IllegalArgumentException("Log file " + logFile + " does not exist."); } + public void writeAdHocConfigFile(String fileName, String content) { + final Path target = configDir.resolve(fileName); + final Path directory = target.getParent(); + if (Files.exists(directory) == false) { + try { + Files.createDirectories(directory); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + try { + Files.writeString(target, content); + } catch (IOException e) { + throw new UncheckedIOException("Could not write ad-hoc config file: " + fileName, e); + } + } + + public void removeAdHocConfigFile(String fileName) { + final Path target = configDir.resolve(fileName); + try { + Files.deleteIfExists(target); + } catch (IOException e) { + throw new UncheckedIOException("Could not remove ad-hoc config file: " + fileName, e); + } + } + public LocalNodeSpec getSpec() { return spec; } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java index 4331728aedb1e..acc90b9ce2aa5 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java @@ -200,6 +200,16 @@ public InputStream getNodeLog(int index, LogType logType) { return nodes.get(index).getLog(logType); } + @Override + public void writeAhHocConfigFile(int index, String fileName, String content) { + nodes.get(index).writeAdHocConfigFile(fileName, content); + } + + @Override + public void deleteAhHocConfigFile(int index, String fileName) { + nodes.get(index).removeAdHocConfigFile(fileName); + } + @Override public void updateStoredSecureSettings() { execute(() -> nodes.parallelStream().forEach(Node::updateStoredSecureSettings)); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java index fca525a2b4d04..b15cbf287772d 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java @@ -169,6 +169,16 @@ public InputStream getNodeLog(int index, LogType logType) { return handle.getNodeLog(index, logType); } + @Override + public void writeAhHocConfigFile(int index, String fileName, String content) { + handle.writeAhHocConfigFile(index, fileName, content); + } + + @Override + public void deleteAhHocConfigFile(int index, String fileName) { + handle.deleteAhHocConfigFile(index, fileName); + } + @Override public void updateStoredSecureSettings() { checkHandle(); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java index 7c55da3a4e1b9..8dba180df5246 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java @@ -108,6 +108,16 @@ public interface LocalClusterHandle extends ClusterHandle { */ InputStream getNodeLog(int index, LogType logType); + /** + * Write an ad-hoc file in the config directory + */ + void writeAhHocConfigFile(int index, String fileName, String content); + + /** + * Delete an ad-hoc file in the config directory + */ + void deleteAhHocConfigFile(int index, String fileName); + /** * Writes secure settings to the relevant secure config file on each node. Use this method if you are dynamically updating secure * settings via a {@link MutableSettingsProvider} and need the update to be written to file, without a cluster restart. From 7fe0911cdce6356e8b9491f5195e951fb08cfe88 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 19 Mar 2025 00:25:14 +1100 Subject: [PATCH 02/10] typo --- .../test/cluster/local/DefaultLocalClusterHandle.java | 4 ++-- .../cluster/local/DefaultLocalElasticsearchCluster.java | 8 ++++---- .../test/cluster/local/LocalClusterHandle.java | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java index acc90b9ce2aa5..97d90d965cabc 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java @@ -201,12 +201,12 @@ public InputStream getNodeLog(int index, LogType logType) { } @Override - public void writeAhHocConfigFile(int index, String fileName, String content) { + public void writeAdHocConfigFile(int index, String fileName, String content) { nodes.get(index).writeAdHocConfigFile(fileName, content); } @Override - public void deleteAhHocConfigFile(int index, String fileName) { + public void deleteAdHocConfigFile(int index, String fileName) { nodes.get(index).removeAdHocConfigFile(fileName); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java index b15cbf287772d..5f1c24f8e7d80 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java @@ -170,13 +170,13 @@ public InputStream getNodeLog(int index, LogType logType) { } @Override - public void writeAhHocConfigFile(int index, String fileName, String content) { - handle.writeAhHocConfigFile(index, fileName, content); + public void writeAdHocConfigFile(int index, String fileName, String content) { + handle.writeAdHocConfigFile(index, fileName, content); } @Override - public void deleteAhHocConfigFile(int index, String fileName) { - handle.deleteAhHocConfigFile(index, fileName); + public void deleteAdHocConfigFile(int index, String fileName) { + handle.deleteAdHocConfigFile(index, fileName); } @Override diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java index 8dba180df5246..a42590758fd83 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java @@ -111,12 +111,12 @@ public interface LocalClusterHandle extends ClusterHandle { /** * Write an ad-hoc file in the config directory */ - void writeAhHocConfigFile(int index, String fileName, String content); + void writeAdHocConfigFile(int index, String fileName, String content); /** * Delete an ad-hoc file in the config directory */ - void deleteAhHocConfigFile(int index, String fileName); + void deleteAdHocConfigFile(int index, String fileName); /** * Writes secure settings to the relevant secure config file on each node. Use this method if you are dynamically updating secure From 46b28493d3346d93292f9eb6346fdd91f58dc6d4 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 19 Mar 2025 13:49:49 +1100 Subject: [PATCH 03/10] add spec option for config dir function --- .../local/AbstractLocalClusterFactory.java | 18 +++++++++++++----- .../local/AbstractLocalClusterSpecBuilder.java | 3 ++- .../local/AbstractLocalSpecBuilder.java | 13 +++++++++++++ .../local/DefaultLocalClusterHandle.java | 10 +++++----- .../test/cluster/local/LocalClusterSpec.java | 14 ++++++++++++-- .../test/cluster/local/LocalSpecBuilder.java | 8 ++++++++ 6 files changed, 53 insertions(+), 13 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index f5c9a3e8eb37c..1d214b3f5b853 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -146,7 +146,11 @@ public Node( this.repoDir = baseWorkingDir.resolve("repo"); this.dataDir = workingDir.resolve("data"); this.logsDir = workingDir.resolve("logs"); - this.configDir = workingDir.resolve("config"); + Path configDir = null; + if (spec.getConfigDirFunction() != null) { + configDir = spec.getConfigDirFunction().apply(name); + } + this.configDir = configDir == null ? workingDir.resolve("config") : configDir; this.tempDir = workingDir.resolve("tmp"); // elasticsearch temporary directory this.debugPort = DefaultLocalClusterHandle.NEXT_DEBUG_PORT.getAndIncrement(); } @@ -320,6 +324,10 @@ Path getWorkingDir() { return workingDir; } + Path getConfigDir() { + return configDir; + } + public void waitUntilReady() { try { Retry.retryUntilTrue(NODE_UP_TIMEOUT, Duration.ofMillis(500), () -> { @@ -452,7 +460,7 @@ private void writeConfiguration() { try (Stream configFiles = Files.walk(distributionDir.resolve("config"))) { for (Path file : configFiles.toList()) { Path relativePath = distributionDir.resolve("config").relativize(file); - Path dest = configDir.resolve(relativePath); + Path dest = configDir.resolve(relativePath.toFile().getPath()); if (Files.exists(dest) == false) { Files.createDirectories(dest.getParent()); Files.copy(file, dest); @@ -666,7 +674,7 @@ private void configureSecurity() { if (operators.isEmpty() == false) { // TODO: Support service accounts here final String operatorUsersFileName = "operator_users.yml"; - final Path destination = workingDir.resolve("config").resolve(operatorUsersFileName); + final Path destination = configDir.resolve(operatorUsersFileName); if (Files.exists(destination)) { throw new IllegalStateException( "Operator users file [" @@ -693,7 +701,7 @@ private void configureSecurity() { } private void writeRolesFile() { - Path destination = workingDir.resolve("config").resolve("roles.yml"); + Path destination = configDir.resolve("roles.yml"); spec.getRolesFiles().forEach(rolesFile -> { try ( Writer writer = Files.newBufferedWriter(destination, StandardOpenOption.APPEND); @@ -883,7 +891,7 @@ private void startElasticsearch() { private Map getEnvironmentVariables() { Map environment = new HashMap<>(spec.resolveEnvironment()); - environment.put("ES_PATH_CONF", workingDir.resolve("config").toString()); + environment.put("ES_PATH_CONF", configDir.toString()); environment.put("ES_TMPDIR", workingDir.resolve("tmp").toString()); // Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR environment.put("TMP", workingDir.resolve("tmp").toString()); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java index 7f1a384ebb43d..c5ebf0b2ae502 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java @@ -212,7 +212,8 @@ private LocalNodeSpec build(LocalClusterSpec cluster, int nodeIndex) { getExtraConfigFiles(), getSystemPropertyProviders(), getSystemProperties(), - getJvmArgs() + getJvmArgs(), + getConfigDirFunction() ); } } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java index 9617cb633aa9b..a95e89393f293 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java @@ -17,6 +17,7 @@ import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.cluster.util.resource.Resource; +import java.nio.file.Path; import java.util.ArrayList; import java.util.EnumSet; import java.util.HashMap; @@ -25,6 +26,7 @@ import java.util.Map; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -47,6 +49,7 @@ public abstract class AbstractLocalSpecBuilder> im private DistributionType distributionType; private Version version; private String keystorePassword; + private Function configDirFunction; protected AbstractLocalSpecBuilder(AbstractLocalSpecBuilder parent) { this.parent = parent; @@ -270,6 +273,16 @@ public String getKeystorePassword() { return inherit(() -> parent.getKeystorePassword(), keystorePassword); } + @Override + public T withConfigDir(Function configDirFunction) { + this.configDirFunction = configDirFunction; + return cast(this); + } + + public Function getConfigDirFunction() { + return inherit(() -> parent.getConfigDirFunction(), configDirFunction); + } + @Override public T version(Version version) { this.version = version; diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java index 97d90d965cabc..060aff3975ac6 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java @@ -247,15 +247,15 @@ private WaitForHttpResource configureWaitForReady() throws MalformedURLException private void configureWaitSecurity(WaitForHttpResource wait, Node node) { String caFile = node.getSpec().getSetting("xpack.security.http.ssl.certificate_authorities", null); if (caFile != null) { - wait.setCertificateAuthorities(node.getWorkingDir().resolve("config").resolve(caFile).toFile()); + wait.setCertificateAuthorities(node.getConfigDir().resolve(caFile).toFile()); } String sslCertFile = node.getSpec().getSetting("xpack.security.http.ssl.certificate", null); if (sslCertFile != null) { - wait.setCertificateAuthorities(node.getWorkingDir().resolve("config").resolve(sslCertFile).toFile()); + wait.setCertificateAuthorities(node.getConfigDir().resolve(sslCertFile).toFile()); } String sslKeystoreFile = node.getSpec().getSetting("xpack.security.http.ssl.keystore.path", null); if (sslKeystoreFile != null && caFile == null) { // Can not set both trust stores and CA - wait.setTrustStoreFile(node.getWorkingDir().resolve("config").resolve(sslKeystoreFile).toFile()); + wait.setTrustStoreFile(node.getConfigDir().resolve(sslKeystoreFile).toFile()); } String keystorePassword = node.getSpec().getSetting("xpack.security.http.ssl.keystore.secure_password", null); if (keystorePassword != null) { @@ -264,7 +264,7 @@ private void configureWaitSecurity(WaitForHttpResource wait, Node node) { } private boolean isSecurityAutoConfigured(Node node) { - Path configFile = node.getWorkingDir().resolve("config").resolve("elasticsearch.yml"); + Path configFile = node.getConfigDir().resolve("elasticsearch.yml"); try (Stream lines = Files.lines(configFile)) { return lines.anyMatch(l -> l.contains("BEGIN SECURITY AUTO CONFIGURATION")); } catch (IOException e) { @@ -283,7 +283,7 @@ private void writeUnicastHostsFile() { LOGGER.info("Skipping writing unicast hosts file for node {}", node.getName()); return; } - Path hostsFile = node.getWorkingDir().resolve("config").resolve("unicast_hosts.txt"); + Path hostsFile = node.getConfigDir().resolve("unicast_hosts.txt"); LOGGER.info("Writing unicast hosts file {} for node {}", hostsFile, node.getName()); Files.writeString(hostsFile, transportUris); } catch (IOException e) { diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java index ed5c0c5d1bbc0..dd8a2e6524a58 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java @@ -19,11 +19,13 @@ import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.cluster.util.resource.Resource; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; public class LocalClusterSpec implements ClusterSpec { @@ -103,6 +105,7 @@ public static class LocalNodeSpec { private final List systemPropertyProviders; private final Map systemProperties; private final List jvmArgs; + private final Function configDirFunction; private Version version; public LocalNodeSpec( @@ -124,7 +127,8 @@ public LocalNodeSpec( Map extraConfigFiles, List systemPropertyProviders, Map systemProperties, - List jvmArgs + List jvmArgs, + Function configDirFunction ) { this.cluster = cluster; this.name = name; @@ -145,6 +149,7 @@ public LocalNodeSpec( this.systemPropertyProviders = systemPropertyProviders; this.systemProperties = systemProperties; this.jvmArgs = jvmArgs; + this.configDirFunction = configDirFunction; } void setVersion(Version version) { @@ -203,6 +208,10 @@ public List getJvmArgs() { return jvmArgs; } + public Function getConfigDirFunction() { + return configDirFunction; + } + public boolean isSecurityEnabled() { return Boolean.parseBoolean(getSetting("xpack.security.enabled", getVersion().onOrAfter("8.0.0") ? "true" : "false")); } @@ -339,7 +348,8 @@ private LocalNodeSpec getFilteredSpec(SettingsProvider filteredProvider, Setting n.extraConfigFiles, n.systemPropertyProviders, n.systemProperties, - n.jvmArgs + n.jvmArgs, + n.configDirFunction ) ) .toList(); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java index 20c92dc2b11d7..6e56a6ef16310 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java @@ -18,7 +18,9 @@ import org.elasticsearch.test.cluster.util.Version; import org.elasticsearch.test.cluster.util.resource.Resource; +import java.nio.file.Path; import java.util.function.Consumer; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -155,4 +157,10 @@ interface LocalSpecBuilder> { * Adds an additional command line argument to node JVM arguments. */ T jvmArg(String arg); + + /** + * Register a function to compute config directory based on the node name. The default config directory + * is used when the function is null or the return value of the function is null. + */ + T withConfigDir(Function configDirFunction); } From a3bd7a23ff5240d3e267809e7c297e6b944eb8d3 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 19 Mar 2025 13:51:59 +1100 Subject: [PATCH 04/10] revert changes for add/remove config files --- .../test/cluster/local/DefaultLocalClusterHandle.java | 10 ---------- .../local/DefaultLocalElasticsearchCluster.java | 10 ---------- .../test/cluster/local/LocalClusterHandle.java | 10 ---------- 3 files changed, 30 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java index 060aff3975ac6..f2e2245e7321e 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterHandle.java @@ -200,16 +200,6 @@ public InputStream getNodeLog(int index, LogType logType) { return nodes.get(index).getLog(logType); } - @Override - public void writeAdHocConfigFile(int index, String fileName, String content) { - nodes.get(index).writeAdHocConfigFile(fileName, content); - } - - @Override - public void deleteAdHocConfigFile(int index, String fileName) { - nodes.get(index).removeAdHocConfigFile(fileName); - } - @Override public void updateStoredSecureSettings() { execute(() -> nodes.parallelStream().forEach(Node::updateStoredSecureSettings)); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java index 5f1c24f8e7d80..fca525a2b4d04 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalElasticsearchCluster.java @@ -169,16 +169,6 @@ public InputStream getNodeLog(int index, LogType logType) { return handle.getNodeLog(index, logType); } - @Override - public void writeAdHocConfigFile(int index, String fileName, String content) { - handle.writeAdHocConfigFile(index, fileName, content); - } - - @Override - public void deleteAdHocConfigFile(int index, String fileName) { - handle.deleteAdHocConfigFile(index, fileName); - } - @Override public void updateStoredSecureSettings() { checkHandle(); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java index a42590758fd83..7c55da3a4e1b9 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterHandle.java @@ -108,16 +108,6 @@ public interface LocalClusterHandle extends ClusterHandle { */ InputStream getNodeLog(int index, LogType logType); - /** - * Write an ad-hoc file in the config directory - */ - void writeAdHocConfigFile(int index, String fileName, String content); - - /** - * Delete an ad-hoc file in the config directory - */ - void deleteAdHocConfigFile(int index, String fileName); - /** * Writes secure settings to the relevant secure config file on each node. Use this method if you are dynamically updating secure * settings via a {@link MutableSettingsProvider} and need the update to be written to file, without a cluster restart. From 942695e6adb368d0f8d5a87404731f94fb528495 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 19 Mar 2025 13:54:11 +1100 Subject: [PATCH 05/10] more removal --- .../local/AbstractLocalClusterFactory.java | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index 1d214b3f5b853..7c530c0b87b46 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -289,33 +289,7 @@ public InputStream getLog(LogType logType) { throw new IllegalArgumentException("Log file " + logFile + " does not exist."); } - - public void writeAdHocConfigFile(String fileName, String content) { - final Path target = configDir.resolve(fileName); - final Path directory = target.getParent(); - if (Files.exists(directory) == false) { - try { - Files.createDirectories(directory); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - try { - Files.writeString(target, content); - } catch (IOException e) { - throw new UncheckedIOException("Could not write ad-hoc config file: " + fileName, e); - } - } - - public void removeAdHocConfigFile(String fileName) { - final Path target = configDir.resolve(fileName); - try { - Files.deleteIfExists(target); - } catch (IOException e) { - throw new UncheckedIOException("Could not remove ad-hoc config file: " + fileName, e); - } - } - + public LocalNodeSpec getSpec() { return spec; } From df7c6e71be5d21057b64bb94f6ac7a961f940942 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 19 Mar 2025 03:03:10 +0000 Subject: [PATCH 06/10] [CI] Auto commit changes from spotless --- .../test/cluster/local/AbstractLocalClusterFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index 7c530c0b87b46..57bc2b41cec78 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -289,7 +289,7 @@ public InputStream getLog(LogType logType) { throw new IllegalArgumentException("Log file " + logFile + " does not exist."); } - + public LocalNodeSpec getSpec() { return spec; } From 0851c4d961bdaf695ba228f82e204fbfe1ab99e6 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Wed, 19 Mar 2025 17:34:15 +1100 Subject: [PATCH 07/10] change to use raw path --- .../cluster/local/AbstractLocalClusterFactory.java | 6 +----- .../local/AbstractLocalClusterSpecBuilder.java | 2 +- .../cluster/local/AbstractLocalSpecBuilder.java | 11 +++++------ .../test/cluster/local/LocalClusterSpec.java | 13 ++++++------- .../test/cluster/local/LocalSpecBuilder.java | 3 +-- 5 files changed, 14 insertions(+), 21 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index 57bc2b41cec78..115d5b266a27e 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -146,11 +146,7 @@ public Node( this.repoDir = baseWorkingDir.resolve("repo"); this.dataDir = workingDir.resolve("data"); this.logsDir = workingDir.resolve("logs"); - Path configDir = null; - if (spec.getConfigDirFunction() != null) { - configDir = spec.getConfigDirFunction().apply(name); - } - this.configDir = configDir == null ? workingDir.resolve("config") : configDir; + this.configDir = spec.getConfigDir() == null ? workingDir.resolve("config") : spec.getConfigDir(); this.tempDir = workingDir.resolve("tmp"); // elasticsearch temporary directory this.debugPort = DefaultLocalClusterHandle.NEXT_DEBUG_PORT.getAndIncrement(); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java index c5ebf0b2ae502..26db016427d8f 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java @@ -213,7 +213,7 @@ private LocalNodeSpec build(LocalClusterSpec cluster, int nodeIndex) { getSystemPropertyProviders(), getSystemProperties(), getJvmArgs(), - getConfigDirFunction() + getConfigDir() ); } } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java index a95e89393f293..a73a5fa5dda6b 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java @@ -26,7 +26,6 @@ import java.util.Map; import java.util.Set; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -49,7 +48,7 @@ public abstract class AbstractLocalSpecBuilder> im private DistributionType distributionType; private Version version; private String keystorePassword; - private Function configDirFunction; + private Path configDir; protected AbstractLocalSpecBuilder(AbstractLocalSpecBuilder parent) { this.parent = parent; @@ -274,13 +273,13 @@ public String getKeystorePassword() { } @Override - public T withConfigDir(Function configDirFunction) { - this.configDirFunction = configDirFunction; + public T withConfigDir(Path configDir) { + this.configDir = configDir; return cast(this); } - public Function getConfigDirFunction() { - return inherit(() -> parent.getConfigDirFunction(), configDirFunction); + public Path getConfigDir() { + return inherit(() -> parent.getConfigDir(), configDir); } @Override diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java index dd8a2e6524a58..cccf2a95234c9 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; public class LocalClusterSpec implements ClusterSpec { @@ -105,7 +104,7 @@ public static class LocalNodeSpec { private final List systemPropertyProviders; private final Map systemProperties; private final List jvmArgs; - private final Function configDirFunction; + private final Path configDir; private Version version; public LocalNodeSpec( @@ -128,7 +127,7 @@ public LocalNodeSpec( List systemPropertyProviders, Map systemProperties, List jvmArgs, - Function configDirFunction + Path configDir ) { this.cluster = cluster; this.name = name; @@ -149,7 +148,7 @@ public LocalNodeSpec( this.systemPropertyProviders = systemPropertyProviders; this.systemProperties = systemProperties; this.jvmArgs = jvmArgs; - this.configDirFunction = configDirFunction; + this.configDir = configDir; } void setVersion(Version version) { @@ -208,8 +207,8 @@ public List getJvmArgs() { return jvmArgs; } - public Function getConfigDirFunction() { - return configDirFunction; + public Path getConfigDir() { + return configDir; } public boolean isSecurityEnabled() { @@ -349,7 +348,7 @@ private LocalNodeSpec getFilteredSpec(SettingsProvider filteredProvider, Setting n.systemPropertyProviders, n.systemProperties, n.jvmArgs, - n.configDirFunction + n.configDir ) ) .toList(); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java index 6e56a6ef16310..ea49243cfa39f 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java @@ -20,7 +20,6 @@ import java.nio.file.Path; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -162,5 +161,5 @@ interface LocalSpecBuilder> { * Register a function to compute config directory based on the node name. The default config directory * is used when the function is null or the return value of the function is null. */ - T withConfigDir(Function configDirFunction); + T withConfigDir(Path configDir); } From d8bb5cd4b358efa8220b079803d31c6aa9ccdc14 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 20 Mar 2025 09:30:31 +1100 Subject: [PATCH 08/10] replace with supplier --- .../cluster/local/AbstractLocalClusterFactory.java | 6 +++++- .../local/AbstractLocalClusterSpecBuilder.java | 2 +- .../cluster/local/AbstractLocalSpecBuilder.java | 10 +++++----- .../test/cluster/local/LocalClusterSpec.java | 13 +++++++------ .../test/cluster/local/LocalSpecBuilder.java | 6 +++--- 5 files changed, 21 insertions(+), 16 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index 115d5b266a27e..d173caaa56d03 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -146,7 +146,11 @@ public Node( this.repoDir = baseWorkingDir.resolve("repo"); this.dataDir = workingDir.resolve("data"); this.logsDir = workingDir.resolve("logs"); - this.configDir = spec.getConfigDir() == null ? workingDir.resolve("config") : spec.getConfigDir(); + Path configDir = null; + if (spec.getConfigDirSupplier() != null) { + configDir = spec.getConfigDirSupplier().get(); + } + this.configDir = configDir == null ? workingDir.resolve("config") : configDir; this.tempDir = workingDir.resolve("tmp"); // elasticsearch temporary directory this.debugPort = DefaultLocalClusterHandle.NEXT_DEBUG_PORT.getAndIncrement(); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java index 26db016427d8f..e6db4f2eadfa9 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java @@ -213,7 +213,7 @@ private LocalNodeSpec build(LocalClusterSpec cluster, int nodeIndex) { getSystemPropertyProviders(), getSystemProperties(), getJvmArgs(), - getConfigDir() + getConfigDirSupplier() ); } } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java index a73a5fa5dda6b..d8dba4e51ac42 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java @@ -48,7 +48,7 @@ public abstract class AbstractLocalSpecBuilder> im private DistributionType distributionType; private Version version; private String keystorePassword; - private Path configDir; + private Supplier configDirSupplier; protected AbstractLocalSpecBuilder(AbstractLocalSpecBuilder parent) { this.parent = parent; @@ -273,13 +273,13 @@ public String getKeystorePassword() { } @Override - public T withConfigDir(Path configDir) { - this.configDir = configDir; + public T withConfigDir(Supplier configDirSupplier) { + this.configDirSupplier = configDirSupplier; return cast(this); } - public Path getConfigDir() { - return inherit(() -> parent.getConfigDir(), configDir); + public Supplier getConfigDirSupplier() { + return inherit(() -> parent.getConfigDirSupplier(), configDirSupplier); } @Override diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java index cccf2a95234c9..e75c687f5052b 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java @@ -25,6 +25,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; public class LocalClusterSpec implements ClusterSpec { @@ -104,7 +105,7 @@ public static class LocalNodeSpec { private final List systemPropertyProviders; private final Map systemProperties; private final List jvmArgs; - private final Path configDir; + private final Supplier configDirSupplier; private Version version; public LocalNodeSpec( @@ -127,7 +128,7 @@ public LocalNodeSpec( List systemPropertyProviders, Map systemProperties, List jvmArgs, - Path configDir + Supplier configDirSupplier ) { this.cluster = cluster; this.name = name; @@ -148,7 +149,7 @@ public LocalNodeSpec( this.systemPropertyProviders = systemPropertyProviders; this.systemProperties = systemProperties; this.jvmArgs = jvmArgs; - this.configDir = configDir; + this.configDirSupplier = configDirSupplier; } void setVersion(Version version) { @@ -207,8 +208,8 @@ public List getJvmArgs() { return jvmArgs; } - public Path getConfigDir() { - return configDir; + public Supplier getConfigDirSupplier() { + return configDirSupplier; } public boolean isSecurityEnabled() { @@ -348,7 +349,7 @@ private LocalNodeSpec getFilteredSpec(SettingsProvider filteredProvider, Setting n.systemPropertyProviders, n.systemProperties, n.jvmArgs, - n.configDir + n.configDirSupplier ) ) .toList(); diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java index ea49243cfa39f..2f4e64b368589 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalSpecBuilder.java @@ -158,8 +158,8 @@ interface LocalSpecBuilder> { T jvmArg(String arg); /** - * Register a function to compute config directory based on the node name. The default config directory - * is used when the function is null or the return value of the function is null. + * Register a supplier to provide the config directory. The default config directory + * is used when the supplier is null or the return value of the supplier is null. */ - T withConfigDir(Path configDir); + T withConfigDir(Supplier configDirSupplier); } From 0651da01e6d306352c71363e41023450ecf9e69c Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 20 Mar 2025 09:59:32 +1100 Subject: [PATCH 09/10] reduce laziness --- .../cluster/local/AbstractLocalClusterFactory.java | 6 +----- .../local/AbstractLocalClusterSpecBuilder.java | 2 +- .../test/cluster/local/LocalClusterSpec.java | 13 ++++++------- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index d173caaa56d03..115d5b266a27e 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -146,11 +146,7 @@ public Node( this.repoDir = baseWorkingDir.resolve("repo"); this.dataDir = workingDir.resolve("data"); this.logsDir = workingDir.resolve("logs"); - Path configDir = null; - if (spec.getConfigDirSupplier() != null) { - configDir = spec.getConfigDirSupplier().get(); - } - this.configDir = configDir == null ? workingDir.resolve("config") : configDir; + this.configDir = spec.getConfigDir() == null ? workingDir.resolve("config") : spec.getConfigDir(); this.tempDir = workingDir.resolve("tmp"); // elasticsearch temporary directory this.debugPort = DefaultLocalClusterHandle.NEXT_DEBUG_PORT.getAndIncrement(); } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java index e6db4f2eadfa9..73c5afdec5b9f 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterSpecBuilder.java @@ -213,7 +213,7 @@ private LocalNodeSpec build(LocalClusterSpec cluster, int nodeIndex) { getSystemPropertyProviders(), getSystemProperties(), getJvmArgs(), - getConfigDirSupplier() + Optional.ofNullable(getConfigDirSupplier()).map(Supplier::get).orElse(null) ); } } diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java index e75c687f5052b..cccf2a95234c9 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java @@ -25,7 +25,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Supplier; import java.util.stream.Collectors; public class LocalClusterSpec implements ClusterSpec { @@ -105,7 +104,7 @@ public static class LocalNodeSpec { private final List systemPropertyProviders; private final Map systemProperties; private final List jvmArgs; - private final Supplier configDirSupplier; + private final Path configDir; private Version version; public LocalNodeSpec( @@ -128,7 +127,7 @@ public LocalNodeSpec( List systemPropertyProviders, Map systemProperties, List jvmArgs, - Supplier configDirSupplier + Path configDir ) { this.cluster = cluster; this.name = name; @@ -149,7 +148,7 @@ public LocalNodeSpec( this.systemPropertyProviders = systemPropertyProviders; this.systemProperties = systemProperties; this.jvmArgs = jvmArgs; - this.configDirSupplier = configDirSupplier; + this.configDir = configDir; } void setVersion(Version version) { @@ -208,8 +207,8 @@ public List getJvmArgs() { return jvmArgs; } - public Supplier getConfigDirSupplier() { - return configDirSupplier; + public Path getConfigDir() { + return configDir; } public boolean isSecurityEnabled() { @@ -349,7 +348,7 @@ private LocalNodeSpec getFilteredSpec(SettingsProvider filteredProvider, Setting n.systemPropertyProviders, n.systemProperties, n.jvmArgs, - n.configDirSupplier + n.configDir ) ) .toList(); From d6b7838c2611561cc47d435145457523a263aeb3 Mon Sep 17 00:00:00 2001 From: Yang Wang Date: Thu, 20 Mar 2025 10:05:47 +1100 Subject: [PATCH 10/10] tweak --- .../test/cluster/local/AbstractLocalClusterFactory.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java index 115d5b266a27e..2c4b86b0d721b 100644 --- a/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java +++ b/test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalClusterFactory.java @@ -54,6 +54,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.UUID; @@ -146,7 +147,7 @@ public Node( this.repoDir = baseWorkingDir.resolve("repo"); this.dataDir = workingDir.resolve("data"); this.logsDir = workingDir.resolve("logs"); - this.configDir = spec.getConfigDir() == null ? workingDir.resolve("config") : spec.getConfigDir(); + this.configDir = Optional.ofNullable(spec.getConfigDir()).orElse(workingDir.resolve("config")); this.tempDir = workingDir.resolve("tmp"); // elasticsearch temporary directory this.debugPort = DefaultLocalClusterHandle.NEXT_DEBUG_PORT.getAndIncrement(); }