From d93650a44ddfb2824291d630b18cb0d6aa88608f Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Tue, 6 Jan 2026 21:17:27 -0500 Subject: [PATCH 1/5] Configure security in TestClustersPlugin Signed-off-by: Craig Perkins --- .../testclusters/TestClustersPlugin.java | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java index 354c2946889ee..a37ca4f6149d9 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java @@ -35,6 +35,7 @@ import org.opensearch.gradle.JdkDownloadPlugin; import org.opensearch.gradle.ReaperPlugin; import org.opensearch.gradle.ReaperService; +import org.opensearch.gradle.VersionProperties; import org.opensearch.gradle.info.BuildParams; import org.opensearch.gradle.info.GlobalBuildInfoPlugin; import org.opensearch.gradle.internal.InternalDistributionDownloadPlugin; @@ -56,6 +57,11 @@ import javax.inject.Inject; import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.net.URL; +import java.util.Arrays; +import java.util.List; import static org.opensearch.gradle.util.GradleUtils.noop; @@ -111,6 +117,126 @@ public void apply(Project project) { // register cluster hooks project.getRootProject().getPluginManager().apply(TestClustersHookPlugin.class); + + // configure installed plugins from -PinstalledPlugins property + configureInstalledPlugins(project, container); + } + + @SuppressWarnings("unchecked") + private void configureInstalledPlugins(Project project, NamedDomainObjectContainer container) { + String installedPluginsProp = System.getProperty("installedPlugins"); + boolean securityEnabled = "true".equals(System.getProperty("security")) + || "true".equals(System.getProperty("security.enabled")) + || "true".equals(System.getProperty("https")); + + if (installedPluginsProp == null && !securityEnabled) { + return; + } + + project.afterEvaluate(p -> { + container.configureEach(cluster -> { + List plugins = new java.util.ArrayList<>(); + + // Parse explicit plugin list if provided + if (installedPluginsProp != null) { + try { + plugins.addAll((List) groovy.util.Eval.me(installedPluginsProp)); + } catch (Exception e) { + throw new IllegalArgumentException("Invalid installedPlugins format. Expected list like: ['plugin1','plugin2']", e); + } + } + + // Auto-add security plugin if security is enabled and not already in list + if (securityEnabled && plugins.stream().noneMatch(p2 -> p2.contains("opensearch-security"))) { + plugins.add("opensearch-security"); + } + + for (String pluginName : plugins) { + // Check if it's a local project plugin + if (project.findProject(":plugins:" + pluginName) != null) { + cluster.plugin("plugins:" + pluginName); + } else { + // Resolve from Maven + project.getRepositories().mavenLocal(); + project.getRepositories().maven(repo -> { + repo.setName("OpenSearch Snapshots"); + repo.setUrl("https://ci.opensearch.org/ci/dbc/snapshots/maven/"); + }); + + String version = VersionProperties.getOpenSearch().replace("-SNAPSHOT", ".0-SNAPSHOT"); + String coords = pluginName.contains(":") ? pluginName : (pluginName + ":" + version); + if (!coords.contains("@")) { + coords += "@zip"; + } + String fullCoords = coords.split(":").length < 3 ? "org.opensearch.plugin:" + coords : coords; + + org.gradle.api.artifacts.Configuration config = project.getConfigurations() + .detachedConfiguration(project.getDependencies().create(fullCoords)); + config.getResolutionStrategy().cacheChangingModulesFor(0, "seconds"); + cluster.plugin(project.getLayout().file(project.provider(() -> config.getSingleFile()))); + + // Configure security if opensearch-security plugin is installed + if (pluginName.contains("opensearch-security")) { + configureSecurityPlugin(project, cluster); + } + } + } + }); + }); + } + + private static final String[] SECURITY_CERT_FILES = { "esnode.pem", "esnode-key.pem", "kirk.pem", "kirk-key.pem", "root-ca.pem" }; + private static final String SECURITY_CERTS_URL = + "https://raw.githubusercontent.com/opensearch-project/security/refs/heads/main/bwc-test/src/test/resources/security/"; + + private void configureSecurityPlugin(Project project, OpenSearchCluster cluster) { + File buildDir = project.getBuildDir(); + + // Download certificates + for (String certFile : SECURITY_CERT_FILES) { + File localFile = new File(buildDir, certFile); + if (!localFile.exists()) { + try { + localFile.getParentFile().mkdirs(); + try ( + InputStream in = new URL(SECURITY_CERTS_URL + certFile).openStream(); + FileOutputStream out = new FileOutputStream(localFile) + ) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + out.write(buffer, 0, bytesRead); + } + } + } catch (Exception e) { + throw new RuntimeException("Failed to download security certificate: " + certFile, e); + } + } + } + + // Configure each node with certificates and security settings + cluster.getNodes().forEach(node -> { + // Add certificate files + Arrays.stream(SECURITY_CERT_FILES).forEach(cert -> node.extraConfigFile(cert, new File(buildDir, cert))); + + // Security settings + node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem"); + node.setting("plugins.security.ssl.transport.pemkey_filepath", "esnode-key.pem"); + node.setting("plugins.security.ssl.transport.pemtrustedcas_filepath", "root-ca.pem"); + node.setting("plugins.security.ssl.transport.enforce_hostname_verification", "false"); + node.setting("plugins.security.ssl.http.enabled", "true"); + node.setting("plugins.security.ssl.http.pemcert_filepath", "esnode.pem"); + node.setting("plugins.security.ssl.http.pemkey_filepath", "esnode-key.pem"); + node.setting("plugins.security.ssl.http.pemtrustedcas_filepath", "root-ca.pem"); + node.setting("plugins.security.allow_unsafe_democertificates", "true"); + node.setting("plugins.security.allow_default_init_securityindex", "true"); + node.setting("plugins.security.authcz.admin_dn", "\n - CN=kirk,OU=client,O=client,L=test,C=de"); + node.setting("plugins.security.audit.type", "internal_opensearch"); + node.setting("plugins.security.enable_snapshot_restore_privilege", "true"); + node.setting("plugins.security.check_snapshot_restore_write_privileges", "true"); + node.setting("plugins.security.restapi.roles_enabled", "[\"all_access\", \"security_rest_api_access\"]"); + node.setting("plugins.security.system_indices.enabled", "true"); + }); } private NamedDomainObjectContainer createTestClustersContainerExtension(Project project, ReaperService reaper) { From c9ba34f7e6c336c386ca991efce753050d319aab Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Wed, 7 Jan 2026 11:53:51 -0500 Subject: [PATCH 2/5] Use non-deprecated method Signed-off-by: Craig Perkins --- .../opensearch/gradle/testclusters/TestClustersPlugin.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java index a37ca4f6149d9..3455c109a6d31 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java @@ -59,7 +59,7 @@ import java.io.File; import java.io.FileOutputStream; import java.io.InputStream; -import java.net.URL; +import java.net.URI; import java.util.Arrays; import java.util.List; @@ -199,7 +199,7 @@ private void configureSecurityPlugin(Project project, OpenSearchCluster cluster) try { localFile.getParentFile().mkdirs(); try ( - InputStream in = new URL(SECURITY_CERTS_URL + certFile).openStream(); + InputStream in = URI.create(SECURITY_CERTS_URL + certFile).toURL().openStream(); FileOutputStream out = new FileOutputStream(localFile) ) { byte[] buffer = new byte[8192]; From 766a6d39121952abad320a9eaa93124bb133dbba Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 8 Jan 2026 14:46:38 -0500 Subject: [PATCH 3/5] Allow files to be read on the test classpath Signed-off-by: Craig Perkins --- .../testclusters/TestClustersPlugin.java | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java index 3455c109a6d31..361f31f16ba1f 100644 --- a/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java +++ b/buildSrc/src/main/java/org/opensearch/gradle/testclusters/TestClustersPlugin.java @@ -53,6 +53,7 @@ import org.gradle.api.logging.Logging; import org.gradle.api.provider.Provider; import org.gradle.api.tasks.TaskState; +import org.gradle.api.tasks.testing.Test; import javax.inject.Inject; @@ -125,9 +126,7 @@ public void apply(Project project) { @SuppressWarnings("unchecked") private void configureInstalledPlugins(Project project, NamedDomainObjectContainer container) { String installedPluginsProp = System.getProperty("installedPlugins"); - boolean securityEnabled = "true".equals(System.getProperty("security")) - || "true".equals(System.getProperty("security.enabled")) - || "true".equals(System.getProperty("https")); + boolean securityEnabled = "true".equals(System.getProperty("security.enabled")); if (installedPluginsProp == null && !securityEnabled) { return; @@ -185,16 +184,24 @@ private void configureInstalledPlugins(Project project, NamedDomainObjectContain }); } - private static final String[] SECURITY_CERT_FILES = { "esnode.pem", "esnode-key.pem", "kirk.pem", "kirk-key.pem", "root-ca.pem" }; + private static final String[] SECURITY_CERT_FILES = { + "esnode.pem", + "esnode-key.pem", + "kirk.pem", + "kirk-key.pem", + "root-ca.pem", + "sample.pem", + "test-kirk.jks" }; private static final String SECURITY_CERTS_URL = "https://raw.githubusercontent.com/opensearch-project/security/refs/heads/main/bwc-test/src/test/resources/security/"; private void configureSecurityPlugin(Project project, OpenSearchCluster cluster) { File buildDir = project.getBuildDir(); + File securityResourcesDir = new File(buildDir, "security-resources"); // Download certificates for (String certFile : SECURITY_CERT_FILES) { - File localFile = new File(buildDir, certFile); + File localFile = new File(securityResourcesDir, certFile); if (!localFile.exists()) { try { localFile.getParentFile().mkdirs(); @@ -214,10 +221,15 @@ private void configureSecurityPlugin(Project project, OpenSearchCluster cluster) } } + // Add security-resources directory to test classpath + project.getTasks().withType(Test.class).configureEach(testTask -> { + testTask.setClasspath(testTask.getClasspath().plus(project.files(securityResourcesDir))); + }); + // Configure each node with certificates and security settings cluster.getNodes().forEach(node -> { // Add certificate files - Arrays.stream(SECURITY_CERT_FILES).forEach(cert -> node.extraConfigFile(cert, new File(buildDir, cert))); + Arrays.stream(SECURITY_CERT_FILES).forEach(cert -> node.extraConfigFile(cert, new File(securityResourcesDir, cert))); // Security settings node.setting("plugins.security.ssl.transport.pemcert_filepath", "esnode.pem"); From bf72ab79660f716e0c5136ef58fbce7838fc1afa Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Thu, 8 Jan 2026 14:52:35 -0500 Subject: [PATCH 4/5] Add to CHANGELOG Signed-off-by: Craig Perkins --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b3e432f12484..c096e6b09fe01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add integ test for simulating node join left event when data node cluster state publication lag because the cluster applier thread being busy ([#19907](https://github.com/opensearch-project/OpenSearch/pull/19907)). - Relax jar hell check when extended plugins share transitive dependencies ([#20103](https://github.com/opensearch-project/OpenSearch/pull/20103)) - Added public getter method in `SourceFieldMapper` to return included field ([#20290](https://github.com/opensearch-project/OpenSearch/pull/20290)) - +- Update TestClustersPlugin to accept system props `installedPlugins` and `security.enabled` for convenience installing plugins for integ tests ([#20393](https://github.com/opensearch-project/OpenSearch/pull/20393)) ### Changed - Handle custom metadata files in subdirectory-store ([#20157](https://github.com/opensearch-project/OpenSearch/pull/20157)) From 625799244c9c9f17a4e76154dc232415f732ed21 Mon Sep 17 00:00:00 2001 From: Craig Perkins Date: Sat, 21 Mar 2026 20:03:29 -0400 Subject: [PATCH 5/5] Remove CHANGELOG entries Signed-off-by: Craig Perkins --- CHANGELOG.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d675aeb16ea8..0da3a49856c71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased 3.x] ### Added -- Add support for forward translog reading ([#20163](https://github.com/opensearch-project/OpenSearch/pull/20163)) -- Added public getter method in `SourceFieldMapper` to return excluded field ([#20205](https://github.com/opensearch-project/OpenSearch/pull/20205)) -- Add integ test for simulating node join left event when data node cluster state publication lag because the cluster applier thread being busy ([#19907](https://github.com/opensearch-project/OpenSearch/pull/19907)). -- Relax jar hell check when extended plugins share transitive dependencies ([#20103](https://github.com/opensearch-project/OpenSearch/pull/20103)) -- Added public getter method in `SourceFieldMapper` to return included field ([#20290](https://github.com/opensearch-project/OpenSearch/pull/20290)) -- Update TestClustersPlugin to accept system props `installedPlugins` and `security.enabled` for convenience installing plugins for integ tests ([#20393](https://github.com/opensearch-project/OpenSearch/pull/20393)) - Add bitmap64 query support ([#20606](https://github.com/opensearch-project/OpenSearch/pull/20606)) - Add ProfilingWrapper interface for plugin access to delegates in profiling decorators ([#20607](https://github.com/opensearch-project/OpenSearch/pull/20607)) - Support expected cluster name with validation in CCS Sniff mode ([#20532](https://github.com/opensearch-project/OpenSearch/pull/20532))