Register test artifacts for service-account security QA project (#94602)

Author: mark-vieira

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/InternalTestArtifactExtension.java

@@ -33,7 +33,7 @@ public void registerTestArtifactFromSourceSet(SourceSet sourceSet) {
         JavaPluginExtension javaPluginExtension = project.getExtensions().getByType(JavaPluginExtension.class);
         javaPluginExtension.registerFeature(name + "Artifacts", featureSpec -> {
             featureSpec.usingSourceSet(sourceSet);
-            featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-test-artifacts", "1.0");
+            featureSpec.capability("org.elasticsearch.gradle", project.getName() + "-" + sourceSet.getName() + "-artifacts", "1.0");
             // This feature is only used internally in the
             // elasticsearch build so we do not need any publication.
             featureSpec.disablePublication();

modules/ingest-geoip/qa/full-cluster-restart/build.gradle

@@ -17,7 +17,7 @@ apply plugin: 'elasticsearch.bwc-test'
 
 dependencies {
   javaRestTestImplementation project(':test:fixtures:geoip-fixture')
-  javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart")))
+  javaRestTestImplementation(testArtifact(project(":qa:full-cluster-restart"), "javaRestTest"))
 }
 
 assert Version.fromString(VersionProperties.getVersions().get("elasticsearch")).getMajor() == 8 :

modules/repository-azure/build.gradle

@@ -12,7 +12,6 @@ import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
  */
 apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
-apply plugin: 'elasticsearch.internal-test-artifact-base'
 
 esplugin {
   description 'The Azure Repository plugin adds support for Azure storage repositories.'
@@ -81,15 +80,6 @@ dependencies {
   testImplementation project(':test:fixtures:azure-fixture')
 }
 
-testArtifacts {
-  registerTestArtifactFromSourceSet(sourceSets.internalClusterTest)
-}
-
-tasks.named("internalClusterTestJar").configure {
-  // for the plugin-security.policy resource
-  from sourceSets.test.output
-}
-
 restResources {
   restApi {
     include '_common', 'cluster', 'nodes', 'snapshot', 'bulk', 'count', 'indices'

modules/repository-gcs/build.gradle

@@ -18,7 +18,6 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  */
 apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
-apply plugin: 'elasticsearch.internal-test-artifact-base'
 
 esplugin {
   description 'The GCS repository plugin adds Google Cloud Storage support for repositories.'
@@ -60,15 +59,6 @@ dependencies {
   testImplementation project(':test:fixtures:gcs-fixture')
 }
 
-testArtifacts {
-  registerTestArtifactFromSourceSet(sourceSets.internalClusterTest)
-}
-
-tasks.named("internalClusterTestJar").configure {
-  // for the plugin-security.policy resource
-  from sourceSets.test.output
-}
-
 restResources {
   restApi {
     include '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'

modules/repository-s3/build.gradle

@@ -15,7 +15,6 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
  */
 apply plugin: 'elasticsearch.legacy-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
-apply plugin: 'elasticsearch.internal-test-artifact-base'
 
 esplugin {
   description 'The S3 repository plugin adds S3 repositories'
@@ -49,15 +48,6 @@ dependencies {
   testImplementation project(':test:fixtures:s3-fixture')
 }
 
-testArtifacts {
-  registerTestArtifactFromSourceSet(sourceSets.internalClusterTest)
-}
-
-tasks.named("internalClusterTestJar").configure {
-  // for the plugin-security.policy resource
-  from sourceSets.test.output
-}
-
 restResources {
   restApi {
     include '_common', 'cluster', 'nodes', 'snapshot','indices', 'index', 'bulk', 'count'

test/test-clusters/build.gradle

@@ -7,6 +7,9 @@ dependencies {
   shadow "junit:junit:${versions.junit}"
   shadow "org.apache.logging.log4j:log4j-api:${versions.log4j}"
 
+  implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+  implementation "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
   implementation "org.elasticsearch.gradle:reaper"
 }
 

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/AbstractLocalSpecBuilder.java

@@ -38,6 +38,7 @@ public abstract class AbstractLocalSpecBuilder<T extends LocalSpecBuilder<?>> im
     private final Map<String, Resource> keystoreFiles = new HashMap<>();
     private final Map<String, Resource> extraConfigFiles = new HashMap<>();
     private final Map<String, String> systemProperties = new HashMap<>();
+    private final Map<String, String> secrets = new HashMap<>();
     private DistributionType distributionType;
     private Version version;
     private String keystorePassword;
@@ -169,6 +170,16 @@ public List<SettingsProvider> getKeystoreProviders() {
         return inherit(() -> parent.getKeystoreProviders(), keystoreProviders);
     }
 
+    @Override
+    public T secret(String key, String value) {
+        this.secrets.put(key, value);
+        return cast(this);
+    }
+
+    public Map<String, String> getSecrets() {
+        return inherit(() -> parent.getSecrets(), secrets);
+    }
+
     @Override
     public T configFile(String fileName, Resource configFile) {
         this.extraConfigFiles.put(fileName, configFile);

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/DefaultLocalClusterSpecBuilder.java

@@ -181,7 +181,8 @@ private LocalNodeSpec build(LocalClusterSpec cluster) {
                 getKeystoreFiles(),
                 getKeystorePassword(),
                 getExtraConfigFiles(),
-                getSystemProperties()
+                getSystemProperties(),
+                getSecrets()
             );
         }
     }

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterFactory.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.test.cluster.local;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.test.cluster.ClusterFactory;
@@ -66,10 +68,12 @@ public class LocalClusterFactory implements ClusterFactory<LocalClusterSpec, Loc
     private static final String ENABLE_DEBUG_JVM_ARGS = "-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=";
     private static final int DEFAULT_DEBUG_PORT = 5007;
 
+    private final ObjectMapper objectMapper;
     private final DistributionResolver distributionResolver;
     private Path baseWorkingDir;
 
     public LocalClusterFactory(DistributionResolver distributionResolver) {
+        this.objectMapper = new ObjectMapper();
         this.distributionResolver = distributionResolver;
     }
 
@@ -136,6 +140,7 @@ public synchronized void start(Version version) {
             createKeystore();
             addKeystoreSettings();
             addKeystoreFiles();
+            writeSecureSecretsFile();
             configureSecurity();
 
             startElasticsearch();
@@ -407,6 +412,21 @@ private void addKeystoreFiles() {
             });
         }
 
+        private void writeSecureSecretsFile() {
+            if (spec.getSecrets().isEmpty() == false) {
+                try {
+                    Path secretsFile = configDir.resolve("secrets/secrets.json");
+                    Files.createDirectories(secretsFile.getParent());
+                    Map<String, Object> secretsFileContent = new HashMap<>();
+                    secretsFileContent.put("secrets", spec.getSecrets());
+                    secretsFileContent.put("metadata", Map.of("version", "1", "compatibility", spec.getVersion().toString()));
+                    Files.writeString(secretsFile, objectMapper.writeValueAsString(secretsFileContent));
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+            }
+        }
+
         private void configureSecurity() {
             if (spec.isSecurityEnabled()) {
                 if (spec.getUsers().isEmpty() == false) {

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/local/LocalClusterSpec.java

@@ -83,6 +83,7 @@ public static class LocalNodeSpec {
         private final String keystorePassword;
         private final Map<String, Resource> extraConfigFiles;
         private final Map<String, String> systemProperties;
+        private final Map<String, String> secrets;
         private Version version;
 
         public LocalNodeSpec(
@@ -102,7 +103,8 @@ public LocalNodeSpec(
             Map<String, Resource> keystoreFiles,
             String keystorePassword,
             Map<String, Resource> extraConfigFiles,
-            Map<String, String> systemProperties
+            Map<String, String> systemProperties,
+            Map<String, String> secrets
         ) {
             this.cluster = cluster;
             this.name = name;
@@ -121,6 +123,7 @@ public LocalNodeSpec(
             this.keystorePassword = keystorePassword;
             this.extraConfigFiles = extraConfigFiles;
             this.systemProperties = systemProperties;
+            this.secrets = secrets;
         }
 
         void setVersion(Version version) {
@@ -179,6 +182,10 @@ public Map<String, String> getSystemProperties() {
             return systemProperties;
         }
 
+        public Map<String, String> getSecrets() {
+            return secrets;
+        }
+
         public boolean isSecurityEnabled() {
             return Boolean.parseBoolean(getSetting("xpack.security.enabled", getVersion().onOrAfter("8.0.0") ? "true" : "false"));
         }
@@ -287,7 +294,8 @@ private LocalNodeSpec getFilteredSpec(SettingsProvider filteredProvider, Setting
                         n.keystoreFiles,
                         n.keystorePassword,
                         n.extraConfigFiles,
-                        n.systemProperties
+                        n.systemProperties,
+                        n.secrets
                     )
                 )
                 .toList();