Add more build logic

Author: JVerwolf

build-tools-internal/build.gradle

@@ -236,6 +236,10 @@ gradlePlugin {
       id = 'elasticsearch.validate-transport-versions'
       implementationClass = 'org.elasticsearch.gradle.internal.transport.ValidateTransportVersionsPlugin'
     }
+    aggregateTransportVersionDeclarationsPlugin{
+      id = 'elasticsearch.aggregate-transport-versionDeclarations-plugin'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.AggregateTransportVersionDeclarationsPlugin'
+    }
   }
 }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
 import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
 import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
+import org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.util.GradleUtils;
@@ -36,6 +37,16 @@ public void apply(Project project) {
         project.getPluginManager().apply(JarHellPrecommitPlugin.class);
         project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
         project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
+        // We need to wire this up
+        /*
+        Old way is to set up a config here, then depend on it in the other plugin.
+        The new way is called variant aware artifacts or something
+        Basically attach some attributes to the artifacts
+        Then those attributes are how we are tying this together.
+        Now instead of saying I want this specific config, we now say I want the artifact from this project that has these attributes
+
+         */
+        project.getPluginManager().apply(LocateTransportVersionsPlugin.class);
         boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
         // Clear default dependencies added by public PluginBuildPlugin as we add our
         // own project dependencies for internal builds

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/AggregateTransportVersionDeclarationsPlugin.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+
+import java.util.Map;
+
+public class AggregateTransportVersionDeclarationsPlugin implements Plugin<Project> {
+    @Override
+    public void apply(Project project) {
+        // need to have this task depend on all the tasks with BaseInternalPluginBuildPlugin registered
+        // need to get the output of all those tasks as input to this task
+        System.out.println("Potato: AggregateTransportVersionDeclarationsPlugin");
+        // First thing is to create a configuration (holder for dependency information. Configurations are how the dep graph is modeled).
+        var configuration = project.getConfigurations().create("aggregateTransportVersionDeclarations");
+        var deps = project.getDependencies();
+
+        project.getRootProject().getSubprojects().stream()
+            .filter(p -> p.getParent().getPath().equals(":modules") || p.getParent().getPath().equals(":plugins") || p.getParent().getPath().equals(":x-pack:plugin"))
+            .forEach(p -> {
+                deps.add(configuration
+                        .getName(),
+                    deps.project(Map.of("path", p.getPath(), "configuration", "locateTransportVersionsConfig"))); // adding a dep to the config we created
+            });
+
+        var aggregationTask = project.getTasks()
+            .register("aggregateTransportVersionDeclarations", AggregateTransportVersionDeclarationsTask.class, t -> {
+                t.dependsOn(configuration); // this task can only run after this config is resolved
+                t.getTransportVersionNameDeclarationsFiles().setFrom(configuration);
+                t.getOutputFile().set(project.getLayout().getBuildDirectory()
+                    .file("generated-transport-info/all-transport-version-names.txt"));
+            });
+
+    }
+
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/AggregateTransportVersionDeclarationsTask.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public abstract class AggregateTransportVersionDeclarationsTask extends DefaultTask {
+
+    @InputFiles
+    public abstract ConfigurableFileCollection getTransportVersionNameDeclarationsFiles();
+
+    @OutputFile
+    public abstract RegularFileProperty getOutputFile();
+
+    @TaskAction
+    public void aggregateTransportVersionDeclarations() {
+        final var files = getTransportVersionNameDeclarationsFiles().getFiles();
+        var allTVNames= files.stream().flatMap(file -> {
+            try {
+                return Files.lines(Path.of(file.getPath()));
+            } catch (IOException e) {
+                throw new RuntimeException("Cannot read Transport Versions name declarations file", e);
+            }
+        }).toList();
+
+        File file = new File(getOutputFile().get().getAsFile().getAbsolutePath());
+        try (FileWriter writer = new FileWriter(file)) {
+            for (String tvName : allTVNames) {
+                writer.write(tvName + "\n");
+            }
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java

@@ -10,7 +10,6 @@
 package org.elasticsearch.gradle.internal.transport;
 
 import com.google.common.collect.Streams;
-
 import org.elasticsearch.gradle.Version;
 import org.elasticsearch.gradle.VersionProperties;
 import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionSetData;
@@ -116,7 +115,9 @@ public void generateTransportVersionData() {
             }).toList();
             if (existingIDsForReleaseVersion.isEmpty() == false) {
                 throw new GradleException(
-                    "A transport version could not be created because one already exists for this release:"
+                    "A transport version could not be created because a preexisting one was found for this name & release."
+                        + " This could be due to another pre-existing TV with the same name, or a result of running this"
+                        + " task twice:"
                         + " Release version: "
                         + tvReleaseVersion
                         + " TransportVersion Id: "
@@ -144,6 +145,7 @@ public void generateTransportVersionData() {
         );
     }
 
+    // TODO Do I need to remove the patch when updating the server portion? NO, but probably need some additional checks
     private static int bumpVersionNumber(
         int tvIDToBump,
         ReleaseVersion releaseVersion,
@@ -153,7 +155,7 @@ private static int bumpVersionNumber(
 
         /* The TV format:
          *
-         * M_NNN_S_PP
+         * MM_NNN_S_PP
          *
          * M - The major version of Elasticsearch
          * NNN - The server version part
@@ -166,7 +168,9 @@ private static int bumpVersionNumber(
                 return releaseVersion.major * 1_000_000; // TODO add check that this doesn't cause overflow out of server versions
             } else {
                 // Bump the server version part if not a major bump.
-                return tvIDToBump + 1000; // TODO add check that this doesn't cause overflow out of server versions
+                // TODO add check that this doesn't cause overflow out of server versions
+                // TODO Do we need to assert on the shape of the number? e.g. no patch version.
+                return tvIDToBump + 1000;
             }
         } else {
             // bump the patch version part
@@ -175,8 +179,9 @@ private static int bumpVersionNumber(
     }
 
     /**
+     * TODO update this
      * Accepts a major.minor version string (e.g. "9.0") and returns the LATEST.json file of the
-     * previous release string (e.g. "8.19-LATEST.json").
+     * previous release string (e.g. "8.19").
      */
     private static ReleaseVersion getPriorReleaseVersion(File tvDataDir, ReleaseVersion releaseVersion) {
         assert tvDataDir != null;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java

@@ -38,7 +38,7 @@ public void generateTransportVersionManifest() {
         var manifestFile = getManifestFile();
         System.out.println("Potato: manifest file location: " + manifestFile.getAsFile().get().getAbsolutePath());
         try (FileWriter writer = new FileWriter(manifestFile.getAsFile().get())) {
-            for (var file : dir.listFiles()) {
+            for (var file : dir. listFiles()) {
                 System.out.println("Potato GenerateTransportVersionManifestTask: " + file.getAbsolutePath());
                 var fileName = file.getName();
                 if (fileName.endsWith("-LATEST.json") == false) {

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java

@@ -9,27 +9,32 @@
 
 package org.elasticsearch.gradle.internal.transport;
 
+import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin;
 import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.file.FileCollection;
+import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.tasks.SourceSet;
 
 public class LocateTransportVersionsPlugin implements Plugin<Project> {
     public static final String TRANSPORT_VERSION_NAMES_FILE = "generated-transport-info/transport-version-set-names.txt";
 
     @Override
     public void apply(Project project) {
-        // TODO figure out what the classpath needs to be to be able to scan the server classes
-        // Does this need to be a lib (to limit scanning by making this a jar, to exclude gradle)? Ask Mark
-        // "/Users/john.verwolf/code/elasticsearch/build-tools-internal/build/classes/java/main"
+        var config = project.getConfigurations().create("locateTransportVersionsConfig");
 
         final var checkTransportVersion = project.getTasks().register("locateTransportVersions", LocateTransportVersionsTask.class, t -> {
             SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME);
+            FileCollection dependencyJars = project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME);
             FileCollection compiledPluginClasses = mainSourceSet.getOutput().getClassesDirs();
-            t.getClassDirs().set(compiledPluginClasses);
+            FileCollection clasDirs = dependencyJars.plus(compiledPluginClasses)
+                .minus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME));
+            t.getClassDirs().set(clasDirs);
 
             t.getOutputFile().set(project.getLayout().getBuildDirectory().file(TRANSPORT_VERSION_NAMES_FILE));
         });
+
+        project.getArtifacts().add(config.getName(), checkTransportVersion);
     }
 }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java

@@ -36,22 +36,22 @@
 import java.util.Set;
 
 /**
- * This task locates all method invocations of org.elasticsearch.TransportVersionSetData#get(java.lang.String) in the
+ * This task locates all method invocations of org.elasticsearch.TransportVersionSet#get(java.lang.String) in the
  * provided directory, and then records the value of string literals passed as arguments. It then records each
  * String on a newline in the provided output file.
  */
 public abstract class LocateTransportVersionsTask extends DefaultTask {
-    public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersionSetData";
+    public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersionSet";
     public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "get";
 
     /**
-     * The directory to scan for TransportVersionSetData#get invocations.
+     * The directory to scan for TransportVersionSet#get invocations.
      */
     @InputFiles
     public abstract Property<FileCollection> getClassDirs();
 
     /**
-     * The output file, with each newline containing the string literal argument of each TransportVersionSetData#get
+     * The output file, with each newline containing the string literal argument of each TransportVersionSet#get
      * invocation.
      */
     @OutputFile
@@ -94,7 +94,7 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri
                                     var abstractInstruction = this.instructions.getLast();
                                     if (abstractInstruction instanceof LdcInsnNode ldcInsnNode) {
                                         if (ldcInsnNode.cst instanceof String tvName && tvName.isEmpty() == false) {
-                                            System.out.println("constant: " + tvName);
+                                            System.out.println("Potato: constant: " + tvName);
                                             results.add(tvName);
                                         } else {
                                             System.out.println(

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java

@@ -12,7 +12,10 @@
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
 import java.io.Serializable;
 import java.nio.file.Path;
 import java.util.List;

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionsTask.java

@@ -30,6 +30,7 @@
  * TODO:
  *  - make this depend on the LocateTransportVersions task/plugin
  *  - make this both a local (per module/plugin) and global task
+ *  - Also validate that there are no duplicate tvs (ids, names?)
  */
 public abstract class ValidateTransportVersionsTask extends DefaultTask {