Build Tasks to Find and Verify TransportVersion Definitions

build-tools-internal/build.gradle

@@ -220,6 +220,14 @@ gradlePlugin {
       id = 'elasticsearch.internal-yaml-rest-test'
       implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
     }
+    transportVersionManagementPlugin {
+      id = 'elasticsearch.transport-version-management'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin'
+    }
+    globalTransportVersionManagementPlugin {
+      id = 'elasticsearch.global-transport-version-management'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin'
+    }
   }
 }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
 import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
 import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
+import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.util.GradleUtils;
@@ -36,6 +37,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(JarHellPrecommitPlugin.class);
         project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
         project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
+        project.getPluginManager().apply(TransportVersionManagementPlugin.class);
         boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
         // Clear default dependencies added by public PluginBuildPlugin as we add our
         // own project dependencies for internal builds

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java

@@ -0,0 +1,150 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.InputFiles;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.TaskAction;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.tree.LdcInsnNode;
+import org.objectweb.asm.tree.MethodNode;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.jar.JarInputStream;
+import java.util.zip.ZipEntry;
+
+/**
+ * This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the
+ * provided directory, and then records the value of string literals passed as arguments. It then records each
+ * string on a newline along with path and line number in the provided output file.
+ */
+public abstract class CollectTransportVersionReferencesTask extends DefaultTask {
+    public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion";
+    public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "fromName";
+    public static final String CLASS_EXTENSION = ".class";
+    public static final String MODULE_INFO = "module-info.class";
+
+    /**
+     * The directory to scan for method invocations.
+     */
+    @InputFiles
+    @Classpath
+    public abstract ConfigurableFileCollection getClassPath();
+
+    /**
+     * The output file, with each newline containing the string literal argument of each method
+     * invocation.
+     */
+    @OutputFile
+    public abstract RegularFileProperty getOutputFile();
+
+    @TaskAction
+    public void checkTransportVersion() throws IOException {
+        var results = new HashSet<TransportVersionUtils.TransportVersionReference>();
+
+        for (var cpElement : getClassPath()) {
+            Path file = cpElement.toPath();
+            if (Files.isDirectory(file)) {
+                addNamesFromClassesDirectory(results, file);
+            } else {
+                assert file.getFileName().toString().endsWith(".jar");
+                addNamesFromJar(results, file);
+            }
+        }
+
+        Path outputFile = getOutputFile().get().getAsFile().toPath();
+        Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList()));
+    }
+
+    private void addNamesFromClassesDirectory(Set<TransportVersionUtils.TransportVersionReference> results, Path file) throws IOException {
+        Files.walkFileTree(file, new SimpleFileVisitor<>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                String filename = file.getFileName().toString();
+                if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) {
+                    try (var inputStream = Files.newInputStream(file)) {
+                        addNamesFromClass(results, inputStream, classname(file.toString()));
+                    }
+                }
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    private void addNamesFromJar(Set<TransportVersionUtils.TransportVersionReference> results, Path file) throws IOException {
+        try (var jar = new JarInputStream(Files.newInputStream(file))) {
+            ZipEntry entry;
+            while ((entry = jar.getNextEntry()) != null) {
+                String filename = entry.getName();
+                if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) {
+                    addNamesFromClass(results, jar, classname(entry.toString()));
+                }
+            }
+        }
+    }
+
+    private void addNamesFromClass(Set<TransportVersionUtils.TransportVersionReference> results, InputStream classBytes, String classname)
+        throws IOException {
+        ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
+            @Override
+            public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+                return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) {
+                    int lineNumber = -1;
+
+                    @Override
+                    public void visitLineNumber(int line, Label start) {
+                        lineNumber = line;
+                    }
+
+                    @Override
+                    public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) {
+                        if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) {
+                            var abstractInstruction = this.instructions.getLast();
+                            String location = classname + " line " + lineNumber;
+                            if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
+                                && ldcInsnNode.cst instanceof String tvName
+                                && tvName.isEmpty() == false) {
+                                results.add(new TransportVersionUtils.TransportVersionReference(tvName, location));
+                            } else {
+                                // The instruction is not a LDC with a String constant (or an empty String), which is not allowed.
+                                throw new RuntimeException(
+                                    "TransportVersion.fromName must be called with a non-empty String literal. " + "See " + location + "."
+                                );
+                            }
+                        }
+                        super.visitMethodInsn(opcode, owner, name, descriptor, isInterface);
+                    }
+                };
+            }
+        };
+        ClassReader classReader = new ClassReader(classBytes);
+        classReader.accept(classVisitor, 0);
+    }
+
+    private static String classname(String filename) {
+        return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replace('/', '.');
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.DirectoryProperty;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.TaskAction;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+public abstract class GenerateTransportVersionManifestTask extends DefaultTask {
+    @InputDirectory
+    public abstract DirectoryProperty getDefinitionsDirectory();
+
+    @OutputFile
+    public abstract RegularFileProperty getManifestFile();
+
+    @TaskAction
+    public void generateTransportVersionManifest() throws IOException {
+        Path constantsDir = getDefinitionsDirectory().get().getAsFile().toPath();
+        Path manifestFile = getManifestFile().get().getAsFile().toPath();
+        try (var writer = Files.newBufferedWriter(manifestFile)) {
+            try (var stream = Files.list(constantsDir)) {
+                for (String filename : stream.map(p -> p.getFileName().toString()).toList()) {
+                    writer.write(filename + "\n");
+                }
+            }
+        }
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.Dependency;
+import org.gradle.api.artifacts.dsl.DependencyHandler;
+import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.tasks.Copy;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class GlobalTransportVersionManagementPlugin implements Plugin<Project> {
+
+    @Override
+    public void apply(Project project) {
+
+        DependencyHandler depsHandler = project.getDependencies();
+        List<Dependency> tvDependencies = new ArrayList<>();
+        // TODO: created a named configuration so deps can be added dynamically?
+        for (String baseProjectPath : List.of(":modules", ":plugins", ":x-pack:plugin")) {
+            Project baseProject = project.project(baseProjectPath);
+            for (var pluginProject : baseProject.getSubprojects()) {
+                if (pluginProject.getParent() != baseProject) {
+                    continue; // skip nested projects
+                }
+                tvDependencies.add(depsHandler.project(Map.of("path", pluginProject.getPath())));
+            }
+        }
+        tvDependencies.add(depsHandler.project(Map.of("path", ":server")));
+
+        Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0]));
+        tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute);
+
+        var validateTask = project.getTasks()
+            .register("validateTransportVersionDefinitions", ValidateTransportVersionDefinitionsTask.class, t -> {
+                t.setGroup("Transport Versions");
+                t.setDescription("Validates that all defined TransportVersion constants are used in at least one project");
+                t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project));
+                t.getReferencesFiles().setFrom(tvReferencesConfig);
+            });
+
+        project.getTasks().named("check").configure(t -> t.dependsOn(validateTask));
+
+        var generateManifestTask = project.getTasks()
+            .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> {
+                t.setGroup("Transport Versions");
+                t.setDescription("Generate a manifest resource for all the known transport version constants");
+                t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project));
+                t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt"));
+            });
+        project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> {
+            t.into("transport/constants", c -> c.from(generateManifestTask));
+        });
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.elasticsearch.gradle.util.GradleUtils;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.tasks.SourceSet;
+
+public class TransportVersionManagementPlugin implements Plugin<Project> {
+
+    @Override
+    public void apply(Project project) {
+        String transportVersionReferencesFile = "transport-version/references.txt";
+        var collectTask = project.getTasks()
+            .register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> {
+                t.setGroup("Transport Versions");
+                t.setDescription("Collects all TransportVersion references used throughout the project");
+                SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME);
+                t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath());
+                t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile));
+            });
+
+        Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> {
+            c.setCanBeConsumed(true);
+            c.setCanBeResolved(false);
+            c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute);
+        });
+
+        project.getArtifacts().add(transportVersionsConfig.getName(), collectTask);
+
+        var validateTask = project.getTasks()
+            .register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> {
+                t.setGroup("Transport Versions");
+                t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file");
+                t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project));
+                t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile));
+                t.dependsOn(collectTask);
+
+            });
+
+        project.getTasks().named("check").configure(t -> t.dependsOn(validateTask));
+    }
+}