[wip] New tv logic - generate definitions files

.gitignore

@@ -69,6 +69,7 @@ testfixtures_shared/
 # Generated
 checkstyle_ide.xml
 x-pack/plugin/esql/src/main/generated-src/generated/
+server/src/main/resources/transport/defined/manifest.txt
 
 # JEnv
 .java-version

build-tools-internal/build.gradle

@@ -220,6 +220,14 @@ gradlePlugin {
       id = 'elasticsearch.internal-yaml-rest-test'
       implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin'
     }
+    transportVersionManagementPlugin {
+      id = 'elasticsearch.transport-version-management'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin'
+    }
+    globalTransportVersionManagementPlugin {
+      id = 'elasticsearch.global-transport-version-management'
+      implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin'
+    }
   }
 }
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.gradle.internal.info.BuildParameterExtension;
 import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin;
 import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin;
+import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin;
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin;
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension;
 import org.elasticsearch.gradle.util.GradleUtils;
@@ -36,6 +37,7 @@ public void apply(Project project) {
         project.getPluginManager().apply(JarHellPrecommitPlugin.class);
         project.getPluginManager().apply(ElasticsearchJavaPlugin.class);
         project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class);
+        project.getPluginManager().apply(TransportVersionManagementPlugin.class);
         boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi();
         // Clear default dependencies added by public PluginBuildPlugin as we add our
         // own project dependencies for internal builds

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.ConfigurableFileCollection;
+import org.gradle.api.file.RegularFileProperty;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.Classpath;
+import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.TaskAction;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.tree.LdcInsnNode;
+import org.objectweb.asm.tree.MethodNode;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the
+ * provided directory, and then records the value of string literals passed as arguments. It then records each
+ * string on a newline along with path and line number in the provided output file.
+ */
+@CacheableTask
+public abstract class CollectTransportVersionReferencesTask extends DefaultTask {
+    public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion";
+    public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "fromName";
+    public static final String CLASS_EXTENSION = ".class";
+    public static final String MODULE_INFO = "module-info.class";
+
+    /**
+     * The directory to scan for method invocations.
+     */
+    @Classpath
+    public abstract ConfigurableFileCollection getClassPath();
+
+    /**
+     * The output file, with each newline containing the string literal argument of each method
+     * invocation.
+     */
+    @OutputFile
+    public abstract RegularFileProperty getOutputFile();
+
+    @TaskAction
+    public void checkTransportVersion() throws IOException {
+        var results = new HashSet<TransportVersionUtils.TransportVersionReference>();
+
+        for (var cpElement : getClassPath()) {
+            Path file = cpElement.toPath();
+            if (Files.isDirectory(file)) {
+                addNamesFromClassesDirectory(results, file);
+            }
+        }
+
+        Path outputFile = getOutputFile().get().getAsFile().toPath();
+        Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList()));
+    }
+
+    private void addNamesFromClassesDirectory(Set<TransportVersionUtils.TransportVersionReference> results, Path file) throws IOException {
+        Files.walkFileTree(file, new SimpleFileVisitor<>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                String filename = file.getFileName().toString();
+                if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) {
+                    try (var inputStream = Files.newInputStream(file)) {
+                        addNamesFromClass(results, inputStream, classname(file.toString()));
+                    }
+                }
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+    private void addNamesFromClass(Set<TransportVersionUtils.TransportVersionReference> results, InputStream classBytes, String classname)
+        throws IOException {
+        ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) {
+            @Override
+            public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) {
+                return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) {
+                    int lineNumber = -1;
+
+                    @Override
+                    public void visitLineNumber(int line, Label start) {
+                        lineNumber = line;
+                    }
+
+                    @Override
+                    public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) {
+                        if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) {
+                            var abstractInstruction = this.instructions.getLast();
+                            String location = classname + " line " + lineNumber;
+                            if (abstractInstruction instanceof LdcInsnNode ldcInsnNode
+                                && ldcInsnNode.cst instanceof String tvName
+                                && tvName.isEmpty() == false) {
+                                results.add(new TransportVersionUtils.TransportVersionReference(tvName, location));
+                            } else {
+                                // The instruction is not a LDC with a String constant (or an empty String), which is not allowed.
+                                throw new RuntimeException(
+                                    "TransportVersion.fromName must be called with a non-empty String literal. " + "See " + location + "."
+                                );
+                            }
+                        }
+                        super.visitMethodInsn(opcode, owner, name, descriptor, isInterface);
+                    }
+                };
+            }
+        };
+        ClassReader classReader = new ClassReader(classBytes);
+        classReader.accept(classVisitor, 0);
+    }
+
+    private static String classname(String filename) {
+        return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replaceAll("[/\\\\]", ".");
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java

@@ -0,0 +1,211 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal.transport;
+
+import org.elasticsearch.gradle.VersionProperties;
+import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.MajorMinor;
+import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionLatest;
+import org.gradle.api.DefaultTask;
+import org.gradle.api.file.DirectoryProperty;
+import org.gradle.api.provider.ListProperty;
+import org.gradle.api.provider.Property;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.InputDirectory;
+import org.gradle.api.tasks.Optional;
+import org.gradle.api.tasks.TaskAction;
+import org.gradle.api.tasks.options.Option;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.PATCH;
+import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.SERVER;
+
+/**
+ * This task generates transport version definition files. These files
+ * are runtime resources that TransportVersion loads statically.
+ * They contain a comma separated list of integer ids. Each file is named the same
+ * as the transport version name itself (with the .csv suffix).
+ */
+public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask {
+
+    /**
+     * Specifies the directory in which contains all TransportVersionSet data files.
+     *
+     * @return
+     */
+    @InputDirectory
+    public abstract DirectoryProperty getTransportResourcesDirectory(); // The plugin should always set this, not optional
+
+    // assumption: this task is always run on main, so we can determine the name by diffing with main and looking for new files added in the
+    // definition directory. (not true: once we generate the file, this will no longer hold true if we then need to update it)
+    /**
+     * Used to set the name of the TransportVersionSet for which a data file will be generated.
+     */
+    @Input
+    @Optional
+    @Option(option = "name", description = "TBD")
+    public abstract Property<String> getTransportVersionName(); // The plugin should always set this, not optional
+
+    /**
+     * Used to set the `major.minor` release version for which the specific TransportVersion ID will be generated.
+     * E.g.: "9.2", "8.18", etc.
+     */
+    @Optional
+    @Input
+    @Option(option = "versions", description = "The minor version(s) for which to generate IDs, e.g. --versions=\"9.2,9.1\"")
+    public abstract ListProperty<String> getMinorVersions();
+
+    // @Optional
+    // @Input
+    // public abstract Property<Function<String, IdIncrement>> getIdIncrementSupplier();
+
+    @TaskAction
+    public void generateTransportVersionData() throws IOException {
+        getLogger().lifecycle("Name: " + getTransportVersionName().get());
+        getLogger().lifecycle("Versions: " + getMinorVersions().get());
+        Path resourcesDir = Objects.requireNonNull(getTransportResourcesDirectory().getAsFile().get()).toPath();
+        String name = getTransportVersionName().isPresent() ? getTransportVersionName().get() : findLocalTransportVersionName();
+        Set<MajorMinor> targetMinorVersions = new HashSet<>(
+            getMinorVersions().isPresent()
+                ? getMinorVersions().get().stream().map(MajorMinor::of).collect(Collectors.toSet())
+                : findTargetMinorVersions()
+        );
+
+        List<Integer> ids = new ArrayList<>();
+        for (MajorMinor minorVersion : getKnownMinorVersions(resourcesDir)) {
+            TransportVersionLatest latest = TransportVersionUtils.readLatestFile(resourcesDir, minorVersion);
+            TransportVersionLatest newLatest = null;
+
+            if (name.equals(latest.name())) {
+                if (targetMinorVersions.contains(minorVersion) == false) {
+                    // Regenerate to make this operation idempotent. Need to undo prior updates to the latest files if the list of minor
+                    // versions has changed.
+                }
+            } else {
+                if (targetMinorVersions.contains(minorVersion)) {
+                    // increment
+                    ids.add(incrementTVId(latest.id(), minorVersion));
+                }
+            }
+
+            if (newLatest != null) {
+                TransportVersionUtils.updateLatestFile(resourcesDir, minorVersion.toString(), newLatest.name(), newLatest.id());
+            }
+        }
+
+        /*
+        final String tvName = Objects.requireNonNull(getTransportVersionName().get());
+        List<String> minorVersions = getMinorVersions().get();
+        //        final var idIncrementSupplier = Objects.requireNonNull(getIdIncrementSupplier().get());
+
+        // TODO
+        //  - [x] do we need to also validate that the minorVersions don't contain duplicates here? How do we enforce idempotency if we don't?
+        //  - is there an order we need to apply? ( I don't think so)
+        //  - Do we need to run this iteratively for backport construction, rather than accepting a list like this? (I don't think so)
+        //  - [x] parse args if run alone
+        //  - check that duplicate versions don't come in?
+        //  - Check that we don't have duplicate names (elsewhere, not here)
+        //  - Do we need to allow creating only patch versions?
+        //  - Must also keep data in sync for removal.
+        //      - We could remove any TVs not associated with a version arg. We then either generate or keep any tvs
+        //        for each version arg, and discard the rest
+        //      - How will this work for follow-up backport PRs that will not have all the version labels?
+        //          - The follow up PR somehow needs to know original IDs. Look at git?  Need a new task?
+        //  -
+
+        // Load the tvSetData for the specified name, if it exists
+        final var tvDefinition = getDefinedFile(tvDataDir, tvName);
+        boolean tvDefinitionExists = tvDefinition != null;
+        final List<Integer> preexistingIds = tvDefinitionExists ? Collections.unmodifiableList(tvDefinition.ids()) : List.of();
+
+        List<Integer> ids = new ArrayList<>();
+        for (var forVersion : forMinorVersions.stream().map(MajorMinor::of).toList()) {
+            // Get the latest transport version data for the specified minor version.
+            final int latestTV = getLatestId(tvDataDir, forVersion.toString());
+
+            // Create the new version id
+            // final int newID = idIncrementSupplier.apply(forVersion).bumpTransportVersion(latestTV);
+            final int newID = incrementTVId(latestTV, forVersion);
+
+            // Check that if we already have a TV ID for this minor version
+            Integer preexistingTVId = retrieveValueInRange(
+                getPriorLatestId(tvDataDir, forVersion.toString()),
+                newID, preexistingIds
+            );
+            if (preexistingTVId != null) {
+                ids.add(preexistingTVId);
+                // TODO: Should we log something here?
+            } else {
+                ids.add(newID);
+                // Update the LATEST file.
+                // TODO need to revert the latest files for anything that has been removed.
+                updateLatestFile(tvDataDir, forVersion.toString(), tvName, newID);
+            }
+        }
+
+        writeDefinitionFile(tvDataDir, tvName, ids.stream().sorted(Comparator.reverseOrder()).toList());
+        */
+    }
+
+    private int incrementTVId(int tvID, MajorMinor version) {
+        // We can only run this task on main, so the ElasticsearchVersion will be for main.
+        final var mainVersion = MajorMinor.of(VersionProperties.getElasticsearchVersion());
+        final var isMain = version.equals(mainVersion);
+        if (isMain) {
+            return SERVER.bumpTransportVersion(tvID);
+        } else {
+            return PATCH.bumpTransportVersion(tvID);
+        }
+        // TODO add serverless check
+    }
+
+    private boolean containsValueInRange(int lowerExclusive, int upperInclusive, List<Integer> ids) {
+        for (var id : ids) {
+            if (lowerExclusive < id && id <= upperInclusive) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private Integer retrieveValueInRange(int lowerExclusive, int upperInclusive, List<Integer> ids) {
+        for (var id : ids) {
+            if (lowerExclusive < id && id <= upperInclusive) {
+                return id;
+            }
+        }
+        return null;
+    }
+
+    private List<MajorMinor> getKnownMinorVersions(Path resourcesDir) {
+        // list files under latest
+        return List.of();
+    }
+
+    private String findLocalTransportVersionName() {
+        // check for missing
+        // if none missing, look at git diff against main
+        return "";
+    }
+
+    private List<MajorMinor> findTargetMinorVersions() {
+        // look for env var indicating github PR link from CI
+        // use github api to find current labels, filter down to version labels
+        // map version labels to branches
+        return List.of();
+    }
+}