From 57193ca9746bab8b3db85e75cf219cb744679cb9 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 23 Jul 2025 11:32:57 -0700 Subject: [PATCH 01/28] Add task to scan for TV instantiations --- build-tools-internal/build.gradle | 4 + .../BaseInternalPluginBuildPlugin.java | 2 + .../LocateTransportVersionsPlugin.java | 41 ++++++ .../LocateTransportVersionsTask.java | 131 ++++++++++++++++++ server/build.gradle | 1 + 5 files changed, 179 insertions(+) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index c04ba9b90d5e7..55fc01f31645f 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -220,6 +220,10 @@ gradlePlugin { id = 'elasticsearch.internal-yaml-rest-test' implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin' } + locateTransportVersionsPlugin { + id = 'elasticsearch.locate-transport-versions' + implementationClass = 'org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin' + } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java index b6f4c99e3d0e6..b338985664984 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java @@ -15,6 +15,7 @@ import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin; import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin; +import org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.util.GradleUtils; @@ -36,6 +37,7 @@ public void apply(Project project) { project.getPluginManager().apply(JarHellPrecommitPlugin.class); project.getPluginManager().apply(ElasticsearchJavaPlugin.class); project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class); + project.getPluginManager().apply(LocateTransportVersionsPlugin.class); boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi(); // Clear default dependencies added by public PluginBuildPlugin as we add our // own project dependencies for internal builds diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java new file mode 100644 index 0000000000000..624bf1f24133b --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java @@ -0,0 +1,41 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.file.FileCollection; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.tasks.SourceSet; + +public class LocateTransportVersionsPlugin implements Plugin { + public static String TASK_NAME = "locateTransportVersions"; + public static final String TRANSPORT_VERSION_NAMES_FILE = "generated-transport-info/transport-version-set-names.txt"; + + @Override + public void apply(Project project) { + var config = project.getConfigurations().create("locateTransportVersionsConfig"); + + final var checkTransportVersion = project.getTasks().register(TASK_NAME, LocateTransportVersionsTask.class, t -> { + SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); + FileCollection dependencyJars = project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME); + FileCollection compiledPluginClasses = mainSourceSet.getOutput().getClassesDirs(); + FileCollection clasDirs = dependencyJars.plus(compiledPluginClasses) + .minus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)); + t.getClassDirs().set(clasDirs); + + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(TRANSPORT_VERSION_NAMES_FILE)); + }); + + project.getArtifacts().add(config.getName(), checkTransportVersion); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java new file mode 100644 index 0000000000000..14ff3be3707ae --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java @@ -0,0 +1,131 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.tree.LdcInsnNode; +import org.objectweb.asm.tree.MethodNode; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * This task locates all method invocations of org.elasticsearch.TransportVersion#getName(java.lang.String) in the + * provided directory, and then records the value of string literals passed as arguments. It then records each + * string on a newline in the provided output file. + */ +public abstract class LocateTransportVersionsTask extends DefaultTask { + public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; + public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "getName"; + public static final String CLASS_EXTENSION = ".class"; + public static final String MODULE_INFO = "module-info.class"; + + /** + * The directory to scan for method invocations. + */ + @InputFiles + public abstract Property getClassDirs(); + + /** + * The output file, with each newline containing the string literal argument of each method + * invocation. + */ + @OutputFile + public abstract RegularFileProperty getOutputFile(); + + @TaskAction + public void checkTransportVersion() { + var classFiles = findJavaClassFiles(getClassDirs().get().getFiles()); + var tvNames = getTVDeclarationNames(classFiles); + + File file = getOutputFile().get().getAsFile(); + try (FileWriter writer = new FileWriter(file)) { + for (String tvName : tvNames) { + writer.write(tvName + "\n"); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static Set getTVDeclarationNames(Collection classfiles) { + var results = new HashSet(); + for (File javaFile : classfiles) { + try (InputStream inputStream = new FileInputStream(javaFile)) { + ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) { + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) { + var abstractInstruction = this.instructions.getLast(); + if (abstractInstruction instanceof LdcInsnNode ldcInsnNode + && ldcInsnNode.cst instanceof String tvName + && tvName.isEmpty() == false) { + results.add(tvName); + } else { + // The instruction is not a LDC with a String constant (or an empty String), + // which is not allowed. + throw new RuntimeException( + "Transport Versions must be declared with a constant and non-empty String. " + + "file: " + + javaFile.getPath() + ); + } + } + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + }; + } + }; + ClassReader classReader = new ClassReader(inputStream); + classReader.accept(classVisitor, 0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return results; + } + + private static List findJavaClassFiles(Collection files) { + List classFiles = new ArrayList<>(); + for (File file : files) { + if (file.isDirectory()) { + File[] subFiles = file.listFiles(); + if (subFiles != null) { + classFiles.addAll(findJavaClassFiles(Arrays.asList(subFiles))); + } + } else if (file.getName().endsWith(CLASS_EXTENSION) && file.getName().endsWith(MODULE_INFO) == false) { + classFiles.add(file); + } + } + return classFiles; + } +} diff --git a/server/build.gradle b/server/build.gradle index be2b43745d0b2..6f5233be773c0 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -12,6 +12,7 @@ apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' +apply plugin: 'elasticsearch.locate-transport-versions' publishing { publications { From de27ca378e802c3765342b0b6d1f0f4f5d02f2c4 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 24 Jul 2025 11:26:28 -0700 Subject: [PATCH 02/28] Initial draft --- build-tools-internal/build.gradle | 6 +- .../BaseInternalPluginBuildPlugin.java | 4 +- ... => CollectTransportVersionNamesTask.java} | 4 +- .../LocateTransportVersionsPlugin.java | 41 ------------- ...ransportVersionGlobalManagementPlugin.java | 23 +++++++ .../TransportVersionManagementPlugin.java | 57 +++++++++++++++++ .../transport/TransportVersionUtils.java | 52 ++++++++++++++++ .../ValidateTransportVersionNamesTask.java | 61 +++++++++++++++++++ server/build.gradle | 2 +- 9 files changed, 201 insertions(+), 49 deletions(-) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/{LocateTransportVersionsTask.java => CollectTransportVersionNamesTask.java} (98%) delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 55fc01f31645f..18834ceac4797 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -220,9 +220,9 @@ gradlePlugin { id = 'elasticsearch.internal-yaml-rest-test' implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin' } - locateTransportVersionsPlugin { - id = 'elasticsearch.locate-transport-versions' - implementationClass = 'org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin' + transportVersionNamesPlugin { + id = 'elasticsearch.transport-version-names' + implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionNamesPlugin' } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java index b338985664984..f89eb3131bc7f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java @@ -15,7 +15,7 @@ import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin; import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin; -import org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin; +import org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.util.GradleUtils; @@ -37,7 +37,7 @@ public void apply(Project project) { project.getPluginManager().apply(JarHellPrecommitPlugin.class); project.getPluginManager().apply(ElasticsearchJavaPlugin.class); project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class); - project.getPluginManager().apply(LocateTransportVersionsPlugin.class); + project.getPluginManager().apply(TransportVersionManagementPlugin.class); boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi(); // Clear default dependencies added by public PluginBuildPlugin as we add our // own project dependencies for internal builds diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java similarity index 98% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java index 14ff3be3707ae..71bf8f71a4055 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java @@ -40,9 +40,9 @@ * provided directory, and then records the value of string literals passed as arguments. It then records each * string on a newline in the provided output file. */ -public abstract class LocateTransportVersionsTask extends DefaultTask { +public abstract class CollectTransportVersionNamesTask extends DefaultTask { public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; - public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "getName"; + public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "fromName"; public static final String CLASS_EXTENSION = ".class"; public static final String MODULE_INFO = "module-info.class"; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java deleted file mode 100644 index 624bf1f24133b..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.transport; - -import org.elasticsearch.gradle.dependencies.CompileOnlyResolvePlugin; -import org.elasticsearch.gradle.util.GradleUtils; -import org.gradle.api.Plugin; -import org.gradle.api.Project; -import org.gradle.api.file.FileCollection; -import org.gradle.api.plugins.JavaPlugin; -import org.gradle.api.tasks.SourceSet; - -public class LocateTransportVersionsPlugin implements Plugin { - public static String TASK_NAME = "locateTransportVersions"; - public static final String TRANSPORT_VERSION_NAMES_FILE = "generated-transport-info/transport-version-set-names.txt"; - - @Override - public void apply(Project project) { - var config = project.getConfigurations().create("locateTransportVersionsConfig"); - - final var checkTransportVersion = project.getTasks().register(TASK_NAME, LocateTransportVersionsTask.class, t -> { - SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - FileCollection dependencyJars = project.getConfigurations().getByName(JavaPlugin.COMPILE_CLASSPATH_CONFIGURATION_NAME); - FileCollection compiledPluginClasses = mainSourceSet.getOutput().getClassesDirs(); - FileCollection clasDirs = dependencyJars.plus(compiledPluginClasses) - .minus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)); - t.getClassDirs().set(clasDirs); - - t.getOutputFile().set(project.getLayout().getBuildDirectory().file(TRANSPORT_VERSION_NAMES_FILE)); - }); - - project.getArtifacts().add(config.getName(), checkTransportVersion); - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java new file mode 100644 index 0000000000000..b64eb6dceebb8 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +public class TransportVersionGlobalManagementPlugin implements Plugin { + + + @Override + public void apply(Project project) { + + + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java new file mode 100644 index 0000000000000..527d0816d91f9 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.file.FileCollection; +import org.gradle.api.tasks.SourceSet; + +public class TransportVersionManagementPlugin implements Plugin { + public static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("is-transport-version-names", Boolean.class); + + @Override + public void apply(Project project) { + final var transportVersionsNamesFile = "generated-transport-info/transport-version-set-names.txt"; + final var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Collects all TransportVersion names used throughout the project"); + SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); + FileCollection clasDirs = mainSourceSet.getRuntimeClasspath(); + t.getClassDirs().set(clasDirs); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + }); + + Configuration transportVersionsConfig = project.getConfigurations().create("transportVersions", c -> { + c.setCanBeConsumed(true); + c.setCanBeResolved(false); + c.attributes(a -> a.attribute(TRANSPORT_VERSION_NAMES_ATTRIBUTE, true)); + }); + + project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); + + final var validateTask = project.getTasks().register("validateTransportVersionNames", ValidateTransportVersionNamesTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); + // TODO: how to ensure this is always references server? + var dir = project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/"); + t.getDataFileDirectory().set(dir); + t.getTransportVersionSetNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + // TODO is this correct? Needs to have both global/per-plugin versions and dependencies + t.dependsOn(collectTask); + + }); + + project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java new file mode 100644 index 0000000000000..2aea0c7f3eb22 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import com.google.common.collect.Comparators; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +public class TransportVersionUtils { + + public record TransportVersionSetData(String name, List ids) { + } + + public static TransportVersionSetData readDataFile(File file) { + try (BufferedReader reader = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { + String[] parts = reader.readLine().replaceAll("\\s+", "").split("\\|"); + if (parts.length < 2) { + throw new IllegalStateException("Invalid transport version file format, " + + "must have both name and id(s): [" + file.getAbsolutePath() + "]"); + } + if (reader.readLine() != null) { + throw new IllegalStateException("Invalid transport version file format, " + + "data file must contain only one line: [" + file.getAbsolutePath() + "]"); + } + String name = parts[0]; + var ids = Arrays.stream(parts).skip(1).map(Integer::parseInt).toList(); + var areIdsSorted = Comparators.isInOrder(ids, Comparator.naturalOrder()); + if (areIdsSorted == false) { + throw new IllegalStateException("invalid transport version file format, " + + "ids are not in order: [" + file.getAbsolutePath() + "], "); + } + return new TransportVersionSetData(name, ids); + } catch (IOException ioe) { + throw new UncheckedIOException("cannot parse transport version [" + file.getAbsolutePath() + "]", ioe); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java new file mode 100644 index 0000000000000..75b6bb8e2b564 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/** + * Validates that each transport version declaration has an associated data file. + */ +public abstract class ValidateTransportVersionNamesTask extends DefaultTask { + + @InputDirectory + public abstract RegularFileProperty getDataFileDirectory(); + + @InputFile + public abstract RegularFileProperty getTransportVersionSetNamesFile(); + + @TaskAction + public void validateTransportVersions() throws IOException { + var dataFileDirectory = getDataFileDirectory(); + var tvDataDir = dataFileDirectory.getAsFile().get(); + + Set tvNamesInDataFiles = new HashSet<>(); + for (var tvDataFile : Objects.requireNonNull(tvDataDir.listFiles())) { + var data = TransportVersionUtils.readDataFile(tvDataFile); + tvNamesInDataFiles.add(data.name()); + } + + var tvSetDeclaredNamesFile = getTransportVersionSetNamesFile().get().getAsFile(); + try (var reader = new BufferedReader(new FileReader(tvSetDeclaredNamesFile))) { + reader.lines().forEach(declaredName -> { + if (tvNamesInDataFiles.contains(declaredName) == false) { + throw new RuntimeException( + "TransportVersionSetData.get(\"" + + declaredName + + "\") was used, but lacks a" + + " data file with a corresponding transport version. This can be generated with the task" // todo + ); + } + }); + } + } +} diff --git a/server/build.gradle b/server/build.gradle index 6f5233be773c0..a93a11fbe7b0c 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -12,7 +12,7 @@ apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' -apply plugin: 'elasticsearch.locate-transport-versions' +apply plugin: 'elasticsearch.transport-version-names' publishing { publications { From 68dbac8b9d22beb6f7933fbfc7788aaaa4e5a0b3 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 24 Jul 2025 14:55:21 -0700 Subject: [PATCH 03/28] wip --- build-tools-internal/build.gradle | 10 ++- .../CollectTransportVersionNamesTask.java | 2 +- ...ransportVersionGlobalManagementPlugin.java | 36 +++++++++++ .../TransportVersionManagementPlugin.java | 23 +++---- .../transport/TransportVersionUtils.java | 60 +++++++++--------- ...ValidateTransportVersionConstantsTask.java | 55 +++++++++++++++++ .../ValidateTransportVersionNamesTask.java | 61 ------------------- ...alidateTransportVersionReferencesTask.java | 52 ++++++++++++++++ server/build.gradle | 3 +- 9 files changed, 193 insertions(+), 109 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java delete mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 18834ceac4797..8c81e4ed53e09 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -220,9 +220,13 @@ gradlePlugin { id = 'elasticsearch.internal-yaml-rest-test' implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin' } - transportVersionNamesPlugin { - id = 'elasticsearch.transport-version-names' - implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionNamesPlugin' + transportVersionManagementPlugin { + id = 'elasticsearch.transport-version-management' + implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin' + } + transportVersionGlobalManagementPlugin { + id = 'elasticsearch.transport-version-global-management' + implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionGlobalManagementPlugin' } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java index 71bf8f71a4055..ff744f111b764 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java @@ -36,7 +36,7 @@ import java.util.Set; /** - * This task locates all method invocations of org.elasticsearch.TransportVersion#getName(java.lang.String) in the + * This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the * provided directory, and then records the value of string literals passed as arguments. It then records each * string on a newline in the provided output file. */ diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java index b64eb6dceebb8..b51f46094f3be 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java @@ -11,6 +11,15 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.artifacts.Dependency; +import org.gradle.api.artifacts.dsl.DependencyHandler; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE; +import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; public class TransportVersionGlobalManagementPlugin implements Plugin { @@ -18,6 +27,33 @@ public class TransportVersionGlobalManagementPlugin implements Plugin { @Override public void apply(Project project) { + DependencyHandler depsHandler = project.getDependencies(); + List tvDependencies = new ArrayList<>(); + // TODO: created a named configuration so deps can be added dynamically? + for (String baseProjectPath : List.of(":modules", ":plugins", ":x-pack:plugin")) { + Project baseProject = project.project(baseProjectPath); + for (var pluginProject : baseProject.getSubprojects()) { + if (pluginProject.getParent() != baseProject) { + continue; // skip nested projects + } + tvDependencies.add(depsHandler.create(pluginProject)); + } + } + tvDependencies.add(depsHandler.create(":server")); + + Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); + tvNamesConfig.attributes(attrs -> { + attrs.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv"); + attrs.attribute(TRANSPORT_VERSION_NAMES_ATTRIBUTE, true); + }); + + var validateTask = project.getTasks().register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getNamesFiles().setFrom(tvNamesConfig); + }); + project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 527d0816d91f9..49b0d6d8abd94 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -13,41 +13,34 @@ import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; -import org.gradle.api.attributes.Attribute; -import org.gradle.api.file.FileCollection; import org.gradle.api.tasks.SourceSet; public class TransportVersionManagementPlugin implements Plugin { - public static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("is-transport-version-names", Boolean.class); @Override public void apply(Project project) { - final var transportVersionsNamesFile = "generated-transport-info/transport-version-set-names.txt"; - final var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { + String transportVersionsNamesFile = "generated-transport-constants/transport-version-set-names.txt"; + var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion names used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - FileCollection clasDirs = mainSourceSet.getRuntimeClasspath(); - t.getClassDirs().set(clasDirs); + t.getClassDirs().set(mainSourceSet.getRuntimeClasspath()); t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); }); - Configuration transportVersionsConfig = project.getConfigurations().create("transportVersions", c -> { + Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { c.setCanBeConsumed(true); c.setCanBeResolved(false); - c.attributes(a -> a.attribute(TRANSPORT_VERSION_NAMES_ATTRIBUTE, true)); + c.attributes(a -> a.attribute(TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE, true)); }); project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); - final var validateTask = project.getTasks().register("validateTransportVersionNames", ValidateTransportVersionNamesTask.class, t -> { + var validateTask = project.getTasks().register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); - // TODO: how to ensure this is always references server? - var dir = project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/"); - t.getDataFileDirectory().set(dir); - t.getTransportVersionSetNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); - // TODO is this correct? Needs to have both global/per-plugin versions and dependencies + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); t.dependsOn(collectTask); }); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 2aea0c7f3eb22..29d2600e5ff36 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -11,42 +11,46 @@ import com.google.common.collect.Comparators; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileReader; +import org.gradle.api.Project; +import org.gradle.api.attributes.Attribute; +import org.gradle.api.file.Directory; + import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; -import java.util.Arrays; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; -public class TransportVersionUtils { +class TransportVersionUtils { - public record TransportVersionSetData(String name, List ids) { - } + static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("transport-version-names", Boolean.class); - public static TransportVersionSetData readDataFile(File file) { - try (BufferedReader reader = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))) { - String[] parts = reader.readLine().replaceAll("\\s+", "").split("\\|"); - if (parts.length < 2) { - throw new IllegalStateException("Invalid transport version file format, " - + "must have both name and id(s): [" + file.getAbsolutePath() + "]"); - } - if (reader.readLine() != null) { - throw new IllegalStateException("Invalid transport version file format, " - + "data file must contain only one line: [" + file.getAbsolutePath() + "]"); - } - String name = parts[0]; - var ids = Arrays.stream(parts).skip(1).map(Integer::parseInt).toList(); - var areIdsSorted = Comparators.isInOrder(ids, Comparator.naturalOrder()); - if (areIdsSorted == false) { - throw new IllegalStateException("invalid transport version file format, " - + "ids are not in order: [" + file.getAbsolutePath() + "], "); + record TransportVersionData(String name, List ids) { } + + static TransportVersionData readDataFile(Path file) throws IOException { + assert file.endsWith(".csv"); + String rawName = file.getFileName().toString(); + String name = rawName.substring(0, rawName.length() - 4); + List ids = new ArrayList<>(); + + for (String rawId : Files.readString(file, StandardCharsets.UTF_8).split(",")) { + try { + ids.add(Integer.parseInt(rawId)); + } catch (NumberFormatException e) { + throw new IOException("Failed to parse id " + rawId + " in " + file, e); } - return new TransportVersionSetData(name, ids); - } catch (IOException ioe) { - throw new UncheckedIOException("cannot parse transport version [" + file.getAbsolutePath() + "]", ioe); } + + if (Comparators.isInOrder(ids, Comparator.naturalOrder()) == false) { + throw new IOException("invalid transport version data file [" + file + "], ids are not in sorted"); + } + return new TransportVersionData(name, ids); + } + + static Directory getConstantsDirectory(Project project) { + Directory serverDir = project.getRootProject().project(":server").getLayout().getProjectDirectory(); + return serverDir.dir("src/main/resources/transport/constants"); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java new file mode 100644 index 0000000000000..b0d5aac5c6ef5 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.TaskAction; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; + +/** + * Validates that each defined transport version constant is referenced by at least one project. + */ +public abstract class ValidateTransportVersionConstantsTask extends DefaultTask { + + @InputDirectory + public abstract DirectoryProperty getConstantsDirectory(); + + @InputFiles + public abstract ConfigurableFileCollection getNamesFiles(); + + @TaskAction + public void validateTransportVersions() throws IOException { + Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); + + Set allTvNames = new HashSet<>(); + for (var tvNamesFile : getNamesFiles()) { + allTvNames.addAll(Files.readAllLines(tvNamesFile.toPath(), StandardCharsets.UTF_8)); + } + + try (var constantsStream = Files.list(constantsDir)) { + for (var constantsFile : constantsStream.toList()) { + var tv = TransportVersionUtils.readDataFile(constantsFile); + if (allTvNames.contains(tv.name()) == false) { + throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced"); + } + } + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java deleted file mode 100644 index 75b6bb8e2b564..0000000000000 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionNamesTask.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.gradle.internal.transport; - -import org.gradle.api.DefaultTask; -import org.gradle.api.file.RegularFileProperty; -import org.gradle.api.tasks.InputDirectory; -import org.gradle.api.tasks.InputFile; -import org.gradle.api.tasks.TaskAction; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; - -/** - * Validates that each transport version declaration has an associated data file. - */ -public abstract class ValidateTransportVersionNamesTask extends DefaultTask { - - @InputDirectory - public abstract RegularFileProperty getDataFileDirectory(); - - @InputFile - public abstract RegularFileProperty getTransportVersionSetNamesFile(); - - @TaskAction - public void validateTransportVersions() throws IOException { - var dataFileDirectory = getDataFileDirectory(); - var tvDataDir = dataFileDirectory.getAsFile().get(); - - Set tvNamesInDataFiles = new HashSet<>(); - for (var tvDataFile : Objects.requireNonNull(tvDataDir.listFiles())) { - var data = TransportVersionUtils.readDataFile(tvDataFile); - tvNamesInDataFiles.add(data.name()); - } - - var tvSetDeclaredNamesFile = getTransportVersionSetNamesFile().get().getAsFile(); - try (var reader = new BufferedReader(new FileReader(tvSetDeclaredNamesFile))) { - reader.lines().forEach(declaredName -> { - if (tvNamesInDataFiles.contains(declaredName) == false) { - throw new RuntimeException( - "TransportVersionSetData.get(\"" - + declaredName - + "\") was used, but lacks a" - + " data file with a corresponding transport version. This can be generated with the task" // todo - ); - } - }); - } - } -} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java new file mode 100644 index 0000000000000..4298edee309f3 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Validates that each transport version named reference has a constant definition. + */ +public abstract class ValidateTransportVersionReferencesTask extends DefaultTask { + + @InputDirectory + public abstract DirectoryProperty getConstantsDirectory(); + + @InputFile + public abstract RegularFileProperty getNamesFile(); + + @TaskAction + public void validateTransportVersions() throws IOException { + Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); + Path namesFile = getNamesFile().get().getAsFile().toPath(); + + for (var tvName : Files.readAllLines(namesFile, StandardCharsets.UTF_8)) { + Path constantFile = constantsDir.resolve(tvName + ".csv"); + if (Files.exists(constantFile) == false) { + throw new RuntimeException( + "TransportVersion.fromName(\"" + + tvName + + "\") was used, but lacks a" + + " transport version constant definition. This can be generated with the task" // todo + ); + } + } + } +} diff --git a/server/build.gradle b/server/build.gradle index a93a11fbe7b0c..62ddb9738c70b 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -12,7 +12,8 @@ apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' -apply plugin: 'elasticsearch.transport-version-names' +apply plugin: 'elasticsearch.transport-version-management' +apply plugin: 'elasticsearch.transport-version-global-management' publishing { publications { From 312235e1e4945970854b3faa95d5cd7696d0ff02 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 24 Jul 2025 15:26:57 -0700 Subject: [PATCH 04/28] add example --- build-tools-internal/build.gradle | 6 +++--- ... => GlobalTransportVersionManagementPlugin.java} | 13 +++---------- .../transport/TransportVersionManagementPlugin.java | 2 +- .../internal/transport/TransportVersionUtils.java | 8 ++++++++ build.gradle | 3 ++- server/build.gradle | 1 - .../java/org/elasticsearch/TransportVersions.java | 2 +- .../transport/constants/esql_spli_on_big_values.csv | 1 + .../read/ValuesSourceReaderOperatorStatus.java | 8 ++------ 9 files changed, 21 insertions(+), 23 deletions(-) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/{TransportVersionGlobalManagementPlugin.java => GlobalTransportVersionManagementPlugin.java} (83%) create mode 100644 server/src/main/resources/transport/constants/esql_spli_on_big_values.csv diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index 8c81e4ed53e09..0b81ab90ceced 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -224,9 +224,9 @@ gradlePlugin { id = 'elasticsearch.transport-version-management' implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionManagementPlugin' } - transportVersionGlobalManagementPlugin { - id = 'elasticsearch.transport-version-global-management' - implementationClass = 'org.elasticsearch.gradle.internal.transport.TransportVersionGlobalManagementPlugin' + globalTransportVersionManagementPlugin { + id = 'elasticsearch.global-transport-version-management' + implementationClass = 'org.elasticsearch.gradle.internal.transport.GlobalTransportVersionManagementPlugin' } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java similarity index 83% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index b51f46094f3be..7f2f1c48e66fd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionGlobalManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -18,12 +18,8 @@ import java.util.ArrayList; import java.util.List; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE; -import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; - -public class TransportVersionGlobalManagementPlugin implements Plugin { - - +public class GlobalTransportVersionManagementPlugin implements Plugin { + @Override public void apply(Project project) { @@ -42,10 +38,7 @@ public void apply(Project project) { tvDependencies.add(depsHandler.create(":server")); Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); - tvNamesConfig.attributes(attrs -> { - attrs.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv"); - attrs.attribute(TRANSPORT_VERSION_NAMES_ATTRIBUTE, true); - }); + tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); var validateTask = project.getTasks().register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { t.setGroup("Transport Versions"); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 49b0d6d8abd94..3d71a790fd06e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -31,7 +31,7 @@ public void apply(Project project) { Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { c.setCanBeConsumed(true); c.setCanBeResolved(false); - c.attributes(a -> a.attribute(TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE, true)); + c.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); }); project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 29d2600e5ff36..6fa444f4f8f3e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -13,6 +13,7 @@ import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; +import org.gradle.api.attributes.AttributeContainer; import org.gradle.api.file.Directory; import java.io.IOException; @@ -23,6 +24,8 @@ import java.util.Comparator; import java.util.List; +import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; + class TransportVersionUtils { static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("transport-version-names", Boolean.class); @@ -53,4 +56,9 @@ static Directory getConstantsDirectory(Project project) { Directory serverDir = project.getRootProject().project(":server").getLayout().getProjectDirectory(); return serverDir.dir("src/main/resources/transport/constants"); } + + static void addTransportVersionNamesAttribute(AttributeContainer attributes) { + attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv"); + attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE, true); + } } diff --git a/build.gradle b/build.gradle index 5b534394b4e1c..d78fa4e4919d4 100644 --- a/build.gradle +++ b/build.gradle @@ -51,6 +51,7 @@ plugins { id 'elasticsearch.repositories' id 'elasticsearch.release-tools' id 'elasticsearch.versions' + id 'elasticsearch.global-transport-version-management' id 'com.gradleup.nmcp.aggregation' } @@ -366,7 +367,7 @@ allprojects { } } } - + proj.tasks.register("bcUpgradeTest$partString") { dependsOn tasks.matching { it.name == 'bcUpgradeTest' } withReleaseBuild { diff --git a/server/build.gradle b/server/build.gradle index 62ddb9738c70b..eb4ada28e7167 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -13,7 +13,6 @@ apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' apply plugin: 'elasticsearch.transport-version-management' -apply plugin: 'elasticsearch.transport-version-global-management' publishing { publications { diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index dad300ae72744..d6f65733b473d 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -337,7 +337,7 @@ static TransportVersion def(int id) { public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00); public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00); public static final TransportVersion ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = def(9_115_0_00); - public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); + //public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); public static final TransportVersion ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS = def(9_117_0_00); public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE = def(9_118_0_00); public static final TransportVersion ESQL_FIXED_INDEX_LIKE = def(9_119_0_00); diff --git a/server/src/main/resources/transport/constants/esql_spli_on_big_values.csv b/server/src/main/resources/transport/constants/esql_spli_on_big_values.csv new file mode 100644 index 0000000000000..3d0c94f8fa4c2 --- /dev/null +++ b/server/src/main/resources/transport/constants/esql_spli_on_big_values.csv @@ -0,0 +1 @@ +9116000,9112001,8841063 diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java index 4a8fcda81f82a..bba6b9b172f0f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java @@ -23,9 +23,6 @@ import static org.elasticsearch.TransportVersions.ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED; import static org.elasticsearch.TransportVersions.ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19; -import static org.elasticsearch.TransportVersions.ESQL_SPLIT_ON_BIG_VALUES; -import static org.elasticsearch.TransportVersions.ESQL_SPLIT_ON_BIG_VALUES_8_19; -import static org.elasticsearch.TransportVersions.ESQL_SPLIT_ON_BIG_VALUES_9_1; public class ValuesSourceReaderOperatorStatus extends AbstractPageMappingToIteratorOperator.Status { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -33,6 +30,7 @@ public class ValuesSourceReaderOperatorStatus extends AbstractPageMappingToItera "values_source_reader", ValuesSourceReaderOperatorStatus::readFrom ); + static final TransportVersion SPLIT_ON_BIG_VALUES = TransportVersion.fromName("esql_split_on_big_values"); private final Map readersBuilt; private final long valuesLoaded; @@ -103,9 +101,7 @@ public void writeTo(StreamOutput out) throws IOException { } private static boolean supportsSplitOnBigValues(TransportVersion version) { - return version.onOrAfter(ESQL_SPLIT_ON_BIG_VALUES) - || version.isPatchFrom(ESQL_SPLIT_ON_BIG_VALUES_9_1) - || version.isPatchFrom(ESQL_SPLIT_ON_BIG_VALUES_8_19); + return version.supports(SPLIT_ON_BIG_VALUES); } private static boolean supportsValuesLoaded(TransportVersion version) { From 818bda4fef11549e052cab68c6c49b8a764aa266 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 24 Jul 2025 22:38:09 +0000 Subject: [PATCH 05/28] [CI] Auto commit changes from spotless --- .../GlobalTransportVersionManagementPlugin.java | 15 ++++++++------- .../TransportVersionManagementPlugin.java | 17 +++++++++-------- .../transport/TransportVersionUtils.java | 2 +- .../org/elasticsearch/TransportVersions.java | 2 +- 4 files changed, 19 insertions(+), 17 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 7f2f1c48e66fd..03e145c105a26 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -19,7 +19,7 @@ import java.util.List; public class GlobalTransportVersionManagementPlugin implements Plugin { - + @Override public void apply(Project project) { @@ -40,12 +40,13 @@ public void apply(Project project) { Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); - var validateTask = project.getTasks().register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { - t.setGroup("Transport Versions"); - t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getNamesFiles().setFrom(tvNamesConfig); - }); + var validateTask = project.getTasks() + .register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getNamesFiles().setFrom(tvNamesConfig); + }); project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 3d71a790fd06e..c142b0f88ea77 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -36,14 +36,15 @@ public void apply(Project project) { project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); - var validateTask = project.getTasks().register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { - t.setGroup("Transport Versions"); - t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); - t.dependsOn(collectTask); - - }); + var validateTask = project.getTasks() + .register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + t.dependsOn(collectTask); + + }); project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 6fa444f4f8f3e..1fe04f2a54166 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -30,7 +30,7 @@ class TransportVersionUtils { static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("transport-version-names", Boolean.class); - record TransportVersionData(String name, List ids) { } + record TransportVersionData(String name, List ids) {} static TransportVersionData readDataFile(Path file) throws IOException { assert file.endsWith(".csv"); diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index d6f65733b473d..b0fe1e567cc35 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -337,7 +337,7 @@ static TransportVersion def(int id) { public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00); public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00); public static final TransportVersion ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = def(9_115_0_00); - //public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); + // public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); public static final TransportVersion ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS = def(9_117_0_00); public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE = def(9_118_0_00); public static final TransportVersion ESQL_FIXED_INDEX_LIKE = def(9_119_0_00); From 1721d2e620002077cc86343c5d75e5f490573b48 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 24 Jul 2025 15:49:33 -0700 Subject: [PATCH 06/28] iter --- .../transport/GlobalTransportVersionManagementPlugin.java | 5 +++-- .../gradle/internal/transport/TransportVersionUtils.java | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 03e145c105a26..8ccd64d841e7e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; public class GlobalTransportVersionManagementPlugin implements Plugin { @@ -32,10 +33,10 @@ public void apply(Project project) { if (pluginProject.getParent() != baseProject) { continue; // skip nested projects } - tvDependencies.add(depsHandler.create(pluginProject)); + tvDependencies.add(depsHandler.project(Map.of("path", pluginProject.getPath()))); } } - tvDependencies.add(depsHandler.create(":server")); + tvDependencies.add(depsHandler.project(Map.of("path", ":server"))); Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 1fe04f2a54166..4dc10c4c3ca1d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -40,13 +40,13 @@ static TransportVersionData readDataFile(Path file) throws IOException { for (String rawId : Files.readString(file, StandardCharsets.UTF_8).split(",")) { try { - ids.add(Integer.parseInt(rawId)); + ids.add(Integer.parseInt(rawId.strip())); } catch (NumberFormatException e) { throw new IOException("Failed to parse id " + rawId + " in " + file, e); } } - if (Comparators.isInOrder(ids, Comparator.naturalOrder()) == false) { + if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { throw new IOException("invalid transport version data file [" + file + "], ids are not in sorted"); } return new TransportVersionData(name, ids); From 8c8c7c433279159cb999fc7e488b658448de2aa1 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 09:28:10 -0700 Subject: [PATCH 07/28] iter and add generate manifest --- .../CollectTransportVersionNamesTask.java | 151 ++++++++++-------- .../GenerateTransportVersionManifestTask.java | 42 +++++ ...lobalTransportVersionManagementPlugin.java | 4 +- .../TransportVersionManagementPlugin.java | 6 +- .../transport/TransportVersionUtils.java | 35 +++- ...ValidateTransportVersionConstantsTask.java | 13 +- ...alidateTransportVersionReferencesTask.java | 15 +- ...alues.csv => esql_split_on_big_values.csv} | 0 .../ValuesSourceReaderOperatorStatus.java | 2 +- 9 files changed, 177 insertions(+), 91 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java rename server/src/main/resources/transport/constants/{esql_spli_on_big_values.csv => esql_split_on_big_values.csv} (100%) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java index ff744f111b764..4aed555e41631 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java @@ -10,35 +10,36 @@ package org.elasticsearch.gradle.internal.transport; import org.gradle.api.DefaultTask; -import org.gradle.api.file.FileCollection; +import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.RegularFileProperty; -import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; import org.gradle.api.tasks.TaskAction; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Label; import org.objectweb.asm.MethodVisitor; import org.objectweb.asm.Opcodes; import org.objectweb.asm.tree.LdcInsnNode; import org.objectweb.asm.tree.MethodNode; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; -import java.util.List; import java.util.Set; +import java.util.jar.JarInputStream; +import java.util.zip.ZipEntry; /** * This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the * provided directory, and then records the value of string literals passed as arguments. It then records each - * string on a newline in the provided output file. + * string on a newline along with path and line number in the provided output file. */ public abstract class CollectTransportVersionNamesTask extends DefaultTask { public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; @@ -50,7 +51,8 @@ public abstract class CollectTransportVersionNamesTask extends DefaultTask { * The directory to scan for method invocations. */ @InputFiles - public abstract Property getClassDirs(); + @Classpath + public abstract ConfigurableFileCollection getClassPath(); /** * The output file, with each newline containing the string literal argument of each method @@ -60,72 +62,89 @@ public abstract class CollectTransportVersionNamesTask extends DefaultTask { public abstract RegularFileProperty getOutputFile(); @TaskAction - public void checkTransportVersion() { - var classFiles = findJavaClassFiles(getClassDirs().get().getFiles()); - var tvNames = getTVDeclarationNames(classFiles); + public void checkTransportVersion() throws IOException { + var results = new HashSet(); - File file = getOutputFile().get().getAsFile(); - try (FileWriter writer = new FileWriter(file)) { - for (String tvName : tvNames) { - writer.write(tvName + "\n"); + for (var cpElement : getClassPath()) { + Path file = cpElement.toPath(); + if (Files.isDirectory(file)) { + addNamesFromClassesDirectory(results, file); + } else { + assert file.getFileName().toString().endsWith(".jar"); + addNamesFromJar(results, file); } - } catch (IOException e) { - throw new RuntimeException(e); } + + Path outputFile = getOutputFile().get().getAsFile().toPath(); + Files.writeString(outputFile, String.join("\n", results.stream().map(Object::toString).sorted().toList())); } - public static Set getTVDeclarationNames(Collection classfiles) { - var results = new HashSet(); - for (File javaFile : classfiles) { - try (InputStream inputStream = new FileInputStream(javaFile)) { - ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { - @Override - public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { - return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) { - @Override - public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { - if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) { - var abstractInstruction = this.instructions.getLast(); - if (abstractInstruction instanceof LdcInsnNode ldcInsnNode - && ldcInsnNode.cst instanceof String tvName - && tvName.isEmpty() == false) { - results.add(tvName); - } else { - // The instruction is not a LDC with a String constant (or an empty String), - // which is not allowed. - throw new RuntimeException( - "Transport Versions must be declared with a constant and non-empty String. " - + "file: " - + javaFile.getPath() - ); - } - } - super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); - } - }; + private void addNamesFromClassesDirectory(Set results, Path file) throws IOException { + Files.walkFileTree(file, new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + String filename = file.getFileName().toString(); + if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) { + try (var inputStream = Files.newInputStream(file)) { + addNamesFromClass(results, inputStream, classname(file.toString())); } - }; - ClassReader classReader = new ClassReader(inputStream); - classReader.accept(classVisitor, 0); - } catch (IOException e) { - throw new RuntimeException(e); + } + return FileVisitResult.CONTINUE; } - } - return results; + }); } - private static List findJavaClassFiles(Collection files) { - List classFiles = new ArrayList<>(); - for (File file : files) { - if (file.isDirectory()) { - File[] subFiles = file.listFiles(); - if (subFiles != null) { - classFiles.addAll(findJavaClassFiles(Arrays.asList(subFiles))); + private void addNamesFromJar(Set results, Path file) throws IOException { + try (var jar = new JarInputStream(Files.newInputStream(file))) { + ZipEntry entry; + while ((entry = jar.getNextEntry()) != null) { + String filename = entry.getName(); + if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) { + addNamesFromClass(results, jar, classname(entry.toString())); } - } else if (file.getName().endsWith(CLASS_EXTENSION) && file.getName().endsWith(MODULE_INFO) == false) { - classFiles.add(file); } } - return classFiles; + } + + private void addNamesFromClass(Set results, InputStream classBytes, String classname) + throws IOException { + ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) { + int lineNumber = -1; + + @Override + public void visitLineNumber(int line, Label start) { + lineNumber = line; + } + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) { + var abstractInstruction = this.instructions.getLast(); + String location = classname + " line " + lineNumber; + if (abstractInstruction instanceof LdcInsnNode ldcInsnNode + && ldcInsnNode.cst instanceof String tvName + && tvName.isEmpty() == false) { + results.add(new TransportVersionUtils.TransportVersionReference(tvName, location)); + } else { + // The instruction is not a LDC with a String constant (or an empty String), which is not allowed. + throw new RuntimeException( + "TransportVersion.fromName must be called with a non-empty String literal. " + "See " + location + "." + ); + } + } + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + }; + } + }; + ClassReader classReader = new ClassReader(classBytes); + classReader.accept(classVisitor, 0); + } + + private static String classname(String filename) { + return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replace('/', '.'); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java new file mode 100644 index 0000000000000..374857e53ac9c --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public abstract class GenerateTransportVersionManifestTask extends DefaultTask { + @InputDirectory + public abstract DirectoryProperty getConstantsDirectory(); + + @OutputFile + public abstract RegularFileProperty getManifestFile(); + + @TaskAction + public void generateTransportVersionManifest() throws IOException { + Path constantsDir = getConstantsDirectory().get().getAsFile().toPath(); + Path manifestFile = getManifestFile().get().getAsFile().toPath(); + try (var writer = Files.newBufferedWriter(manifestFile)) { + try (var stream = Files.list(constantsDir)) { + for (String filename : stream.map(p -> p.getFileName().toString()).toList()) { + writer.write(filename + "\n"); + } + } + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 8ccd64d841e7e..d6a27d20bdda7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -39,14 +39,14 @@ public void apply(Project project) { tvDependencies.add(depsHandler.project(Map.of("path", ":server"))); Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); - tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); + tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); var validateTask = project.getTasks() .register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getNamesFiles().setFrom(tvNamesConfig); + t.getReferencesFiles().setFrom(tvNamesConfig); }); project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index c142b0f88ea77..85a1259f7f7ac 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -24,14 +24,14 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion names used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - t.getClassDirs().set(mainSourceSet.getRuntimeClasspath()); + t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); }); Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { c.setCanBeConsumed(true); c.setCanBeResolved(false); - c.attributes(TransportVersionUtils::addTransportVersionNamesAttribute); + c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); }); project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); @@ -41,7 +41,7 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getNamesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); t.dependsOn(collectTask); }); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 4dc10c4c3ca1d..e8eb8b2174280 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -28,11 +28,18 @@ class TransportVersionUtils { - static final Attribute TRANSPORT_VERSION_NAMES_ATTRIBUTE = Attribute.of("transport-version-names", Boolean.class); + static final Attribute TRANSPORT_VERSION_REFERENCES_ATTRIBUTE = Attribute.of("transport-version-references", Boolean.class); - record TransportVersionData(String name, List ids) {} + record TransportVersionConstant(String name, List ids) {} - static TransportVersionData readDataFile(Path file) throws IOException { + record TransportVersionReference(String name, String location) { + @Override + public String toString() { + return name + " " + location; + } + } + + static TransportVersionConstant readConstantFile(Path file) throws IOException { assert file.endsWith(".csv"); String rawName = file.getFileName().toString(); String name = rawName.substring(0, rawName.length() - 4); @@ -49,7 +56,20 @@ static TransportVersionData readDataFile(Path file) throws IOException { if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { throw new IOException("invalid transport version data file [" + file + "], ids are not in sorted"); } - return new TransportVersionData(name, ids); + return new TransportVersionConstant(name, ids); + } + + static List readReferencesFile(Path file) throws IOException { + assert file.endsWith(".txt"); + List results = new ArrayList<>(); + for (String line : Files.readAllLines(file, StandardCharsets.UTF_8)) { + String[] parts = line.split(" ", 2); + if (parts.length != 2) { + throw new IOException("Invalid transport version data file [" + file + "]: " + line); + } + results.add(new TransportVersionReference(parts[0], parts[1])); + } + return results; } static Directory getConstantsDirectory(Project project) { @@ -57,8 +77,9 @@ static Directory getConstantsDirectory(Project project) { return serverDir.dir("src/main/resources/transport/constants"); } - static void addTransportVersionNamesAttribute(AttributeContainer attributes) { - attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "csv"); - attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_NAMES_ATTRIBUTE, true); + static void addTransportVersionReferencesAttribute(AttributeContainer attributes) { + attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "txt"); + attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_REFERENCES_ATTRIBUTE, true); } + } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java index b0d5aac5c6ef5..c7841b777199f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.transport; +import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionReference; import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DirectoryProperty; @@ -17,12 +18,14 @@ import org.gradle.api.tasks.TaskAction; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; import java.util.Set; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readConstantFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readReferencesFile; + /** * Validates that each defined transport version constant is referenced by at least one project. */ @@ -32,20 +35,20 @@ public abstract class ValidateTransportVersionConstantsTask extends DefaultTask public abstract DirectoryProperty getConstantsDirectory(); @InputFiles - public abstract ConfigurableFileCollection getNamesFiles(); + public abstract ConfigurableFileCollection getReferencesFiles(); @TaskAction public void validateTransportVersions() throws IOException { Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); Set allTvNames = new HashSet<>(); - for (var tvNamesFile : getNamesFiles()) { - allTvNames.addAll(Files.readAllLines(tvNamesFile.toPath(), StandardCharsets.UTF_8)); + for (var tvReferencesFile : getReferencesFiles()) { + readReferencesFile(tvReferencesFile.toPath()).stream().map(TransportVersionReference::name).forEach(allTvNames::add); } try (var constantsStream = Files.list(constantsDir)) { for (var constantsFile : constantsStream.toList()) { - var tv = TransportVersionUtils.readDataFile(constantsFile); + var tv = readConstantFile(constantsFile); if (allTvNames.contains(tv.name()) == false) { throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index 4298edee309f3..03202065a9801 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -17,7 +17,6 @@ import org.gradle.api.tasks.TaskAction; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -30,20 +29,22 @@ public abstract class ValidateTransportVersionReferencesTask extends DefaultTask public abstract DirectoryProperty getConstantsDirectory(); @InputFile - public abstract RegularFileProperty getNamesFile(); + public abstract RegularFileProperty getReferencesFile(); @TaskAction public void validateTransportVersions() throws IOException { Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); - Path namesFile = getNamesFile().get().getAsFile().toPath(); + Path namesFile = getReferencesFile().get().getAsFile().toPath(); - for (var tvName : Files.readAllLines(namesFile, StandardCharsets.UTF_8)) { - Path constantFile = constantsDir.resolve(tvName + ".csv"); + for (var tvReference : TransportVersionUtils.readReferencesFile(namesFile)) { + Path constantFile = constantsDir.resolve(tvReference.name() + ".csv"); if (Files.exists(constantFile) == false) { throw new RuntimeException( "TransportVersion.fromName(\"" - + tvName - + "\") was used, but lacks a" + + tvReference.name() + + "\") was used at " + + tvReference.location() + + ", but lacks a" + " transport version constant definition. This can be generated with the task" // todo ); } diff --git a/server/src/main/resources/transport/constants/esql_spli_on_big_values.csv b/server/src/main/resources/transport/constants/esql_split_on_big_values.csv similarity index 100% rename from server/src/main/resources/transport/constants/esql_spli_on_big_values.csv rename to server/src/main/resources/transport/constants/esql_split_on_big_values.csv diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java index bba6b9b172f0f..13ed5e4c84b8f 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java @@ -30,7 +30,7 @@ public class ValuesSourceReaderOperatorStatus extends AbstractPageMappingToItera "values_source_reader", ValuesSourceReaderOperatorStatus::readFrom ); - static final TransportVersion SPLIT_ON_BIG_VALUES = TransportVersion.fromName("esql_split_on_big_values"); + private static final TransportVersion SPLIT_ON_BIG_VALUES = TransportVersion.fromName("esql_split_on_big_values"); private final Map readersBuilt; private final long valuesLoaded; From 3d3006221334402d294f8d4bb783a2e37fd3456d Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 10:08:10 -0700 Subject: [PATCH 08/28] wired manifest task --- ...lobalTransportVersionManagementPlugin.java | 19 ++++++++++++++++--- .../TransportVersionManagementPlugin.java | 6 +++--- build.gradle | 1 - server/build.gradle | 1 + 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index d6a27d20bdda7..6618fe1610177 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -14,6 +14,8 @@ import org.gradle.api.artifacts.Configuration; import org.gradle.api.artifacts.Dependency; import org.gradle.api.artifacts.dsl.DependencyHandler; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.tasks.Copy; import java.util.ArrayList; import java.util.List; @@ -38,15 +40,26 @@ public void apply(Project project) { } tvDependencies.add(depsHandler.project(Map.of("path", ":server"))); - Configuration tvNamesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); - tvNamesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); + tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + + var generateManifestTask = project.getTasks() + .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Generate a manifest resource for all the known transport version constants"); + t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt")); + }); + project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> { + t.into("transport/constants", c -> c.from(generateManifestTask)); + }); var validateTask = project.getTasks() .register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getReferencesFiles().setFrom(tvNamesConfig); + t.getReferencesFiles().setFrom(tvReferencesConfig); }); project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 85a1259f7f7ac..af26576c15243 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -19,13 +19,13 @@ public class TransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { - String transportVersionsNamesFile = "generated-transport-constants/transport-version-set-names.txt"; + String transportVersionReferencesFile = "generated-transport-constants/transport-version-set-names.txt"; var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion names used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); - t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); }); Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { @@ -41,7 +41,7 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionsNamesFile)); + t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); t.dependsOn(collectTask); }); diff --git a/build.gradle b/build.gradle index d78fa4e4919d4..f99fbc01def0e 100644 --- a/build.gradle +++ b/build.gradle @@ -51,7 +51,6 @@ plugins { id 'elasticsearch.repositories' id 'elasticsearch.release-tools' id 'elasticsearch.versions' - id 'elasticsearch.global-transport-version-management' id 'com.gradleup.nmcp.aggregation' } diff --git a/server/build.gradle b/server/build.gradle index eb4ada28e7167..59615e5f42ac5 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -13,6 +13,7 @@ apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' apply plugin: 'elasticsearch.transport-version-management' +apply plugin: 'elasticsearch.global-transport-version-management' publishing { publications { From d94594706e1ab85fe9f854ba72aba338060bd6a0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 13:30:53 -0700 Subject: [PATCH 09/28] done --- ...ollectTransportVersionReferencesTask.java} | 2 +- .../GenerateTransportVersionManifestTask.java | 4 ++-- ...lobalTransportVersionManagementPlugin.java | 22 +++++++++---------- .../TransportVersionManagementPlugin.java | 10 ++++----- .../transport/TransportVersionUtils.java | 6 ++--- ...idateTransportVersionDefinitionsTask.java} | 10 ++++----- ...alidateTransportVersionReferencesTask.java | 4 ++-- 7 files changed, 29 insertions(+), 29 deletions(-) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/{CollectTransportVersionNamesTask.java => CollectTransportVersionReferencesTask.java} (98%) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/{ValidateTransportVersionConstantsTask.java => ValidateTransportVersionDefinitionsTask.java} (86%) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java similarity index 98% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java index 4aed555e41631..c7e46748391aa 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionNamesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java @@ -41,7 +41,7 @@ * provided directory, and then records the value of string literals passed as arguments. It then records each * string on a newline along with path and line number in the provided output file. */ -public abstract class CollectTransportVersionNamesTask extends DefaultTask { +public abstract class CollectTransportVersionReferencesTask extends DefaultTask { public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "fromName"; public static final String CLASS_EXTENSION = ".class"; diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java index 374857e53ac9c..b568214aff77d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java @@ -22,14 +22,14 @@ public abstract class GenerateTransportVersionManifestTask extends DefaultTask { @InputDirectory - public abstract DirectoryProperty getConstantsDirectory(); + public abstract DirectoryProperty getDefinitionsDirectory(); @OutputFile public abstract RegularFileProperty getManifestFile(); @TaskAction public void generateTransportVersionManifest() throws IOException { - Path constantsDir = getConstantsDirectory().get().getAsFile().toPath(); + Path constantsDir = getDefinitionsDirectory().get().getAsFile().toPath(); Path manifestFile = getManifestFile().get().getAsFile().toPath(); try (var writer = Files.newBufferedWriter(manifestFile)) { try (var stream = Files.list(constantsDir)) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 6618fe1610177..0afc6252e68ef 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -43,25 +43,25 @@ public void apply(Project project) { Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + var validateTask = project.getTasks() + .register("validateTransportVersionConstants", ValidateTransportVersionDefinitionsTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); + t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); + t.getReferencesFiles().setFrom(tvReferencesConfig); + }); + + project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); + var generateManifestTask = project.getTasks() .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Generate a manifest resource for all the known transport version constants"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt")); }); project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> { t.into("transport/constants", c -> c.from(generateManifestTask)); }); - - var validateTask = project.getTasks() - .register("validateTransportVersionConstants", ValidateTransportVersionConstantsTask.class, t -> { - t.setGroup("Transport Versions"); - t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); - t.getReferencesFiles().setFrom(tvReferencesConfig); - }); - - project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index af26576c15243..03e868e372253 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -19,10 +19,10 @@ public class TransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { - String transportVersionReferencesFile = "generated-transport-constants/transport-version-set-names.txt"; - var collectTask = project.getTasks().register("collectTransportVersionNames", CollectTransportVersionNamesTask.class, t -> { + String transportVersionReferencesFile = "transport-version/references.txt"; + var collectTask = project.getTasks().register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); - t.setDescription("Collects all TransportVersion names used throughout the project"); + t.setDescription("Collects all TransportVersion references used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); @@ -39,8 +39,8 @@ public void apply(Project project) { var validateTask = project.getTasks() .register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); - t.setDescription("Validates that all TransportVersion names used in the project have an associated data file"); - t.getConstantsDirectory().set(TransportVersionUtils.getConstantsDirectory(project)); + t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file"); + t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); t.dependsOn(collectTask); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index e8eb8b2174280..0978b38a0867b 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -39,7 +39,7 @@ public String toString() { } } - static TransportVersionConstant readConstantFile(Path file) throws IOException { + static TransportVersionConstant readDefinitionFile(Path file) throws IOException { assert file.endsWith(".csv"); String rawName = file.getFileName().toString(); String name = rawName.substring(0, rawName.length() - 4); @@ -72,9 +72,9 @@ static List readReferencesFile(Path file) throws IOEx return results; } - static Directory getConstantsDirectory(Project project) { + static Directory getDefinitionsDirectory(Project project) { Directory serverDir = project.getRootProject().project(":server").getLayout().getProjectDirectory(); - return serverDir.dir("src/main/resources/transport/constants"); + return serverDir.dir("src/main/resources/transport/defined"); } static void addTransportVersionReferencesAttribute(AttributeContainer attributes) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java similarity index 86% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index c7841b777199f..86df1344a62f0 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionConstantsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -23,23 +23,23 @@ import java.util.HashSet; import java.util.Set; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readConstantFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readDefinitionFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readReferencesFile; /** * Validates that each defined transport version constant is referenced by at least one project. */ -public abstract class ValidateTransportVersionConstantsTask extends DefaultTask { +public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTask { @InputDirectory - public abstract DirectoryProperty getConstantsDirectory(); + public abstract DirectoryProperty getDefinitionsDirectory(); @InputFiles public abstract ConfigurableFileCollection getReferencesFiles(); @TaskAction public void validateTransportVersions() throws IOException { - Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); + Path constantsDir = getDefinitionsDirectory().getAsFile().get().toPath(); Set allTvNames = new HashSet<>(); for (var tvReferencesFile : getReferencesFiles()) { @@ -48,7 +48,7 @@ public void validateTransportVersions() throws IOException { try (var constantsStream = Files.list(constantsDir)) { for (var constantsFile : constantsStream.toList()) { - var tv = readConstantFile(constantsFile); + var tv = readDefinitionFile(constantsFile); if (allTvNames.contains(tv.name()) == false) { throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index 03202065a9801..b5df69d6be012 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -26,14 +26,14 @@ public abstract class ValidateTransportVersionReferencesTask extends DefaultTask { @InputDirectory - public abstract DirectoryProperty getConstantsDirectory(); + public abstract DirectoryProperty getDefinitionsDirectory(); @InputFile public abstract RegularFileProperty getReferencesFile(); @TaskAction public void validateTransportVersions() throws IOException { - Path constantsDir = getConstantsDirectory().getAsFile().get().toPath(); + Path constantsDir = getDefinitionsDirectory().getAsFile().get().toPath(); Path namesFile = getReferencesFile().get().getAsFile().toPath(); for (var tvReference : TransportVersionUtils.readReferencesFile(namesFile)) { From c6251da0e8a4b3443dfb2b9c49d7f7fe136812f9 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 13:32:34 -0700 Subject: [PATCH 10/28] spotless and rename --- .../TransportVersionManagementPlugin.java | 15 ++++++++------- .../esql_split_on_big_values.csv | 0 2 files changed, 8 insertions(+), 7 deletions(-) rename server/src/main/resources/transport/{constants => defined}/esql_split_on_big_values.csv (100%) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 03e868e372253..e0a1fdb9e9aae 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -20,13 +20,14 @@ public class TransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { String transportVersionReferencesFile = "transport-version/references.txt"; - var collectTask = project.getTasks().register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> { - t.setGroup("Transport Versions"); - t.setDescription("Collects all TransportVersion references used throughout the project"); - SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); - t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); - }); + var collectTask = project.getTasks() + .register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Collects all TransportVersion references used throughout the project"); + SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); + t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); + }); Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { c.setCanBeConsumed(true); diff --git a/server/src/main/resources/transport/constants/esql_split_on_big_values.csv b/server/src/main/resources/transport/defined/esql_split_on_big_values.csv similarity index 100% rename from server/src/main/resources/transport/constants/esql_split_on_big_values.csv rename to server/src/main/resources/transport/defined/esql_split_on_big_values.csv From 97503ab5aedc07e1c9f3289a0ba0013357d282e8 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 13:41:57 -0700 Subject: [PATCH 11/28] moved to defined --- server/src/main/java/org/elasticsearch/TransportVersion.java | 4 ++-- .../resources/transport/{constant => defined}/manifest.txt | 0 .../test/resources/transport/{constant => defined}/test_0.csv | 0 .../test/resources/transport/{constant => defined}/test_1.csv | 0 .../test/resources/transport/{constant => defined}/test_2.csv | 0 .../test/resources/transport/{constant => defined}/test_3.csv | 0 .../test/resources/transport/{constant => defined}/test_4.csv | 0 7 files changed, 2 insertions(+), 2 deletions(-) rename server/src/test/resources/transport/{constant => defined}/manifest.txt (100%) rename server/src/test/resources/transport/{constant => defined}/test_0.csv (100%) rename server/src/test/resources/transport/{constant => defined}/test_1.csv (100%) rename server/src/test/resources/transport/{constant => defined}/test_2.csv (100%) rename server/src/test/resources/transport/{constant => defined}/test_3.csv (100%) rename server/src/test/resources/transport/{constant => defined}/test_4.csv (100%) diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index 2ac4c1bf72ab6..d31bb2a5e5495 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -399,7 +399,7 @@ private static Map loadTransportVersionsByName() { throw new UncheckedIOException("latest transport version file not found at [" + latestLocation + "]", ioe); } - String manifestLocation = "/transport/constant/manifest.txt"; + String manifestLocation = "/transport/defined/manifest.txt"; List versionFileNames = null; if (latestId > -1) { try (InputStream inputStream = TransportVersion.class.getResourceAsStream(manifestLocation)) { @@ -414,7 +414,7 @@ private static Map loadTransportVersionsByName() { if (versionFileNames != null) { for (String name : versionFileNames) { - String versionLocation = "/transport/constant/" + name; + String versionLocation = "/transport/defined/" + name; try (InputStream inputStream = TransportVersion.class.getResourceAsStream(versionLocation)) { if (inputStream == null) { throw new IllegalStateException("transport version file not found at [" + versionLocation + "]"); diff --git a/server/src/test/resources/transport/constant/manifest.txt b/server/src/test/resources/transport/defined/manifest.txt similarity index 100% rename from server/src/test/resources/transport/constant/manifest.txt rename to server/src/test/resources/transport/defined/manifest.txt diff --git a/server/src/test/resources/transport/constant/test_0.csv b/server/src/test/resources/transport/defined/test_0.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_0.csv rename to server/src/test/resources/transport/defined/test_0.csv diff --git a/server/src/test/resources/transport/constant/test_1.csv b/server/src/test/resources/transport/defined/test_1.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_1.csv rename to server/src/test/resources/transport/defined/test_1.csv diff --git a/server/src/test/resources/transport/constant/test_2.csv b/server/src/test/resources/transport/defined/test_2.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_2.csv rename to server/src/test/resources/transport/defined/test_2.csv diff --git a/server/src/test/resources/transport/constant/test_3.csv b/server/src/test/resources/transport/defined/test_3.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_3.csv rename to server/src/test/resources/transport/defined/test_3.csv diff --git a/server/src/test/resources/transport/constant/test_4.csv b/server/src/test/resources/transport/defined/test_4.csv similarity index 100% rename from server/src/test/resources/transport/constant/test_4.csv rename to server/src/test/resources/transport/defined/test_4.csv From 508aec1c5a1014472d2b73de3f208f0ec8f4a54e Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 13:47:47 -0700 Subject: [PATCH 12/28] fix task name --- .../transport/GlobalTransportVersionManagementPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 0afc6252e68ef..bb92ac9a1a226 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -44,7 +44,7 @@ public void apply(Project project) { tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); var validateTask = project.getTasks() - .register("validateTransportVersionConstants", ValidateTransportVersionDefinitionsTask.class, t -> { + .register("validateTransportVersionDefinitions", ValidateTransportVersionDefinitionsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); From 832b5953847d4c58c42dba99daceb13c492f2a95 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Fri, 25 Jul 2025 14:52:57 -0700 Subject: [PATCH 13/28] First draft --- .../GenerateTransportVersionDataTask.java | 184 ++++++++++++++++++ .../transport/TransportVersionUtils.java | 27 +-- ...lidateTransportVersionDefinitionsTask.java | 8 +- 3 files changed, 204 insertions(+), 15 deletions(-) create mode 100644 build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java new file mode 100644 index 0000000000000..4f461f366b7eb --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java @@ -0,0 +1,184 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import com.google.common.collect.Streams; +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionSetData; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.TaskAction; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.LATEST_SUFFIX; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getTVSetDataFilePath; + +/** + * This task generates TransportVersionSetData data files that contain information about transport versions. These files + * are added to the server project's resource directory at `server/src/main/resources/org/elasticsearch/transport/`. + * They have the following format: + * 
+ * Filename: my-transport-version-set.json  // Must be the same as the name of the transport version set.
+ * {
+ *   "name": "my-transport-version-set", // The name of the transport version set used for reference in the code.
+ *   "ids": [
+ *     9109000,  // The transport version introduced to the main branch.
+ *     8841059   // The transport version backported to a previous release branch.
+ *   ]
+ * }
+ *
+ */ +public abstract class GenerateTransportVersionDataTask extends DefaultTask { + + /** + * Specifies the directory in which contains all TransportVersionSet data files. + * + * @return + */ + @InputDirectory + public abstract RegularFileProperty getDataFileDirectory(); + + /** + * Used to set the name of the TransportVersionSet for which a data file will be generated. + */ + @Input + public abstract Property getTVName(); + + /** + * Used to set the `major.minor` release version for which the specific TransportVersion ID will be generated. + * E.g.: "9.2", "8.18", etc. + */ + @Input + public abstract Property getMinorVersionForTV(); + + @TaskAction + public void generateTransportVersionData() throws IOException { + final var tvDataDir = Objects.requireNonNull(getDataFileDirectory().getAsFile().get()); + final var tvSetName = Objects.requireNonNull(getTVName().get()); + final var minorVersion = MinorVersion.fromString(Objects.requireNonNull(getMinorVersionForTV().get())); + + // Get the latest transport version data for the specified minor version. + final var latestTV = TransportVersionUtils.getLatestFile(tvDataDir.toPath(), minorVersion.toString()); + + // Create the new version + final var mainReleaseVersion = MinorVersion.of(VersionProperties.getElasticsearchVersion()); + final var isReleaseVersionMain = minorVersion.equals(mainReleaseVersion); + int newVersion = bumpVersionNumber(latestTV.ids().getFirst(), minorVersion, isReleaseVersionMain); + + // Load the tvSetData for the specified name, if it exists + final var tvSetDataFromFile = TransportVersionUtils.readDefinitionFile(tvDataDir.toPath(), tvSetName); + final var tvSetFileExists = tvSetDataFromFile != null; + + // Create/update the data file + if (tvSetFileExists) { + // This is not a new TVSet. We are creating a backport version for an existing TVSet. + // Check to ensure that there isn't already a TV id for this release version (e.g., if this task has been run twice). + var existingIDsForReleaseVersion = tvSetDataFromFile.ids().stream().filter(id -> { + var priorLatestID = priorLatestTVSetData.ids().getFirst(); + return priorLatestID < id && id <= newVersion; + }).toList(); + if (existingIDsForReleaseVersion.isEmpty() == false) { + throw new GradleException( + "A transport version could not be created because a preexisting one was found for this name & release." + + " This could be due to another pre-existing TV with the same name, or a result of running this" + + " task twice:" + + " Release version: " + + minorVersion + + " TransportVersion Id: " + + existingIDsForReleaseVersion.getFirst() + + " File: " + + getTVSetDataFilePath(tvDataDir, tvSetName) + ); + } + // 9.2:123 | 9.1:456 | 9.0:789 | + + // Update the existing data file for the backport. + new TransportVersionSetData( + tvSetName, + Streams.concat(tvSetDataFromFile.ids().stream(), Stream.of(newVersion)).sorted().toList().reversed() + ).writeToDataDir(tvDataDir); + } else { + // Create a new data file for the case where this is a new TV + new TransportVersionSetData(tvSetName, List.of(newVersion)).writeToDataDir(tvDataDir); + } + + // Update the LATEST file. + TransportVersionUtils.writeTVSetData( + tvDataDir, + formatLatestTVSetFilename(minorVersion), + new TransportVersionSetData(tvSetName, List.of(newVersion)) + ); + } + + // TODO Do I need to remove the patch when updating the server portion? NO, but probably need some additional checks + private static int bumpVersionNumber( + int tvIDToBump, + MinorVersion minorVersion, + boolean majorVersionBump, + boolean isTVReleaseVersionMain + ) { + + /* The TV format: + * + * MM_NNN_S_PP + * + * M - The major version of Elasticsearch + * NNN - The server version part + * S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories. + * PP - The patch version part + */ + if (isTVReleaseVersionMain) { + if (majorVersionBump) { + // Bump the major version part, set all other parts to zero. + return minorVersion.major * 1_000_000; // TODO add check that this doesn't cause overflow out of server versions + } else { + // Bump the server version part if not a major bump. + // TODO add check that this doesn't cause overflow out of server versions + // TODO Do we need to assert on the shape of the number? e.g. no patch version. + return tvIDToBump + 1000; + } + } else { + // bump the patch version part + return tvIDToBump + 1; // TODO add check that this doesn't cause overflow out of patch versions + } + } + + + private static String formatLatestTVSetFilename(MinorVersion minorVersion) { + return minorVersion.toString() + LATEST_SUFFIX; + } + + private record MinorVersion(int major, int minor) { + public static MinorVersion fromString(String string) { + String[] versionParts = string.split("\\."); + assert versionParts.length == 2; + return new MinorVersion(Integer.parseInt(versionParts[0]), Integer.parseInt(versionParts[1])); + } + + public static MinorVersion of(Version version) { + return new MinorVersion(version.getMajor(), version.getMinor()); + } + + @Override + public @NotNull String toString() { + return major + "." + minor; + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 0978b38a0867b..05a94014987a7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -10,7 +10,6 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; - import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; import org.gradle.api.attributes.AttributeContainer; @@ -27,10 +26,10 @@ import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; class TransportVersionUtils { - static final Attribute TRANSPORT_VERSION_REFERENCES_ATTRIBUTE = Attribute.of("transport-version-references", Boolean.class); - record TransportVersionConstant(String name, List ids) {} + record TransportVersionDefinition(String name, List ids) { + } record TransportVersionReference(String name, String location) { @Override @@ -39,24 +38,29 @@ public String toString() { } } - static TransportVersionConstant readDefinitionFile(Path file) throws IOException { + static TransportVersionDefinition getLatestFile(Path latestDataDir, String majorMinor) throws IOException { + return readDefinitionFile(latestDataDir.resolve(majorMinor + ".csv"), true); + } + + static TransportVersionDefinition readDefinitionFile(Path file, boolean nameInFile) throws IOException { assert file.endsWith(".csv"); String rawName = file.getFileName().toString(); - String name = rawName.substring(0, rawName.length() - 4); - List ids = new ArrayList<>(); - for (String rawId : Files.readString(file, StandardCharsets.UTF_8).split(",")) { + String[] parts = Files.readString(file, StandardCharsets.UTF_8).split(","); + String name = nameInFile ? parts[0] : rawName.substring(0, rawName.length() - 4); + List ids = new ArrayList<>(); + for (int i = nameInFile ? 1 : 0; i < parts.length; ++i) { try { - ids.add(Integer.parseInt(rawId.strip())); - } catch (NumberFormatException e) { - throw new IOException("Failed to parse id " + rawId + " in " + file, e); + ids.add(Integer.parseInt(parts[i])); + } catch (NumberFormatException nfe) { + throw new IllegalStateException("invalid transport version file format [" + file + "]", nfe); } } if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { throw new IOException("invalid transport version data file [" + file + "], ids are not in sorted"); } - return new TransportVersionConstant(name, ids); + return new TransportVersionDefinition(name, ids); } static List readReferencesFile(Path file) throws IOException { @@ -82,4 +86,5 @@ static void addTransportVersionReferencesAttribute(AttributeContainer attributes attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_REFERENCES_ATTRIBUTE, true); } + } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index 86df1344a62f0..b18e663229733 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -39,16 +39,16 @@ public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTas @TaskAction public void validateTransportVersions() throws IOException { - Path constantsDir = getDefinitionsDirectory().getAsFile().get().toPath(); + Path definitionsDir = getDefinitionsDirectory().getAsFile().get().toPath(); Set allTvNames = new HashSet<>(); for (var tvReferencesFile : getReferencesFiles()) { readReferencesFile(tvReferencesFile.toPath()).stream().map(TransportVersionReference::name).forEach(allTvNames::add); } - try (var constantsStream = Files.list(constantsDir)) { - for (var constantsFile : constantsStream.toList()) { - var tv = readDefinitionFile(constantsFile); + try (var definitionsStream = Files.list(definitionsDir)) { + for (var constantsFile : definitionsStream.toList()) { + var tv = readDefinitionFile(constantsFile, false); if (allTvNames.contains(tv.name()) == false) { throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced"); } From 1e3fb6503c94a6a3f77f5237e5d2108ffcfdec01 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 15:39:59 -0700 Subject: [PATCH 14/28] address review comments --- .gitignore | 1 + .../CollectTransportVersionReferencesTask.java | 5 +++-- .../GlobalTransportVersionManagementPlugin.java | 4 +++- .../transport/TransportVersionManagementPlugin.java | 12 ++++++------ .../internal/transport/TransportVersionUtils.java | 2 +- .../ValidateTransportVersionDefinitionsTask.java | 2 ++ .../ValidateTransportVersionReferencesTask.java | 2 ++ 7 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.gitignore b/.gitignore index 8b2da4dc0832a..cac5a799012e1 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,7 @@ testfixtures_shared/ # Generated checkstyle_ide.xml x-pack/plugin/esql/src/main/generated-src/generated/ +server/src/main/resources/transport/defined/manifest.txt # JEnv .java-version diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java index c7e46748391aa..c3f3470413b7f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java @@ -12,6 +12,7 @@ import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; @@ -41,6 +42,7 @@ * provided directory, and then records the value of string literals passed as arguments. It then records each * string on a newline along with path and line number in the provided output file. */ +@CacheableTask public abstract class CollectTransportVersionReferencesTask extends DefaultTask { public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersion"; public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "fromName"; @@ -50,7 +52,6 @@ public abstract class CollectTransportVersionReferencesTask extends DefaultTask /** * The directory to scan for method invocations. */ - @InputFiles @Classpath public abstract ConfigurableFileCollection getClassPath(); @@ -145,6 +146,6 @@ public void visitMethodInsn(int opcode, String owner, String name, String descri } private static String classname(String filename) { - return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replace('/', '.'); + return filename.substring(0, filename.length() - CLASS_EXTENSION.length()).replaceAll("[/\\\\]", "."); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index bb92ac9a1a226..323706de7dfa6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -16,6 +16,7 @@ import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.tasks.Copy; +import org.gradle.language.base.plugins.LifecycleBasePlugin; import java.util.ArrayList; import java.util.List; @@ -25,6 +26,7 @@ public class GlobalTransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { + project.getPluginManager().apply(LifecycleBasePlugin.class); DependencyHandler depsHandler = project.getDependencies(); List tvDependencies = new ArrayList<>(); @@ -51,7 +53,7 @@ public void apply(Project project) { t.getReferencesFiles().setFrom(tvReferencesConfig); }); - project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); + project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); var generateManifestTask = project.getTasks() .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index e0a1fdb9e9aae..04ee595e67e64 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -14,19 +14,21 @@ import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; import org.gradle.api.tasks.SourceSet; +import org.gradle.language.base.plugins.LifecycleBasePlugin; public class TransportVersionManagementPlugin implements Plugin { @Override public void apply(Project project) { - String transportVersionReferencesFile = "transport-version/references.txt"; + project.getPluginManager().apply(LifecycleBasePlugin.class); + var collectTask = project.getTasks() .register("collectTransportVersionReferences", CollectTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion references used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); - t.getOutputFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.txt")); }); Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { @@ -42,11 +44,9 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file"); t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); - t.getReferencesFile().set(project.getLayout().getBuildDirectory().file(transportVersionReferencesFile)); - t.dependsOn(collectTask); - + t.getReferencesFile().set(collectTask.get().getOutputFile()); }); - project.getTasks().named("check").configure(t -> t.dependsOn(validateTask)); + project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 0978b38a0867b..68716a5d6399c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -73,7 +73,7 @@ static List readReferencesFile(Path file) throws IOEx } static Directory getDefinitionsDirectory(Project project) { - Directory serverDir = project.getRootProject().project(":server").getLayout().getProjectDirectory(); + Directory serverDir = project.project(":server").getLayout().getProjectDirectory(); return serverDir.dir("src/main/resources/transport/defined"); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index 86df1344a62f0..f5d079dddb204 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -13,6 +13,7 @@ import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DirectoryProperty; +import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.TaskAction; @@ -29,6 +30,7 @@ /** * Validates that each defined transport version constant is referenced by at least one project. */ +@CacheableTask public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTask { @InputDirectory diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index b5df69d6be012..bafd7bc2e594f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -12,6 +12,7 @@ import org.gradle.api.DefaultTask; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFile; import org.gradle.api.tasks.TaskAction; @@ -23,6 +24,7 @@ /** * Validates that each transport version named reference has a constant definition. */ +@CacheableTask public abstract class ValidateTransportVersionReferencesTask extends DefaultTask { @InputDirectory From 49dca335ef3c1e116432df63b46d731cd40dd710 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 15:50:21 -0700 Subject: [PATCH 15/28] iter --- ...lobalTransportVersionManagementPlugin.java | 25 ++++++++----------- .../TransportVersionManagementPlugin.java | 11 +++----- .../transport/TransportVersionUtils.java | 8 ++++-- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 323706de7dfa6..bf74f52ee1912 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.transport; +import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; @@ -29,22 +30,17 @@ public void apply(Project project) { project.getPluginManager().apply(LifecycleBasePlugin.class); DependencyHandler depsHandler = project.getDependencies(); - List tvDependencies = new ArrayList<>(); - // TODO: created a named configuration so deps can be added dynamically? - for (String baseProjectPath : List.of(":modules", ":plugins", ":x-pack:plugin")) { - Project baseProject = project.project(baseProjectPath); - for (var pluginProject : baseProject.getSubprojects()) { - if (pluginProject.getParent() != baseProject) { - continue; // skip nested projects - } - tvDependencies.add(depsHandler.project(Map.of("path", pluginProject.getPath()))); - } - } - tvDependencies.add(depsHandler.project(Map.of("path", ":server"))); - - Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(tvDependencies.toArray(new Dependency[0])); + Dependency selfDependency = depsHandler.project(Map.of("path", project.getPath())); + Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(selfDependency); tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + // iterate through all projects, and if the ES plugin build plugin is applied, add that project back as a dep to check + for (Project subProject : project.getRootProject().getSubprojects()) { + subProject.getPlugins().withType(BaseInternalPluginBuildPlugin.class).configureEach(plugin -> { + tvReferencesConfig.getDependencies().add(depsHandler.project(Map.of("path", subProject.getPath()))); + }); + } + var validateTask = project.getTasks() .register("validateTransportVersionDefinitions", ValidateTransportVersionDefinitionsTask.class, t -> { t.setGroup("Transport Versions"); @@ -52,7 +48,6 @@ public void apply(Project project) { t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getReferencesFiles().setFrom(tvReferencesConfig); }); - project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); var generateManifestTask = project.getTasks() diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 04ee595e67e64..6b11756cc8352 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -31,13 +31,9 @@ public void apply(Project project) { t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.txt")); }); - Configuration transportVersionsConfig = project.getConfigurations().create("transportVersionNames", c -> { - c.setCanBeConsumed(true); - c.setCanBeResolved(false); - c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); - }); - - project.getArtifacts().add(transportVersionsConfig.getName(), collectTask); + Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(); + tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + project.getArtifacts().add(tvReferencesConfig.getName(), collectTask); var validateTask = project.getTasks() .register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { @@ -46,7 +42,6 @@ public void apply(Project project) { t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getReferencesFile().set(collectTask.get().getOutputFile()); }); - project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 68716a5d6399c..ca932adadae25 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -73,8 +73,12 @@ static List readReferencesFile(Path file) throws IOEx } static Directory getDefinitionsDirectory(Project project) { - Directory serverDir = project.project(":server").getLayout().getProjectDirectory(); - return serverDir.dir("src/main/resources/transport/defined"); + var projectName = project.findProperty("org.elasticsearch.transport.definitionsProject"); + if (projectName == null) { + projectName = ":server"; + } + Directory projectDir = project.project(projectName.toString()).getLayout().getProjectDirectory(); + return projectDir.dir("src/main/resources/transport/defined"); } static void addTransportVersionReferencesAttribute(AttributeContainer attributes) { From a4c62df246f2b19b7ae1495659a84c8be28207f0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 25 Jul 2025 18:40:59 -0700 Subject: [PATCH 16/28] more feedback --- .../CollectTransportVersionReferencesTask.java | 18 ++---------------- ...GlobalTransportVersionManagementPlugin.java | 7 +++---- .../TransportVersionManagementPlugin.java | 9 ++++++--- ...alidateTransportVersionDefinitionsTask.java | 4 ++++ ...ValidateTransportVersionReferencesTask.java | 4 ++++ x-pack/plugin/esql/compute/build.gradle | 1 + 6 files changed, 20 insertions(+), 23 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java index c3f3470413b7f..e4de8a0051366 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java @@ -14,8 +14,9 @@ import org.gradle.api.file.RegularFileProperty; import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; -import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; @@ -70,9 +71,6 @@ public void checkTransportVersion() throws IOException { Path file = cpElement.toPath(); if (Files.isDirectory(file)) { addNamesFromClassesDirectory(results, file); - } else { - assert file.getFileName().toString().endsWith(".jar"); - addNamesFromJar(results, file); } } @@ -95,18 +93,6 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO }); } - private void addNamesFromJar(Set results, Path file) throws IOException { - try (var jar = new JarInputStream(Files.newInputStream(file))) { - ZipEntry entry; - while ((entry = jar.getNextEntry()) != null) { - String filename = entry.getName(); - if (filename.endsWith(CLASS_EXTENSION) && filename.endsWith(MODULE_INFO) == false) { - addNamesFromClass(results, jar, classname(entry.toString())); - } - } - } - } - private void addNamesFromClass(Set results, InputStream classBytes, String classname) throws IOException { ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index bf74f52ee1912..3d96c3868d885 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -30,13 +30,12 @@ public void apply(Project project) { project.getPluginManager().apply(LifecycleBasePlugin.class); DependencyHandler depsHandler = project.getDependencies(); - Dependency selfDependency = depsHandler.project(Map.of("path", project.getPath())); - Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(selfDependency); + Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(); tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); - // iterate through all projects, and if the ES plugin build plugin is applied, add that project back as a dep to check + // iterate through all projects, and if the management plugin is applied, add that project back as a dep to check for (Project subProject : project.getRootProject().getSubprojects()) { - subProject.getPlugins().withType(BaseInternalPluginBuildPlugin.class).configureEach(plugin -> { + subProject.getPlugins().withType(TransportVersionManagementPlugin.class).configureEach(plugin -> { tvReferencesConfig.getDependencies().add(depsHandler.project(Map.of("path", subProject.getPath()))); }); } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index 6b11756cc8352..b168aed47f5f1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -27,12 +27,15 @@ public void apply(Project project) { t.setGroup("Transport Versions"); t.setDescription("Collects all TransportVersion references used throughout the project"); SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); - t.getClassPath().setFrom(mainSourceSet.getRuntimeClasspath()); + t.getClassPath().setFrom(mainSourceSet.getOutput()); t.getOutputFile().set(project.getLayout().getBuildDirectory().file("transport-version/references.txt")); }); - Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(); - tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + Configuration tvReferencesConfig = project.getConfigurations().create("transportVersionReferences", c -> { + c.setCanBeConsumed(true); + c.setCanBeResolved(false); + c.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); + }); project.getArtifacts().add(tvReferencesConfig.getName(), collectTask); var validateTask = project.getTasks() diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index f5d079dddb204..65941f1a245bc 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -16,6 +16,8 @@ import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import java.io.IOException; @@ -34,9 +36,11 @@ public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTask { @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) public abstract DirectoryProperty getDefinitionsDirectory(); @InputFiles + @PathSensitive(PathSensitivity.RELATIVE) public abstract ConfigurableFileCollection getReferencesFiles(); @TaskAction diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java index bafd7bc2e594f..f5b8126977db4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionReferencesTask.java @@ -15,6 +15,8 @@ import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import java.io.IOException; @@ -28,9 +30,11 @@ public abstract class ValidateTransportVersionReferencesTask extends DefaultTask { @InputDirectory + @PathSensitive(PathSensitivity.RELATIVE) public abstract DirectoryProperty getDefinitionsDirectory(); @InputFile + @PathSensitive(PathSensitivity.RELATIVE) public abstract RegularFileProperty getReferencesFile(); @TaskAction diff --git a/x-pack/plugin/esql/compute/build.gradle b/x-pack/plugin/esql/compute/build.gradle index 66867ae668fcc..ebdf3074e9c8b 100644 --- a/x-pack/plugin/esql/compute/build.gradle +++ b/x-pack/plugin/esql/compute/build.gradle @@ -3,6 +3,7 @@ import org.elasticsearch.gradle.internal.util.SourceDirectoryCommandLineArgument apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.string-templates' apply plugin: 'elasticsearch.publish' +apply plugin: 'elasticsearch.transport-version-management' base { archivesName = 'x-pack-esql-compute' From 5aab19cb26596aa8fb0f6d05392b4d0fec480f49 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Sat, 26 Jul 2025 01:55:24 +0000 Subject: [PATCH 17/28] [CI] Auto commit changes from spotless --- .../transport/CollectTransportVersionReferencesTask.java | 4 ---- .../transport/GlobalTransportVersionManagementPlugin.java | 4 ---- 2 files changed, 8 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java index e4de8a0051366..76d0d48f0db1f 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/CollectTransportVersionReferencesTask.java @@ -15,8 +15,6 @@ import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.OutputFile; -import org.gradle.api.tasks.PathSensitive; -import org.gradle.api.tasks.PathSensitivity; import org.gradle.api.tasks.TaskAction; import org.objectweb.asm.ClassReader; import org.objectweb.asm.ClassVisitor; @@ -35,8 +33,6 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; import java.util.Set; -import java.util.jar.JarInputStream; -import java.util.zip.ZipEntry; /** * This task locates all method invocations of org.elasticsearch.TransportVersion#fromName(java.lang.String) in the diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 3d96c3868d885..9b2f666da46a1 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -9,18 +9,14 @@ package org.elasticsearch.gradle.internal.transport; -import org.elasticsearch.gradle.internal.BaseInternalPluginBuildPlugin; import org.gradle.api.Plugin; import org.gradle.api.Project; import org.gradle.api.artifacts.Configuration; -import org.gradle.api.artifacts.Dependency; import org.gradle.api.artifacts.dsl.DependencyHandler; import org.gradle.api.plugins.JavaPlugin; import org.gradle.api.tasks.Copy; import org.gradle.language.base.plugins.LifecycleBasePlugin; -import java.util.ArrayList; -import java.util.List; import java.util.Map; public class GlobalTransportVersionManagementPlugin implements Plugin { From cb6420fecb420fe7d93803aca02fdf600d3e09b8 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Mon, 28 Jul 2025 12:34:40 -0700 Subject: [PATCH 18/28] Update generate task --- .../GenerateTransportVersionDataTask.java | 141 +++++++----------- .../transport/TransportVersionUtils.java | 52 ++++++- ...lidateTransportVersionDefinitionsTask.java | 6 +- 3 files changed, 101 insertions(+), 98 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java index 4f461f366b7eb..e11e17dab88ac 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java @@ -10,25 +10,22 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Streams; -import org.elasticsearch.gradle.Version; import org.elasticsearch.gradle.VersionProperties; -import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionSetData; import org.gradle.api.DefaultTask; -import org.gradle.api.GradleException; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.provider.Property; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.TaskAction; -import org.jetbrains.annotations.NotNull; import java.io.IOException; +import java.nio.file.Path; import java.util.List; import java.util.Objects; import java.util.stream.Stream; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.LATEST_SUFFIX; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getTVSetDataFilePath; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.updateLatestFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; /** * This task generates TransportVersionSetData data files that contain information about transport versions. These files @@ -70,69 +67,45 @@ public abstract class GenerateTransportVersionDataTask extends DefaultTask { @TaskAction public void generateTransportVersionData() throws IOException { - final var tvDataDir = Objects.requireNonNull(getDataFileDirectory().getAsFile().get()); - final var tvSetName = Objects.requireNonNull(getTVName().get()); - final var minorVersion = MinorVersion.fromString(Objects.requireNonNull(getMinorVersionForTV().get())); + final Path tvDataDir = Objects.requireNonNull(getDataFileDirectory().getAsFile().get()).toPath(); + final var tvName = Objects.requireNonNull(getTVName().get()); + final var forMinorVersion = Objects.requireNonNull(getMinorVersionForTV().get()); // Get the latest transport version data for the specified minor version. - final var latestTV = TransportVersionUtils.getLatestFile(tvDataDir.toPath(), minorVersion.toString()); + final var latestTV = TransportVersionUtils.getLatestFile(tvDataDir, forMinorVersion); // Create the new version - final var mainReleaseVersion = MinorVersion.of(VersionProperties.getElasticsearchVersion()); - final var isReleaseVersionMain = minorVersion.equals(mainReleaseVersion); - int newVersion = bumpVersionNumber(latestTV.ids().getFirst(), minorVersion, isReleaseVersionMain); + final var mainReleaseVersion = VersionProperties.getElasticsearchVersion(); + final var areWeOnMain = forMinorVersion.equals(mainReleaseVersion); + int newVersion = bumpVersionNumber( + latestTV.ids().getFirst(), + areWeOnMain ? PartToBump.SERVER : PartToBump.PATCH + ); // Load the tvSetData for the specified name, if it exists - final var tvSetDataFromFile = TransportVersionUtils.readDefinitionFile(tvDataDir.toPath(), tvSetName); + final var tvSetDataFromFile = TransportVersionUtils.getDefinedFile(tvDataDir, tvName); final var tvSetFileExists = tvSetDataFromFile != null; - // Create/update the data file - if (tvSetFileExists) { - // This is not a new TVSet. We are creating a backport version for an existing TVSet. - // Check to ensure that there isn't already a TV id for this release version (e.g., if this task has been run twice). - var existingIDsForReleaseVersion = tvSetDataFromFile.ids().stream().filter(id -> { - var priorLatestID = priorLatestTVSetData.ids().getFirst(); - return priorLatestID < id && id <= newVersion; - }).toList(); - if (existingIDsForReleaseVersion.isEmpty() == false) { - throw new GradleException( - "A transport version could not be created because a preexisting one was found for this name & release." - + " This could be due to another pre-existing TV with the same name, or a result of running this" - + " task twice:" - + " Release version: " - + minorVersion - + " TransportVersion Id: " - + existingIDsForReleaseVersion.getFirst() - + " File: " - + getTVSetDataFilePath(tvDataDir, tvSetName) - ); - } - // 9.2:123 | 9.1:456 | 9.0:789 | - - // Update the existing data file for the backport. - new TransportVersionSetData( - tvSetName, - Streams.concat(tvSetDataFromFile.ids().stream(), Stream.of(newVersion)).sorted().toList().reversed() - ).writeToDataDir(tvDataDir); - } else { - // Create a new data file for the case where this is a new TV - new TransportVersionSetData(tvSetName, List.of(newVersion)).writeToDataDir(tvDataDir); - } + // Write the definition file. + final var ids = tvSetFileExists + ? Streams.concat(tvSetDataFromFile.ids().stream(), Stream.of(newVersion)).sorted().toList().reversed() + : List.of(newVersion); + writeDefinitionFile(tvDataDir, tvName, ids); // Update the LATEST file. - TransportVersionUtils.writeTVSetData( - tvDataDir, - formatLatestTVSetFilename(minorVersion), - new TransportVersionSetData(tvSetName, List.of(newVersion)) - ); + updateLatestFile(tvDataDir, forMinorVersion, tvName, newVersion); + } + + public enum PartToBump { + SERVER, + SUBSIDIARY, + PATCH } // TODO Do I need to remove the patch when updating the server portion? NO, but probably need some additional checks private static int bumpVersionNumber( int tvIDToBump, - MinorVersion minorVersion, - boolean majorVersionBump, - boolean isTVReleaseVersionMain + PartToBump partToBump ) { /* The TV format: @@ -144,41 +117,31 @@ private static int bumpVersionNumber( * S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories. * PP - The patch version part */ - if (isTVReleaseVersionMain) { - if (majorVersionBump) { - // Bump the major version part, set all other parts to zero. - return minorVersion.major * 1_000_000; // TODO add check that this doesn't cause overflow out of server versions - } else { - // Bump the server version part if not a major bump. - // TODO add check that this doesn't cause overflow out of server versions - // TODO Do we need to assert on the shape of the number? e.g. no patch version. - return tvIDToBump + 1000; + return switch (partToBump) { + case SERVER -> { + var newId = tvIDToBump + 1000; + if ((newId / 1000) % 100 == 0) { + throw new IllegalStateException("Insufficient server version section in TransportVersion: " + tvIDToBump + + ", Cannot bump."); + } + yield tvIDToBump + 1000; } - } else { - // bump the patch version part - return tvIDToBump + 1; // TODO add check that this doesn't cause overflow out of patch versions - } - } - - - private static String formatLatestTVSetFilename(MinorVersion minorVersion) { - return minorVersion.toString() + LATEST_SUFFIX; - } - - private record MinorVersion(int major, int minor) { - public static MinorVersion fromString(String string) { - String[] versionParts = string.split("\\."); - assert versionParts.length == 2; - return new MinorVersion(Integer.parseInt(versionParts[0]), Integer.parseInt(versionParts[1])); - } - - public static MinorVersion of(Version version) { - return new MinorVersion(version.getMajor(), version.getMinor()); - } - - @Override - public @NotNull String toString() { - return major + "." + minor; - } + case SUBSIDIARY -> { + var newId = tvIDToBump + 100; + if ((newId / 100) == 0) { + throw new IllegalStateException("Insufficient subsidiary version section in TransportVersion: " + tvIDToBump + + ", Cannot bump."); + } + yield newId; + } + case PATCH -> { + var newId = tvIDToBump + 1; + if (newId % 10 == 0) { + throw new IllegalStateException("Insufficient patch version section in TransportVersion: " + tvIDToBump + + ", Cannot bump."); + } + yield newId; + } + }; } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 05a94014987a7..116d27575ff71 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; +import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; import org.gradle.api.attributes.AttributeContainer; @@ -22,12 +23,17 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; class TransportVersionUtils { static final Attribute TRANSPORT_VERSION_REFERENCES_ATTRIBUTE = Attribute.of("transport-version-references", Boolean.class); + private static final String LATEST_DIR = "latest"; + private static final String DEFINED_DIR = "defined"; + record TransportVersionDefinition(String name, List ids) { } @@ -38,8 +44,44 @@ public String toString() { } } - static TransportVersionDefinition getLatestFile(Path latestDataDir, String majorMinor) throws IOException { - return readDefinitionFile(latestDataDir.resolve(majorMinor + ".csv"), true); + static TransportVersionDefinition getLatestFile(Path dataDir, String majorMinor) throws IOException { + return readDefinitionFile(dataDir.resolve(LATEST_DIR).resolve(majorMinor + ".csv"), true); + } + + static TransportVersionDefinition getDefinedFile(Path dataDir, String name) throws IOException { + validateNameFormat(name); + var filePath = dataDir.resolve(DEFINED_DIR).resolve(name); + if (Files.isRegularFile(filePath) == false) { + return null; + } + return readDefinitionFile(filePath, false); + } + + static void writeDefinitionFile(Path dataDir, String name, List ids) throws IOException { + validateNameFormat(name); + assert ids != null && ids.isEmpty() == false : "Ids must be non-empty"; + Files.writeString( + dataDir.resolve(DEFINED_DIR).resolve(name + ".csv"), + ids.stream().map(String::valueOf).collect(Collectors.joining(",")) + "\n", + StandardCharsets.UTF_8 + ); + } + + static void updateLatestFile(Path dataDir, String majorMinor, String name, int id) throws IOException { + validateNameFormat(name); + var path = dataDir.resolve(LATEST_DIR).resolve(majorMinor + ".csv"); + assert Files.isRegularFile(path) : "\"Latest\" file was not found at" + path + ", but is required: "; + Files.writeString( + path, + name + "," + id + "\n", + StandardCharsets.UTF_8 + ); + } + + static void validateNameFormat(String name) { + if (Pattern.compile("^\\w+$").matcher(name).matches() == false) { + throw new GradleException("The TransportVersion name must only contain underscores and alphanumeric characters."); + } } static TransportVersionDefinition readDefinitionFile(Path file, boolean nameInFile) throws IOException { @@ -53,12 +95,12 @@ static TransportVersionDefinition readDefinitionFile(Path file, boolean nameInFi try { ids.add(Integer.parseInt(parts[i])); } catch (NumberFormatException nfe) { - throw new IllegalStateException("invalid transport version file format [" + file + "]", nfe); + throw new IllegalStateException("Invalid transport version file format [" + file + "], id could not be parsed", nfe); } } if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { - throw new IOException("invalid transport version data file [" + file + "], ids are not in sorted"); + throw new IOException("Invalid transport version data file [" + file + "], ids are not in sorted"); } return new TransportVersionDefinition(name, ids); } @@ -85,6 +127,4 @@ static void addTransportVersionReferencesAttribute(AttributeContainer attributes attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "txt"); attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_REFERENCES_ATTRIBUTE, true); } - - } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index b18e663229733..211fb2a176562 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -27,7 +27,7 @@ import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readReferencesFile; /** - * Validates that each defined transport version constant is referenced by at least one project. + * Validates that each defined transport version definition file is referenced by at least one project. */ public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTask { @@ -47,8 +47,8 @@ public void validateTransportVersions() throws IOException { } try (var definitionsStream = Files.list(definitionsDir)) { - for (var constantsFile : definitionsStream.toList()) { - var tv = readDefinitionFile(constantsFile, false); + for (var definitionFile : definitionsStream.toList()) { + var tv = readDefinitionFile(definitionFile, false); if (allTvNames.contains(tv.name()) == false) { throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced"); } From 949f8364cddf784ef56e0b3064106effc34f61d0 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Mon, 28 Jul 2025 12:40:52 -0700 Subject: [PATCH 19/28] spotless --- .../GenerateTransportVersionDataTask.java | 26 +++++++++---------- .../transport/TransportVersionUtils.java | 10 +++---- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java index e11e17dab88ac..f5c392aa6ec91 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java @@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Streams; + import org.elasticsearch.gradle.VersionProperties; import org.gradle.api.DefaultTask; import org.gradle.api.file.RegularFileProperty; @@ -77,10 +78,7 @@ public void generateTransportVersionData() throws IOException { // Create the new version final var mainReleaseVersion = VersionProperties.getElasticsearchVersion(); final var areWeOnMain = forMinorVersion.equals(mainReleaseVersion); - int newVersion = bumpVersionNumber( - latestTV.ids().getFirst(), - areWeOnMain ? PartToBump.SERVER : PartToBump.PATCH - ); + int newVersion = bumpVersionNumber(latestTV.ids().getFirst(), areWeOnMain ? PartToBump.SERVER : PartToBump.PATCH); // Load the tvSetData for the specified name, if it exists final var tvSetDataFromFile = TransportVersionUtils.getDefinedFile(tvDataDir, tvName); @@ -103,10 +101,7 @@ public enum PartToBump { } // TODO Do I need to remove the patch when updating the server portion? NO, but probably need some additional checks - private static int bumpVersionNumber( - int tvIDToBump, - PartToBump partToBump - ) { + private static int bumpVersionNumber(int tvIDToBump, PartToBump partToBump) { /* The TV format: * @@ -121,24 +116,27 @@ private static int bumpVersionNumber( case SERVER -> { var newId = tvIDToBump + 1000; if ((newId / 1000) % 100 == 0) { - throw new IllegalStateException("Insufficient server version section in TransportVersion: " + tvIDToBump - + ", Cannot bump."); + throw new IllegalStateException( + "Insufficient server version section in TransportVersion: " + tvIDToBump + ", Cannot bump." + ); } yield tvIDToBump + 1000; } case SUBSIDIARY -> { var newId = tvIDToBump + 100; if ((newId / 100) == 0) { - throw new IllegalStateException("Insufficient subsidiary version section in TransportVersion: " + tvIDToBump - + ", Cannot bump."); + throw new IllegalStateException( + "Insufficient subsidiary version section in TransportVersion: " + tvIDToBump + ", Cannot bump." + ); } yield newId; } case PATCH -> { var newId = tvIDToBump + 1; if (newId % 10 == 0) { - throw new IllegalStateException("Insufficient patch version section in TransportVersion: " + tvIDToBump - + ", Cannot bump."); + throw new IllegalStateException( + "Insufficient patch version section in TransportVersion: " + tvIDToBump + ", Cannot bump." + ); } yield newId; } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 116d27575ff71..d3f8cc8b11e8d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; + import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; @@ -34,8 +35,7 @@ class TransportVersionUtils { private static final String LATEST_DIR = "latest"; private static final String DEFINED_DIR = "defined"; - record TransportVersionDefinition(String name, List ids) { - } + record TransportVersionDefinition(String name, List ids) {} record TransportVersionReference(String name, String location) { @Override @@ -71,11 +71,7 @@ static void updateLatestFile(Path dataDir, String majorMinor, String name, int i validateNameFormat(name); var path = dataDir.resolve(LATEST_DIR).resolve(majorMinor + ".csv"); assert Files.isRegularFile(path) : "\"Latest\" file was not found at" + path + ", but is required: "; - Files.writeString( - path, - name + "," + id + "\n", - StandardCharsets.UTF_8 - ); + Files.writeString(path, name + "," + id + "\n", StandardCharsets.UTF_8); } static void validateNameFormat(String name) { From 7130d8ba0437ce0ee6b3a9e2759cdc8c4573d2ce Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Mon, 28 Jul 2025 15:01:06 -0700 Subject: [PATCH 20/28] wip --- .../GenerateTransportVersionDataTask.java | 101 ++++++------------ ...lobalTransportVersionManagementPlugin.java | 2 +- .../transport/TransportVersionUtils.java | 41 ++++++- ...lidateTransportVersionDefinitionsTask.java | 33 +++++- 4 files changed, 104 insertions(+), 73 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java index f5c392aa6ec91..4e511a643bbaf 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java @@ -10,21 +10,26 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Streams; - import org.elasticsearch.gradle.VersionProperties; import org.gradle.api.DefaultTask; import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.Property; import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.TaskAction; import java.io.IOException; import java.nio.file.Path; import java.util.List; import java.util.Objects; +import java.util.function.Function; import java.util.stream.Stream; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getDefinedFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getLatestFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.updateLatestFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; @@ -50,12 +55,14 @@ public abstract class GenerateTransportVersionDataTask extends DefaultTask { * * @return */ + @Optional @InputDirectory public abstract RegularFileProperty getDataFileDirectory(); /** * Used to set the name of the TransportVersionSet for which a data file will be generated. */ + @Optional @Input public abstract Property getTVName(); @@ -63,83 +70,45 @@ public abstract class GenerateTransportVersionDataTask extends DefaultTask { * Used to set the `major.minor` release version for which the specific TransportVersion ID will be generated. * E.g.: "9.2", "8.18", etc. */ + @Optional + @Input + public abstract ListProperty getMinorVersionsForTV(); + + @Optional @Input - public abstract Property getMinorVersionForTV(); + public abstract Property> getIdIncrementSupplier(); + @TaskAction public void generateTransportVersionData() throws IOException { final Path tvDataDir = Objects.requireNonNull(getDataFileDirectory().getAsFile().get()).toPath(); final var tvName = Objects.requireNonNull(getTVName().get()); - final var forMinorVersion = Objects.requireNonNull(getMinorVersionForTV().get()); + final var forMinorVersions = Objects.requireNonNull(getMinorVersionsForTV().get()); - // Get the latest transport version data for the specified minor version. - final var latestTV = TransportVersionUtils.getLatestFile(tvDataDir, forMinorVersion); + // TODO + // - do we need to validate that the minorVersions don't contain duplicates? + // - is there an order we need to apply? ( I don't think so) - // Create the new version - final var mainReleaseVersion = VersionProperties.getElasticsearchVersion(); - final var areWeOnMain = forMinorVersion.equals(mainReleaseVersion); - int newVersion = bumpVersionNumber(latestTV.ids().getFirst(), areWeOnMain ? PartToBump.SERVER : PartToBump.PATCH); + for (var forMinorVersion : forMinorVersions) { + // Get the latest transport version data for the specified minor version. + final var latestTV = getLatestFile(tvDataDir, forMinorVersion); - // Load the tvSetData for the specified name, if it exists - final var tvSetDataFromFile = TransportVersionUtils.getDefinedFile(tvDataDir, tvName); - final var tvSetFileExists = tvSetDataFromFile != null; + // Create the new version - // Write the definition file. - final var ids = tvSetFileExists - ? Streams.concat(tvSetDataFromFile.ids().stream(), Stream.of(newVersion)).sorted().toList().reversed() - : List.of(newVersion); - writeDefinitionFile(tvDataDir, tvName, ids); + // TODO - // Update the LATEST file. - updateLatestFile(tvDataDir, forMinorVersion, tvName, newVersion); - } + // Load the tvSetData for the specified name, if it exists + final var tvSetDataFromFile = getDefinedFile(tvDataDir, tvName); + final var tvSetFileExists = tvSetDataFromFile != null; - public enum PartToBump { - SERVER, - SUBSIDIARY, - PATCH - } + // Write the definition file. + final var ids = tvSetFileExists + ? Streams.concat(tvSetDataFromFile.ids().stream(), Stream.of(newVersion)).sorted().toList().reversed() + : List.of(newVersion); + writeDefinitionFile(tvDataDir, tvName, ids); - // TODO Do I need to remove the patch when updating the server portion? NO, but probably need some additional checks - private static int bumpVersionNumber(int tvIDToBump, PartToBump partToBump) { - - /* The TV format: - * - * MM_NNN_S_PP - * - * M - The major version of Elasticsearch - * NNN - The server version part - * S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories. - * PP - The patch version part - */ - return switch (partToBump) { - case SERVER -> { - var newId = tvIDToBump + 1000; - if ((newId / 1000) % 100 == 0) { - throw new IllegalStateException( - "Insufficient server version section in TransportVersion: " + tvIDToBump + ", Cannot bump." - ); - } - yield tvIDToBump + 1000; - } - case SUBSIDIARY -> { - var newId = tvIDToBump + 100; - if ((newId / 100) == 0) { - throw new IllegalStateException( - "Insufficient subsidiary version section in TransportVersion: " + tvIDToBump + ", Cannot bump." - ); - } - yield newId; - } - case PATCH -> { - var newId = tvIDToBump + 1; - if (newId % 10 == 0) { - throw new IllegalStateException( - "Insufficient patch version section in TransportVersion: " + tvIDToBump + ", Cannot bump." - ); - } - yield newId; - } - }; + // Update the LATEST file. + updateLatestFile(tvDataDir, forMinorVersion, tvName, newVersion); + } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 0afc6252e68ef..6891e1cd188eb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -47,7 +47,7 @@ public void apply(Project project) { .register("validateTransportVersionConstants", ValidateTransportVersionDefinitionsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); - t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); + t.getTVDataDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getReferencesFiles().setFrom(tvReferencesConfig); }); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index d3f8cc8b11e8d..4810d0c0df8fa 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -10,7 +10,6 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; - import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; @@ -32,8 +31,8 @@ class TransportVersionUtils { static final Attribute TRANSPORT_VERSION_REFERENCES_ATTRIBUTE = Attribute.of("transport-version-references", Boolean.class); - private static final String LATEST_DIR = "latest"; - private static final String DEFINED_DIR = "defined"; + static final String LATEST_DIR = "latest"; + static final String DEFINED_DIR = "defined"; record TransportVersionDefinition(String name, List ids) {} @@ -123,4 +122,40 @@ static void addTransportVersionReferencesAttribute(AttributeContainer attributes attributes.attribute(ARTIFACT_TYPE_ATTRIBUTE, "txt"); attributes.attribute(TransportVersionUtils.TRANSPORT_VERSION_REFERENCES_ATTRIBUTE, true); } + + /** + * Specifies which part of the TransportVersion id to bump. The TV format: + *

+ * MM_NNN_S_PP + *

+ * M - The major version of Elasticsearch + * NNN - The server version part + * S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories. + * PP - The patch version part + */ + public enum IdIncrement { + MAJOR(1_000_0_00, 2), + SERVER(1_0_00, 3), + SUBSIDIARY(1_00, 1), + PATCH(1, 2); + + private final int value; + private final int max; + + IdIncrement(int value, int numDigits) { + this.value = value; + this.max = (int) Math.pow(10, numDigits); + } + + public int bumpVersionNumber(int tvIDToBump) { + int zeroesCleared = (tvIDToBump / value) * value; + int newId = zeroesCleared + value; + if ((newId / value) % max == 0) { + throw new IllegalStateException( + "Insufficient" + name() + " version section in TransportVersion: " + tvIDToBump + ", Cannot bump." + ); + } + return newId; + } + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index 211fb2a176562..5a174a8536ae2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -9,6 +9,7 @@ package org.elasticsearch.gradle.internal.transport; +import com.google.common.collect.Comparators; import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionReference; import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; @@ -20,9 +21,11 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Comparator; import java.util.HashSet; import java.util.Set; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.DEFINED_DIR; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readDefinitionFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readReferencesFile; @@ -32,27 +35,51 @@ public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTask { @InputDirectory - public abstract DirectoryProperty getDefinitionsDirectory(); + public abstract DirectoryProperty getTVDataDirectory(); @InputFiles public abstract ConfigurableFileCollection getReferencesFiles(); @TaskAction public void validateTransportVersions() throws IOException { - Path definitionsDir = getDefinitionsDirectory().getAsFile().get().toPath(); + Path dataDir = getTVDataDirectory().getAsFile().get().toPath(); + Path definitionsDir = dataDir.resolve(DEFINED_DIR); Set allTvNames = new HashSet<>(); for (var tvReferencesFile : getReferencesFiles()) { readReferencesFile(tvReferencesFile.toPath()).stream().map(TransportVersionReference::name).forEach(allTvNames::add); } + // TODO validate that all files: + // - have only have a single ID per release version + // - have TVs in order + // - have the correct name + // - have the correct data format + // - Don't have duplicate IDs across any files + // - no duplicate names? Should be impossible due to filename conflicts + + try (var definitionsStream = Files.list(definitionsDir)) { for (var definitionFile : definitionsStream.toList()) { + // Validate that all definitions are referenced in the code. var tv = readDefinitionFile(definitionFile, false); if (allTvNames.contains(tv.name()) == false) { - throw new IllegalStateException("Transport version constant " + tv.name() + " is not referenced"); + throw new IllegalStateException("Transport version definition " + tv.name() + + " in file " + definitionFile + "is not referenced in the code."); } + + // Validate that all Ids are in decending order: + if (Comparators.isInOrder(tv.ids(), Comparator.reverseOrder()) == false) { + throw new IllegalStateException("Transport version definition file " + definitionFile + + " does not have ordered ids"); + } + + + + } } } + + } From 4e192547f0c94c200faf43bde9752e86aa2311bd Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Tue, 29 Jul 2025 14:51:30 -0700 Subject: [PATCH 21/28] iter --- .../GenerateTransportVersionDataTask.java | 58 ++++++++++++------- .../transport/TransportVersionUtils.java | 43 +++++++++++--- ...lidateTransportVersionDefinitionsTask.java | 26 ++++----- 3 files changed, 85 insertions(+), 42 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java index 4e511a643bbaf..8564a65f7dd52 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java @@ -9,8 +9,6 @@ package org.elasticsearch.gradle.internal.transport; -import com.google.common.collect.Streams; -import org.elasticsearch.gradle.VersionProperties; import org.gradle.api.DefaultTask; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.provider.ListProperty; @@ -22,14 +20,17 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.function.Function; -import java.util.stream.Stream; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getDefinedFile; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getLatestFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getLatestId; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getPriorLatestId; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.updateLatestFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; @@ -78,37 +79,54 @@ public abstract class GenerateTransportVersionDataTask extends DefaultTask { @Input public abstract Property> getIdIncrementSupplier(); - @TaskAction public void generateTransportVersionData() throws IOException { final Path tvDataDir = Objects.requireNonNull(getDataFileDirectory().getAsFile().get()).toPath(); final var tvName = Objects.requireNonNull(getTVName().get()); final var forMinorVersions = Objects.requireNonNull(getMinorVersionsForTV().get()); + final var idIncrementSupplier = Objects.requireNonNull(getIdIncrementSupplier().get()); // TODO - // - do we need to validate that the minorVersions don't contain duplicates? - // - is there an order we need to apply? ( I don't think so) - + // - do we need to also validate that the minorVersions don't contain duplicates here? How do we enforce idempotency if we don't? + // - is there an order we need to apply? ( I don't think so) + // - Do we need to run this iteratively for backport construction, rather than accepting a list like this? (I don't think so) + // - also parse args if run alone + // - check that duplicate tags don't come in too + + // Load the tvSetData for the specified name, if it exists + final var tvDefinition = getDefinedFile(tvDataDir, tvName); + boolean tvDefinitionExists = tvDefinition != null; + final List definitionIds = tvDefinitionExists ? tvDefinition.ids() : List.of(); + + var seenIds = new HashSet(); + var ids = new ArrayList<>(definitionIds); for (var forMinorVersion : forMinorVersions) { // Get the latest transport version data for the specified minor version. - final var latestTV = getLatestFile(tvDataDir, forMinorVersion); + final int latestTV = getLatestId(tvDataDir, forMinorVersion); // Create the new version + final int newVersion = idIncrementSupplier.apply(forMinorVersion).bumpVersionNumber(latestTV); - // TODO - - // Load the tvSetData for the specified name, if it exists - final var tvSetDataFromFile = getDefinedFile(tvDataDir, tvName); - final var tvSetFileExists = tvSetDataFromFile != null; - - // Write the definition file. - final var ids = tvSetFileExists - ? Streams.concat(tvSetDataFromFile.ids().stream(), Stream.of(newVersion)).sorted().toList().reversed() - : List.of(newVersion); - writeDefinitionFile(tvDataDir, tvName, ids); + // Check that we don't already have an ID for this minor version + var priorLatestID = getPriorLatestId(tvDataDir, forMinorVersion); + if (containsValueInRange(priorLatestID, newVersion, ids)) { + // TODO: Should we log something here? + continue; + } // Update the LATEST file. updateLatestFile(tvDataDir, forMinorVersion, tvName, newVersion); } + + writeDefinitionFile(tvDataDir, tvName, ids.stream().sorted(Comparator.reverseOrder()).toList()); + } + + private boolean containsValueInRange(int lowerExclusive, int upperInclusive, List ids) { + for (var id : ids) { + if (lowerExclusive < id && id <= upperInclusive) { + return true; + } + } + return false; } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 4810d0c0df8fa..9d1b3430805fd 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -1,15 +1,16 @@ /* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ +* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +* or more contributor license agreements. Licensed under the "Elastic License +* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side +* Public License v 1"; you may not use this file except in compliance with, at +* your election, the "Elastic License 2.0", the "GNU Affero General Public +* License v3.0 only", or the "Server Side Public License, v 1". +*/ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; + import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; @@ -23,8 +24,10 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Objects; import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.gradle.api.artifacts.type.ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE; @@ -43,6 +46,10 @@ public String toString() { } } + static int getLatestId(Path dataDir, String majorMinor) throws IOException { + return getLatestFile(dataDir, majorMinor).ids().getFirst(); + } + static TransportVersionDefinition getLatestFile(Path dataDir, String majorMinor) throws IOException { return readDefinitionFile(dataDir.resolve(LATEST_DIR).resolve(majorMinor + ".csv"), true); } @@ -158,4 +165,26 @@ public int bumpVersionNumber(int tvIDToBump) { return newId; } } + + public static int getPriorLatestId(Path dataDir, String majorMinor) throws IOException { + var pattern = Pattern.compile("^(\\d+)\\.(\\d+)\\.csv$"); + + var matcher = pattern.matcher(majorMinor); + var major = Integer.parseInt(matcher.group(1)); + var minor = Integer.parseInt(matcher.group(2)); + if (minor > 0) { + return getLatestId(dataDir, major + "." + (minor - 1)); + } + + try (var pathStream = Files.list(Objects.requireNonNull(dataDir.resolve(LATEST_DIR)))) { + var highestMinorOfPrevMajor = pathStream.flatMap(path -> { + var m = pattern.matcher(path.getFileName().toString()); + var fileMajor = Integer.parseInt(m.group(1)); + var fileMinor = Integer.parseInt(m.group(2)); + return fileMajor == major - 1 ? Stream.of(fileMinor) : Stream.empty(); + }).sorted().toList().getLast(); + return getLatestId(dataDir, (major - 1) + "." + highestMinorOfPrevMajor); + } + } + } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index 5a174a8536ae2..ec337a24125ec 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; + import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionReference; import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; @@ -51,35 +52,30 @@ public void validateTransportVersions() throws IOException { } // TODO validate that all files: - // - have only have a single ID per release version - // - have TVs in order - // - have the correct name - // - have the correct data format - // - Don't have duplicate IDs across any files - // - no duplicate names? Should be impossible due to filename conflicts - + // - have only have a single ID per release version + // - have TVs in order + // - have the correct name + // - have the correct data format + // - Don't have duplicate IDs across any files + // - no duplicate names? Should be impossible due to filename conflicts try (var definitionsStream = Files.list(definitionsDir)) { for (var definitionFile : definitionsStream.toList()) { // Validate that all definitions are referenced in the code. var tv = readDefinitionFile(definitionFile, false); if (allTvNames.contains(tv.name()) == false) { - throw new IllegalStateException("Transport version definition " + tv.name() - + " in file " + definitionFile + "is not referenced in the code."); + throw new IllegalStateException( + "Transport version definition " + tv.name() + " in file " + definitionFile + "is not referenced in the code." + ); } // Validate that all Ids are in decending order: if (Comparators.isInOrder(tv.ids(), Comparator.reverseOrder()) == false) { - throw new IllegalStateException("Transport version definition file " + definitionFile - + " does not have ordered ids"); + throw new IllegalStateException("Transport version definition file " + definitionFile + " does not have ordered ids"); } - - - } } } - } From 1cfa4121824cc7441a71cbe6a86d5e340a780602 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 29 Jul 2025 08:34:18 -0700 Subject: [PATCH 22/28] wip --- .../transport/GlobalTransportVersionManagementPlugin.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 9b2f666da46a1..622c1aa2a349c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -26,7 +26,9 @@ public void apply(Project project) { project.getPluginManager().apply(LifecycleBasePlugin.class); DependencyHandler depsHandler = project.getDependencies(); - Configuration tvReferencesConfig = project.getConfigurations().detachedConfiguration(); + Configuration tvReferencesConfig = project.getConfigurations().create("globalTvReferences"); + tvReferencesConfig.setCanBeConsumed(false); + tvReferencesConfig.setCanBeResolved(true); tvReferencesConfig.attributes(TransportVersionUtils::addTransportVersionReferencesAttribute); // iterate through all projects, and if the management plugin is applied, add that project back as a dep to check @@ -48,12 +50,12 @@ public void apply(Project project) { var generateManifestTask = project.getTasks() .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { t.setGroup("Transport Versions"); - t.setDescription("Generate a manifest resource for all the known transport version constants"); + t.setDescription("Generate a manifest resource for all the known transport version definitions"); t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt")); }); project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> { - t.into("transport/constants", c -> c.from(generateManifestTask)); + t.into("transport/defined", c -> c.from(generateManifestTask)); }); } } From 59ffce92bed898b8bf58c65ca3ad70e950e2ce67 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Wed, 30 Jul 2025 11:54:09 -0700 Subject: [PATCH 23/28] iter --- .../GenerateTransportVersionDataTask.java | 107 ++++++++++++------ .../transport/TransportVersionUtils.java | 40 ++++--- 2 files changed, 101 insertions(+), 46 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java index 8564a65f7dd52..d4b98d15d3fcb 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java @@ -9,6 +9,8 @@ package org.elasticsearch.gradle.internal.transport; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.MajorMinor; import org.gradle.api.DefaultTask; import org.gradle.api.file.RegularFileProperty; import org.gradle.api.provider.ListProperty; @@ -17,17 +19,18 @@ import org.gradle.api.tasks.InputDirectory; import org.gradle.api.tasks.Optional; import org.gradle.api.tasks.TaskAction; +import org.gradle.api.tasks.options.Option; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; -import java.util.HashSet; import java.util.List; import java.util.Objects; -import java.util.function.Function; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.PATCH; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.SERVER; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getDefinedFile; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getLatestId; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getPriorLatestId; @@ -56,16 +59,14 @@ public abstract class GenerateTransportVersionDataTask extends DefaultTask { * * @return */ - @Optional @InputDirectory - public abstract RegularFileProperty getDataFileDirectory(); + public abstract RegularFileProperty getDataFileDirectory(); // The plugin should always set this, not optional /** * Used to set the name of the TransportVersionSet for which a data file will be generated. */ - @Optional @Input - public abstract Property getTVName(); + public abstract Property getTVName(); // The plugin should always set this, not optional /** * Used to set the `major.minor` release version for which the specific TransportVersion ID will be generated. @@ -75,52 +76,85 @@ public abstract class GenerateTransportVersionDataTask extends DefaultTask { @Input public abstract ListProperty getMinorVersionsForTV(); + // TODO can the Option annotation be on getMinorVersionsForTV() so that we don't have a separate getter? @Optional @Input - public abstract Property> getIdIncrementSupplier(); + @Option(option = "versions", description = "The minor version(s) for which to generate IDs, e.g. -Pversions=\"9.2,9.1\"") + public abstract ListProperty getMinorVersionsForTVCmdLine(); + +// @Optional +// @Input +// public abstract Property> getIdIncrementSupplier(); @TaskAction public void generateTransportVersionData() throws IOException { final Path tvDataDir = Objects.requireNonNull(getDataFileDirectory().getAsFile().get()).toPath(); - final var tvName = Objects.requireNonNull(getTVName().get()); - final var forMinorVersions = Objects.requireNonNull(getMinorVersionsForTV().get()); - final var idIncrementSupplier = Objects.requireNonNull(getIdIncrementSupplier().get()); + final String tvName = Objects.requireNonNull(getTVName().get()); + final var forMinorVersions = getMinorVersionsForTV().getOrElse( + Objects.requireNonNull(getMinorVersionsForTVCmdLine().get()) + ); +// final var idIncrementSupplier = Objects.requireNonNull(getIdIncrementSupplier().get()); // TODO - // - do we need to also validate that the minorVersions don't contain duplicates here? How do we enforce idempotency if we don't? - // - is there an order we need to apply? ( I don't think so) - // - Do we need to run this iteratively for backport construction, rather than accepting a list like this? (I don't think so) - // - also parse args if run alone - // - check that duplicate tags don't come in too + // - [x] do we need to also validate that the minorVersions don't contain duplicates here? How do we enforce idempotency if we don't? + // - is there an order we need to apply? ( I don't think so) + // - Do we need to run this iteratively for backport construction, rather than accepting a list like this? (I don't think so) + // - [x] parse args if run alone + // - check that duplicate versions don't come in? + // - Check that we don't have duplicate names (elsewhere, not here) + // - Do we need to allow creating only patch versions? + // - Must also keep data in sync for removal. + // - We could remove any TVs not associated with a version arg. We then either generate or keep any tvs + // for each version arg, and discard the rest + // - How will this work for follow-up backport PRs that will not have all the version labels? + // - The follow up PR somehow needs to know original IDs. Look at git? Need a new task? + // - // Load the tvSetData for the specified name, if it exists final var tvDefinition = getDefinedFile(tvDataDir, tvName); boolean tvDefinitionExists = tvDefinition != null; - final List definitionIds = tvDefinitionExists ? tvDefinition.ids() : List.of(); + final List preexistingIds = tvDefinitionExists ? Collections.unmodifiableList(tvDefinition.ids()) : List.of(); - var seenIds = new HashSet(); - var ids = new ArrayList<>(definitionIds); - for (var forMinorVersion : forMinorVersions) { + List ids = new ArrayList<>(); + for (var forVersion : forMinorVersions.stream().map(MajorMinor::of).toList()) { // Get the latest transport version data for the specified minor version. - final int latestTV = getLatestId(tvDataDir, forMinorVersion); - - // Create the new version - final int newVersion = idIncrementSupplier.apply(forMinorVersion).bumpVersionNumber(latestTV); - - // Check that we don't already have an ID for this minor version - var priorLatestID = getPriorLatestId(tvDataDir, forMinorVersion); - if (containsValueInRange(priorLatestID, newVersion, ids)) { + final int latestTV = getLatestId(tvDataDir, forVersion.toString()); + + // Create the new version id + // final int newID = idIncrementSupplier.apply(forVersion).bumpTransportVersion(latestTV); + final int newID = incrementTVId(latestTV, forVersion); + + // Check that if we already have a TV ID for this minor version + Integer preexistingTVId = retrieveValueInRange( + getPriorLatestId(tvDataDir, forVersion.toString()), + newID, preexistingIds + ); + if (preexistingTVId != null) { + ids.add(preexistingTVId); // TODO: Should we log something here? - continue; + } else { + ids.add(newID); + // Update the LATEST file. + // TODO need to revert the latest files for anything that has been removed. + updateLatestFile(tvDataDir, forVersion.toString(), tvName, newID); } - - // Update the LATEST file. - updateLatestFile(tvDataDir, forMinorVersion, tvName, newVersion); } writeDefinitionFile(tvDataDir, tvName, ids.stream().sorted(Comparator.reverseOrder()).toList()); } + private int incrementTVId(int tvID, MajorMinor version) { + // We can only run this task on main, so the ElasticsearchVersion will be for main. + final var mainVersion = MajorMinor.of(VersionProperties.getElasticsearchVersion()); + final var isMain = version.equals(mainVersion); + if (isMain) { + return SERVER.bumpTransportVersion(tvID); + } else { + return PATCH.bumpTransportVersion(tvID); + } + // TODO add serverless check + } + private boolean containsValueInRange(int lowerExclusive, int upperInclusive, List ids) { for (var id : ids) { if (lowerExclusive < id && id <= upperInclusive) { @@ -129,4 +163,13 @@ private boolean containsValueInRange(int lowerExclusive, int upperInclusive, Lis } return false; } + + private Integer retrieveValueInRange(int lowerExclusive, int upperInclusive, List ids) { + for (var id : ids) { + if (lowerExclusive < id && id <= upperInclusive) { + return id; + } + } + return null; + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 9d1b3430805fd..9c5a7d4b9c5a6 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -10,12 +10,13 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; - +import org.elasticsearch.gradle.Version; import org.gradle.api.GradleException; import org.gradle.api.Project; import org.gradle.api.attributes.Attribute; import org.gradle.api.attributes.AttributeContainer; import org.gradle.api.file.Directory; +import org.jetbrains.annotations.NotNull; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -154,7 +155,7 @@ public enum IdIncrement { this.max = (int) Math.pow(10, numDigits); } - public int bumpVersionNumber(int tvIDToBump) { + public int bumpTransportVersion(int tvIDToBump) { int zeroesCleared = (tvIDToBump / value) * value; int newId = zeroesCleared + value; if ((newId / value) % max == 0) { @@ -166,24 +167,35 @@ public int bumpVersionNumber(int tvIDToBump) { } } - public static int getPriorLatestId(Path dataDir, String majorMinor) throws IOException { - var pattern = Pattern.compile("^(\\d+)\\.(\\d+)\\.csv$"); + public record MajorMinor(int major, int minor) { + public static MajorMinor of(String majorMinor) { + String[] versionParts = majorMinor.split("\\."); + assert versionParts.length == 2; + return new MajorMinor(Integer.parseInt(versionParts[0]), Integer.parseInt(versionParts[1])); + } - var matcher = pattern.matcher(majorMinor); - var major = Integer.parseInt(matcher.group(1)); - var minor = Integer.parseInt(matcher.group(2)); - if (minor > 0) { - return getLatestId(dataDir, major + "." + (minor - 1)); + public static MajorMinor of(Version version) { + return new MajorMinor(version.getMajor(), version.getMinor()); } + @Override + public @NotNull String toString() { + return major + "." + minor; + } + } + + public static int getPriorLatestId(Path dataDir, String majorMinor) throws IOException { + var version = MajorMinor.of(majorMinor); + if (version.minor() > 0) { + return getLatestId(dataDir, version.major + "." + (version.minor - 1)); + } try (var pathStream = Files.list(Objects.requireNonNull(dataDir.resolve(LATEST_DIR)))) { var highestMinorOfPrevMajor = pathStream.flatMap(path -> { - var m = pattern.matcher(path.getFileName().toString()); - var fileMajor = Integer.parseInt(m.group(1)); - var fileMinor = Integer.parseInt(m.group(2)); - return fileMajor == major - 1 ? Stream.of(fileMinor) : Stream.empty(); + var fileMajorMinor = path.getFileName().toString().replace(".csv", ""); + var fileVersion = MajorMinor.of(fileMajorMinor); + return fileVersion.major == version.major - 1 ? Stream.of(fileVersion.minor) : Stream.empty(); }).sorted().toList().getLast(); - return getLatestId(dataDir, (major - 1) + "." + highestMinorOfPrevMajor); + return getLatestId(dataDir, (version.major - 1) + "." + highestMinorOfPrevMajor); } } From 3dd41ffcb1a7db749e30b1da5ffb5ab7aaa6fb46 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 30 Jul 2025 12:42:23 -0700 Subject: [PATCH 24/28] remove constants --- server/src/main/java/org/elasticsearch/TransportVersions.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 182c6a766dfb4..397ccefd8b955 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -211,7 +211,6 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_COHERE_API_VERSION_8_19 = def(8_841_0_60); public static final TransportVersion ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19 = def(8_841_0_61); public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN_8_19 = def(8_841_0_62); - public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_8_19 = def(8_841_0_63); public static final TransportVersion ESQL_FIXED_INDEX_LIKE_8_19 = def(8_841_0_64); public static final TransportVersion V_9_0_0 = def(9_000_0_09); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10); @@ -330,7 +329,6 @@ static TransportVersion def(int id) { public static final TransportVersion ML_INFERENCE_COHERE_API_VERSION = def(9_110_0_00); public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN = def(9_111_0_00); public static final TransportVersion MAPPINGS_IN_DATA_STREAMS = def(9_112_0_00); - public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_9_1 = def(9_112_0_01); public static final TransportVersion ESQL_FIXED_INDEX_LIKE_9_1 = def(9_112_0_02); public static final TransportVersion ESQL_SAMPLE_OPERATOR_STATUS_9_1 = def(9_112_0_03); // Below is the first version in 9.2 and NOT in 9.1. @@ -338,7 +336,6 @@ static TransportVersion def(int id) { public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00); public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00); public static final TransportVersion ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = def(9_115_0_00); - // public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); public static final TransportVersion ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS = def(9_117_0_00); public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE = def(9_118_0_00); public static final TransportVersion ESQL_FIXED_INDEX_LIKE = def(9_119_0_00); From faab72830d55395e3702953ac2ac63a089e01855 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 30 Jul 2025 16:12:21 -0700 Subject: [PATCH 25/28] wip, worked on generate task --- ...nerateTransportVersionDefinitionTask.java} | 95 +++++++++++++------ ...lobalTransportVersionManagementPlugin.java | 8 +- .../TransportVersionManagementPlugin.java | 2 +- .../transport/TransportVersionUtils.java | 14 ++- 4 files changed, 82 insertions(+), 37 deletions(-) rename build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/{GenerateTransportVersionDataTask.java => GenerateTransportVersionDefinitionTask.java} (66%) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java similarity index 66% rename from build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java rename to build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index d4b98d15d3fcb..cf104781de243 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -11,8 +11,9 @@ import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.MajorMinor; +import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionDefinition; import org.gradle.api.DefaultTask; -import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.file.DirectoryProperty; import org.gradle.api.provider.ListProperty; import org.gradle.api.provider.Property; import org.gradle.api.tasks.Input; @@ -26,8 +27,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Objects; +import java.util.Set; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.PATCH; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.SERVER; @@ -38,21 +41,12 @@ import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; /** - * This task generates TransportVersionSetData data files that contain information about transport versions. These files - * are added to the server project's resource directory at `server/src/main/resources/org/elasticsearch/transport/`. - * They have the following format: - * 

- * Filename: my-transport-version-set.json  // Must be the same as the name of the transport version set.
- * {
- *   "name": "my-transport-version-set", // The name of the transport version set used for reference in the code.
- *   "ids": [
- *     9109000,  // The transport version introduced to the main branch.
- *     8841059   // The transport version backported to a previous release branch.
- *   ]
- * }
- *
+ * This task generates transport version definition files. These files + * are runtime resources that TransportVersion loads statically. + * They contain a comma separated list of integer ids. Each file is named the same + * as the transport version name itself (with the .csv suffix). */ -public abstract class GenerateTransportVersionDataTask extends DefaultTask { +public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask { /** * Specifies the directory in which contains all TransportVersionSet data files. @@ -60,13 +54,17 @@ public abstract class GenerateTransportVersionDataTask extends DefaultTask { * @return */ @InputDirectory - public abstract RegularFileProperty getDataFileDirectory(); // The plugin should always set this, not optional + public abstract DirectoryProperty getTransportResourcesDirectory(); // The plugin should always set this, not optional + + // assumption: this task is always run on main, so we can determine the name by diffing with main and looking for new files added in the definition directory /** * Used to set the name of the TransportVersionSet for which a data file will be generated. */ @Input - public abstract Property getTVName(); // The plugin should always set this, not optional + @Optional + @Option(option = "name", description = "TBD") + public abstract Property getTransportVersionName(); // The plugin should always set this, not optional /** * Used to set the `major.minor` release version for which the specific TransportVersion ID will be generated. @@ -74,13 +72,8 @@ public abstract class GenerateTransportVersionDataTask extends DefaultTask { */ @Optional @Input - public abstract ListProperty getMinorVersionsForTV(); - - // TODO can the Option annotation be on getMinorVersionsForTV() so that we don't have a separate getter? - @Optional - @Input - @Option(option = "versions", description = "The minor version(s) for which to generate IDs, e.g. -Pversions=\"9.2,9.1\"") - public abstract ListProperty getMinorVersionsForTVCmdLine(); + @Option(option = "versions", description = "The minor version(s) for which to generate IDs, e.g. --versions=\"9.2,9.1\"") + public abstract ListProperty getMinorVersions(); // @Optional // @Input @@ -88,11 +81,36 @@ public abstract class GenerateTransportVersionDataTask extends DefaultTask { @TaskAction public void generateTransportVersionData() throws IOException { - final Path tvDataDir = Objects.requireNonNull(getDataFileDirectory().getAsFile().get()).toPath(); - final String tvName = Objects.requireNonNull(getTVName().get()); - final var forMinorVersions = getMinorVersionsForTV().getOrElse( - Objects.requireNonNull(getMinorVersionsForTVCmdLine().get()) - ); + getLogger().lifecycle("Name: " + getTransportVersionName().get()); + getLogger().lifecycle("Versions: " + getMinorVersions().get()); + Path resourcesDir = Objects.requireNonNull(getTransportResourcesDirectory().getAsFile().get()).toPath(); + String name = getTransportVersionName().isPresent() ? getTransportVersionName().get() : findLocalTransportVersionName(); + Set targetMinorVersions = new HashSet<>(getMinorVersions().isPresent() ? getMinorVersions().get() : findTargetMinorVersions()); + + List ids = new ArrayList<>(); + for (String minorVersion : getKnownMinorVersions(resourcesDir)) { + TransportVersionDefinition latest = TransportVersionUtils.getLatestFile(resourcesDir, minorVersion); + TransportVersionDefinition newLatest = null; + + if (name.equals(latest.name())) { + if (targetMinorVersions.contains(minorVersion) == false) { + // regenerate + } + } else { + if (targetMinorVersions.contains(minorVersion)) { + // increment + } + } + + if (newLatest != null) { + assert newLatest.ids().size() == 1; + TransportVersionUtils.updateLatestFile(resourcesDir, minorVersion, newLatest.name(), newLatest.ids().getFirst()); + } + } + +/* + final String tvName = Objects.requireNonNull(getTransportVersionName().get()); + List minorVersions = getMinorVersions().get(); // final var idIncrementSupplier = Objects.requireNonNull(getIdIncrementSupplier().get()); // TODO @@ -141,6 +159,7 @@ public void generateTransportVersionData() throws IOException { } writeDefinitionFile(tvDataDir, tvName, ids.stream().sorted(Comparator.reverseOrder()).toList()); + */ } private int incrementTVId(int tvID, MajorMinor version) { @@ -172,4 +191,22 @@ private Integer retrieveValueInRange(int lowerExclusive, int upperInclusive, Lis } return null; } + + private List getKnownMinorVersions(Path resourcesDir) { + // list files under latest + return List.of(); + } + + private String findLocalTransportVersionName() { + // check for missing + // if none missing, look at git diff against main + return ""; + } + + private List findTargetMinorVersions() { + // look for env var indicating github PR link from CI + // use github api to find current labels, filter down to version labels + // map version labels to branches + return List.of(); + } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index 622c1aa2a349c..d0409fb350063 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -42,7 +42,7 @@ public void apply(Project project) { .register("validateTransportVersionDefinitions", ValidateTransportVersionDefinitionsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); - t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); + t.getDefinitionsDirectory().set(TransportVersionUtils.getTransportDefinitionsDirectory(project)); t.getReferencesFiles().setFrom(tvReferencesConfig); }); project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); @@ -51,11 +51,15 @@ public void apply(Project project) { .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Generate a manifest resource for all the known transport version definitions"); - t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); + t.getDefinitionsDirectory().set(TransportVersionUtils.getTransportDefinitionsDirectory(project)); t.getManifestFile().set(project.getLayout().getBuildDirectory().file("generated-resources/manifest.txt")); }); project.getTasks().named(JavaPlugin.PROCESS_RESOURCES_TASK_NAME, Copy.class).configure(t -> { t.into("transport/defined", c -> c.from(generateManifestTask)); }); + + project.getTasks().register("generateTransportVersionDefinition", GenerateTransportVersionDefinitionTask.class, t -> { + t.getTransportResourcesDirectory().set(TransportVersionUtils.getTransportResourcesDirectory(project)); + }); } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java index b168aed47f5f1..5020292c5be34 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionManagementPlugin.java @@ -42,7 +42,7 @@ public void apply(Project project) { .register("validateTransportVersionReferences", ValidateTransportVersionReferencesTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all TransportVersion references used in the project have an associated definition file"); - t.getDefinitionsDirectory().set(TransportVersionUtils.getDefinitionsDirectory(project)); + t.getDefinitionsDirectory().set(TransportVersionUtils.getTransportDefinitionsDirectory(project)); t.getReferencesFile().set(collectTask.get().getOutputFile()); }); project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 785c58304840c..7f2780a8fecce 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -51,8 +51,8 @@ static int getLatestId(Path dataDir, String majorMinor) throws IOException { return getLatestFile(dataDir, majorMinor).ids().getFirst(); } - static TransportVersionDefinition getLatestFile(Path dataDir, String majorMinor) throws IOException { - return readDefinitionFile(dataDir.resolve(LATEST_DIR).resolve(majorMinor + ".csv"), true); + static TransportVersionDefinition getLatestFile(Path resourcesDir, String minorVersion) throws IOException { + return readDefinitionFile(resourcesDir.resolve(LATEST_DIR).resolve(minorVersion + ".csv"), true); } static TransportVersionDefinition getDefinedFile(Path dataDir, String name) throws IOException { @@ -121,13 +121,17 @@ static List readReferencesFile(Path file) throws IOEx return results; } - static Directory getDefinitionsDirectory(Project project) { + static Directory getTransportDefinitionsDirectory(Project project) { + return getTransportResourcesDirectory(project).dir("defined"); + } + + static Directory getTransportResourcesDirectory(Project project) { var projectName = project.findProperty("org.elasticsearch.transport.definitionsProject"); if (projectName == null) { projectName = ":server"; } - Directory projectDir = project.project(projectName.toString()).getLayout().getProjectDirectory(); - return projectDir.dir("src/main/resources/transport/defined"); + Directory serverDir = project.getRootProject().project(projectName.toString()).getLayout().getProjectDirectory(); + return serverDir.dir("src/main/resources/transport"); } static void addTransportVersionReferencesAttribute(AttributeContainer attributes) { From 77da1557a6417a44f64076416ea92f27d7a768f5 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 30 Jul 2025 23:22:08 +0000 Subject: [PATCH 26/28] [CI] Auto commit changes from spotless --- ...enerateTransportVersionDefinitionTask.java | 25 ++++++++----------- .../transport/TransportVersionUtils.java | 1 + 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index cf104781de243..9d34c8def341c 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -25,8 +25,6 @@ import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -34,11 +32,6 @@ import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.PATCH; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.SERVER; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getDefinedFile; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getLatestId; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getPriorLatestId; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.updateLatestFile; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.writeDefinitionFile; /** * This task generates transport version definition files. These files @@ -56,8 +49,8 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @InputDirectory public abstract DirectoryProperty getTransportResourcesDirectory(); // The plugin should always set this, not optional - - // assumption: this task is always run on main, so we can determine the name by diffing with main and looking for new files added in the definition directory + // assumption: this task is always run on main, so we can determine the name by diffing with main and looking for new files added in the + // definition directory /** * Used to set the name of the TransportVersionSet for which a data file will be generated. */ @@ -75,9 +68,9 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask @Option(option = "versions", description = "The minor version(s) for which to generate IDs, e.g. --versions=\"9.2,9.1\"") public abstract ListProperty getMinorVersions(); -// @Optional -// @Input -// public abstract Property> getIdIncrementSupplier(); + // @Optional + // @Input + // public abstract Property> getIdIncrementSupplier(); @TaskAction public void generateTransportVersionData() throws IOException { @@ -85,7 +78,9 @@ public void generateTransportVersionData() throws IOException { getLogger().lifecycle("Versions: " + getMinorVersions().get()); Path resourcesDir = Objects.requireNonNull(getTransportResourcesDirectory().getAsFile().get()).toPath(); String name = getTransportVersionName().isPresent() ? getTransportVersionName().get() : findLocalTransportVersionName(); - Set targetMinorVersions = new HashSet<>(getMinorVersions().isPresent() ? getMinorVersions().get() : findTargetMinorVersions()); + Set targetMinorVersions = new HashSet<>( + getMinorVersions().isPresent() ? getMinorVersions().get() : findTargetMinorVersions() + ); List ids = new ArrayList<>(); for (String minorVersion : getKnownMinorVersions(resourcesDir)) { @@ -108,10 +103,10 @@ public void generateTransportVersionData() throws IOException { } } -/* + /* final String tvName = Objects.requireNonNull(getTransportVersionName().get()); List minorVersions = getMinorVersions().get(); -// final var idIncrementSupplier = Objects.requireNonNull(getIdIncrementSupplier().get()); + // final var idIncrementSupplier = Objects.requireNonNull(getIdIncrementSupplier().get()); // TODO // - [x] do we need to also validate that the minorVersions don't contain duplicates here? How do we enforce idempotency if we don't? diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 7f2780a8fecce..4d58d0f292119 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; + import org.elasticsearch.gradle.Version; import org.gradle.api.GradleException; import org.gradle.api.Project; From 99255e05c43e60a650357b0b0407cc0226da8f26 Mon Sep 17 00:00:00 2001 From: John Verwolf Date: Thu, 31 Jul 2025 14:37:48 -0700 Subject: [PATCH 27/28] iter, add more validation --- ...enerateTransportVersionDefinitionTask.java | 28 +++--- ...lobalTransportVersionManagementPlugin.java | 2 +- .../transport/TransportVersionUtils.java | 97 ++++++++++++------- ...lidateTransportVersionDefinitionsTask.java | 50 ++++++---- 4 files changed, 107 insertions(+), 70 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 9d34c8def341c..956fad80d1b96 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -11,7 +11,7 @@ import org.elasticsearch.gradle.VersionProperties; import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.MajorMinor; -import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionDefinition; +import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionLatest; import org.gradle.api.DefaultTask; import org.gradle.api.file.DirectoryProperty; import org.gradle.api.provider.ListProperty; @@ -29,6 +29,7 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.PATCH; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.IdIncrement.SERVER; @@ -50,7 +51,7 @@ public abstract class GenerateTransportVersionDefinitionTask extends DefaultTask public abstract DirectoryProperty getTransportResourcesDirectory(); // The plugin should always set this, not optional // assumption: this task is always run on main, so we can determine the name by diffing with main and looking for new files added in the - // definition directory + // definition directory. (not true: once we generate the file, this will no longer hold true if we then need to update it) /** * Used to set the name of the TransportVersionSet for which a data file will be generated. */ @@ -78,28 +79,31 @@ public void generateTransportVersionData() throws IOException { getLogger().lifecycle("Versions: " + getMinorVersions().get()); Path resourcesDir = Objects.requireNonNull(getTransportResourcesDirectory().getAsFile().get()).toPath(); String name = getTransportVersionName().isPresent() ? getTransportVersionName().get() : findLocalTransportVersionName(); - Set targetMinorVersions = new HashSet<>( - getMinorVersions().isPresent() ? getMinorVersions().get() : findTargetMinorVersions() + Set targetMinorVersions = new HashSet<>( + getMinorVersions().isPresent() + ? getMinorVersions().get().stream().map(MajorMinor::of).collect(Collectors.toSet()) + : findTargetMinorVersions() ); List ids = new ArrayList<>(); - for (String minorVersion : getKnownMinorVersions(resourcesDir)) { - TransportVersionDefinition latest = TransportVersionUtils.getLatestFile(resourcesDir, minorVersion); - TransportVersionDefinition newLatest = null; + for (MajorMinor minorVersion : getKnownMinorVersions(resourcesDir)) { + TransportVersionLatest latest = TransportVersionUtils.readLatestFile(resourcesDir, minorVersion); + TransportVersionLatest newLatest = null; + if (name.equals(latest.name())) { if (targetMinorVersions.contains(minorVersion) == false) { - // regenerate + // Regenerate to make this operation idempotent. Need to undo prior updates to the latest files if the list of minor versions has changed. } } else { if (targetMinorVersions.contains(minorVersion)) { // increment + ids.add(incrementTVId(latest.id(), minorVersion)); } } if (newLatest != null) { - assert newLatest.ids().size() == 1; - TransportVersionUtils.updateLatestFile(resourcesDir, minorVersion, newLatest.name(), newLatest.ids().getFirst()); + TransportVersionUtils.updateLatestFile(resourcesDir, minorVersion.toString(), newLatest.name(), newLatest.id()); } } @@ -187,7 +191,7 @@ private Integer retrieveValueInRange(int lowerExclusive, int upperInclusive, Lis return null; } - private List getKnownMinorVersions(Path resourcesDir) { + private List getKnownMinorVersions(Path resourcesDir) { // list files under latest return List.of(); } @@ -198,7 +202,7 @@ private String findLocalTransportVersionName() { return ""; } - private List findTargetMinorVersions() { + private List findTargetMinorVersions() { // look for env var indicating github PR link from CI // use github api to find current labels, filter down to version labels // map version labels to branches diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java index d0409fb350063..612c6bb13780e 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GlobalTransportVersionManagementPlugin.java @@ -42,7 +42,7 @@ public void apply(Project project) { .register("validateTransportVersionDefinitions", ValidateTransportVersionDefinitionsTask.class, t -> { t.setGroup("Transport Versions"); t.setDescription("Validates that all defined TransportVersion constants are used in at least one project"); - t.getDefinitionsDirectory().set(TransportVersionUtils.getTransportDefinitionsDirectory(project)); + t.getTransportResourcesDirectory().set(TransportVersionUtils.getTransportResourcesDirectory(project)); t.getReferencesFiles().setFrom(tvReferencesConfig); }); project.getTasks().named(LifecycleBasePlugin.CHECK_TASK_NAME).configure(t -> t.dependsOn(validateTask)); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 4d58d0f292119..6e7a9da8673e7 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -10,7 +10,6 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; - import org.elasticsearch.gradle.Version; import org.gradle.api.GradleException; import org.gradle.api.Project; @@ -20,10 +19,12 @@ import org.jetbrains.annotations.NotNull; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.List; import java.util.Objects; @@ -39,45 +40,90 @@ class TransportVersionUtils { static final String LATEST_DIR = "latest"; static final String DEFINED_DIR = "defined"; - record TransportVersionDefinition(String name, List ids) {} + private static final String CSV_SUFFIX = ".csv"; + + record TransportVersionDefinition(String name, List ids) { + String path(Path resourcesDir) { + return resourcesDir.resolve(DEFINED_DIR).resolve(name + CSV_SUFFIX).toString(); + } + } + + record TransportVersionLatest(MajorMinor version, String name, int id) { + String path(Path resourcesDir) { + return resourcesDir.resolve(LATEST_DIR).resolve(version.toString() + CSV_SUFFIX).toString(); + } + } record TransportVersionReference(String name, String location) { @Override - public String toString() { + public @NotNull String toString() { return name + " " + location; } } - static int getLatestId(Path dataDir, String majorMinor) throws IOException { - return getLatestFile(dataDir, majorMinor).ids().getFirst(); + static int getLatestId(Path resourcesDir, String majorMinor) throws IOException { + return readLatestFile(resourcesDir, MajorMinor.of(majorMinor)).id(); } - static TransportVersionDefinition getLatestFile(Path resourcesDir, String minorVersion) throws IOException { - return readDefinitionFile(resourcesDir.resolve(LATEST_DIR).resolve(minorVersion + ".csv"), true); + static TransportVersionLatest readLatestFile(Path resourcesDir, MajorMinor version) throws IOException { + Path filePath = resourcesDir.resolve(LATEST_DIR).resolve(version.toString() + CSV_SUFFIX); + String[] parts = Files.readString(filePath, StandardCharsets.UTF_8).split(","); + assert parts.length == 2; + return new TransportVersionLatest( + version, + parts[0], + Integer.parseInt(parts[1]) + ); } - static TransportVersionDefinition getDefinedFile(Path dataDir, String name) throws IOException { + static TransportVersionDefinition readDefinitionFile(Path resourcesDir, String name) { validateNameFormat(name); - var filePath = dataDir.resolve(DEFINED_DIR).resolve(name); + var filePath = resourcesDir.resolve(DEFINED_DIR).resolve(name + CSV_SUFFIX); if (Files.isRegularFile(filePath) == false) { + System.out.println("Potato file was not found at " + filePath); return null; } - return readDefinitionFile(filePath, false); + try { + String[] parts = Files.readString(filePath, StandardCharsets.UTF_8).split(","); + List ids = Arrays.stream(parts).map(rawId -> Integer.parseInt(rawId.strip())).toList(); + + if (ids.isEmpty()) { + throw new IllegalStateException("Invalid transport version data file [" + filePath + "], no ids"); + } + if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { + throw new IllegalStateException("Invalid transport version data file [" + filePath + "], ids are not in sorted"); + } + return new TransportVersionDefinition(name, ids); + } catch (IOException e) { + throw new UncheckedIOException("Unable to read definition file", e); + } + } + + + static Stream readAllDefinitionFiles(Path resourcesDir) throws IOException { + var definitionsStream = Files.list(resourcesDir.resolve(DEFINED_DIR)); + return definitionsStream.map(path -> { + String fileName = path.getFileName().toString(); + assert fileName.endsWith(CSV_SUFFIX); + String name = fileName.substring(0, fileName.length() - 4); + System.out.println("Potato path.getparent" + path.getParent()); + return readDefinitionFile(resourcesDir, name); + }); } - static void writeDefinitionFile(Path dataDir, String name, List ids) throws IOException { + static void writeDefinitionFile(Path resourcesDir, String name, List ids) throws IOException { validateNameFormat(name); assert ids != null && ids.isEmpty() == false : "Ids must be non-empty"; Files.writeString( - dataDir.resolve(DEFINED_DIR).resolve(name + ".csv"), + resourcesDir.resolve(DEFINED_DIR).resolve(name + CSV_SUFFIX), ids.stream().map(String::valueOf).collect(Collectors.joining(",")) + "\n", StandardCharsets.UTF_8 ); } - static void updateLatestFile(Path dataDir, String majorMinor, String name, int id) throws IOException { + static void updateLatestFile(Path resourcesDir, String majorMinor, String name, int id) throws IOException { validateNameFormat(name); - var path = dataDir.resolve(LATEST_DIR).resolve(majorMinor + ".csv"); + var path = resourcesDir.resolve(LATEST_DIR).resolve(majorMinor + CSV_SUFFIX); assert Files.isRegularFile(path) : "\"Latest\" file was not found at" + path + ", but is required: "; Files.writeString(path, name + "," + id + "\n", StandardCharsets.UTF_8); } @@ -88,27 +134,6 @@ static void validateNameFormat(String name) { } } - static TransportVersionDefinition readDefinitionFile(Path file, boolean nameInFile) throws IOException { - assert file.endsWith(".csv"); - String rawName = file.getFileName().toString(); - - String[] parts = Files.readString(file, StandardCharsets.UTF_8).split(","); - String name = nameInFile ? parts[0] : rawName.substring(0, rawName.length() - 4); - List ids = new ArrayList<>(); - for (int i = nameInFile ? 1 : 0; i < parts.length; ++i) { - try { - ids.add(Integer.parseInt(parts[i])); - } catch (NumberFormatException nfe) { - throw new IllegalStateException("Invalid transport version file format [" + file + "], id could not be parsed", nfe); - } - } - - if (Comparators.isInOrder(ids, Comparator.reverseOrder()) == false) { - throw new IOException("Invalid transport version data file [" + file + "], ids are not in sorted"); - } - return new TransportVersionDefinition(name, ids); - } - static List readReferencesFile(Path file) throws IOException { assert file.endsWith(".txt"); List results = new ArrayList<>(); @@ -200,7 +225,7 @@ public static int getPriorLatestId(Path dataDir, String majorMinor) throws IOExc } try (var pathStream = Files.list(Objects.requireNonNull(dataDir.resolve(LATEST_DIR)))) { var highestMinorOfPrevMajor = pathStream.flatMap(path -> { - var fileMajorMinor = path.getFileName().toString().replace(".csv", ""); + var fileMajorMinor = path.getFileName().toString().replace(CSV_SUFFIX, ""); var fileVersion = MajorMinor.of(fileMajorMinor); return fileVersion.major == version.major - 1 ? Stream.of(fileVersion.minor) : Stream.empty(); }).sorted().toList().getLast(); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index d3dc6a4a2ff56..b15f692106816 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -10,7 +10,6 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; - import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionReference; import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; @@ -23,15 +22,14 @@ import org.gradle.api.tasks.TaskAction; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.Comparator; import java.util.HashSet; import java.util.Set; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.DEFINED_DIR; -import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readDefinitionFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readAllDefinitionFiles; import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.readReferencesFile; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.validateNameFormat; /** * Validates that each defined transport version definition file is referenced by at least one project. @@ -41,7 +39,7 @@ public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTas @InputDirectory @PathSensitive(PathSensitivity.RELATIVE) - public abstract DirectoryProperty getDefinitionsDirectory(); + public abstract DirectoryProperty getTransportResourcesDirectory(); @InputFiles @PathSensitive(PathSensitivity.RELATIVE) @@ -49,8 +47,7 @@ public abstract class ValidateTransportVersionDefinitionsTask extends DefaultTas @TaskAction public void validateTransportVersions() throws IOException { - Path dataDir = getDefinitionsDirectory().getAsFile().get().toPath(); - Path definitionsDir = dataDir.resolve(DEFINED_DIR); + Path resourcesDir = getTransportResourcesDirectory().getAsFile().get().toPath(); Set allTvNames = new HashSet<>(); for (var tvReferencesFile : getReferencesFiles()) { @@ -59,28 +56,39 @@ public void validateTransportVersions() throws IOException { // TODO validate that all files: // - have only have a single ID per release version - // - have TVs in order - // - have the correct name + // - [x] have TVs in order + // - [x] have a name in the correct format // - have the correct data format - // - Don't have duplicate IDs across any files + // - [x] Don't have duplicate IDs across any files // - no duplicate names? Should be impossible due to filename conflicts - try (var definitionsStream = Files.list(definitionsDir)) { - for (var definitionFile : definitionsStream.toList()) { - // Validate that all definitions are referenced in the code. - var tv = readDefinitionFile(definitionFile, false); - if (allTvNames.contains(tv.name()) == false) { - throw new IllegalStateException( - "Transport version definition " + tv.name() + " in file " + definitionFile + "is not referenced in the code." - ); + HashSet seenIds = new HashSet<>(); + try (var allDefinitions = readAllDefinitionFiles(resourcesDir)) { + allDefinitions.forEach(definition -> { + // Validate that all definitions are referenced in the codebase: + if (allTvNames.contains(definition.name()) == false) { + throw new IllegalStateException("Transport version definition file " + + definition.path(resourcesDir) + " is not referenced in the codebase."); } // Validate that all Ids are in decending order: - if (Comparators.isInOrder(tv.ids(), Comparator.reverseOrder()) == false) { - throw new IllegalStateException("Transport version definition file " + definitionFile + " does not have ordered ids"); + if (Comparators.isInOrder(definition.ids(), Comparator.reverseOrder()) == false) { + throw new IllegalStateException("Transport version definition file " + + definition.path(resourcesDir) + " does not have ordered ids"); } - } + // Validate that the name is in the correct format: + validateNameFormat(definition.name()); + + // Validate that there are no duplicate ids across any files: + for (var id : definition.ids()) { + if (seenIds.contains(id)) { + throw new IllegalStateException("Transport version definition file " + + definition.path(resourcesDir) + " contains an id also present in another file"); + } + seenIds.add(id); + } + }); } } From 6db990cf4e86f6e5ac250ead4258f086ff7b9951 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 31 Jul 2025 21:48:40 +0000 Subject: [PATCH 28/28] [CI] Auto commit changes from spotless --- ...GenerateTransportVersionDefinitionTask.java | 4 ++-- .../transport/TransportVersionUtils.java | 8 ++------ ...alidateTransportVersionDefinitionsTask.java | 18 ++++++++++++------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java index 956fad80d1b96..777855de93d22 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDefinitionTask.java @@ -90,10 +90,10 @@ public void generateTransportVersionData() throws IOException { TransportVersionLatest latest = TransportVersionUtils.readLatestFile(resourcesDir, minorVersion); TransportVersionLatest newLatest = null; - if (name.equals(latest.name())) { if (targetMinorVersions.contains(minorVersion) == false) { - // Regenerate to make this operation idempotent. Need to undo prior updates to the latest files if the list of minor versions has changed. + // Regenerate to make this operation idempotent. Need to undo prior updates to the latest files if the list of minor + // versions has changed. } } else { if (targetMinorVersions.contains(minorVersion)) { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java index 6e7a9da8673e7..20fa7737ce66d 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; + import org.elasticsearch.gradle.Version; import org.gradle.api.GradleException; import org.gradle.api.Project; @@ -69,11 +70,7 @@ static TransportVersionLatest readLatestFile(Path resourcesDir, MajorMinor versi Path filePath = resourcesDir.resolve(LATEST_DIR).resolve(version.toString() + CSV_SUFFIX); String[] parts = Files.readString(filePath, StandardCharsets.UTF_8).split(","); assert parts.length == 2; - return new TransportVersionLatest( - version, - parts[0], - Integer.parseInt(parts[1]) - ); + return new TransportVersionLatest(version, parts[0], Integer.parseInt(parts[1])); } static TransportVersionDefinition readDefinitionFile(Path resourcesDir, String name) { @@ -99,7 +96,6 @@ static TransportVersionDefinition readDefinitionFile(Path resourcesDir, String n } } - static Stream readAllDefinitionFiles(Path resourcesDir) throws IOException { var definitionsStream = Files.list(resourcesDir.resolve(DEFINED_DIR)); return definitionsStream.map(path -> { diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java index b15f692106816..86773ed94e7f4 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDefinitionsTask.java @@ -10,6 +10,7 @@ package org.elasticsearch.gradle.internal.transport; import com.google.common.collect.Comparators; + import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionReference; import org.gradle.api.DefaultTask; import org.gradle.api.file.ConfigurableFileCollection; @@ -67,14 +68,16 @@ public void validateTransportVersions() throws IOException { allDefinitions.forEach(definition -> { // Validate that all definitions are referenced in the codebase: if (allTvNames.contains(definition.name()) == false) { - throw new IllegalStateException("Transport version definition file " - + definition.path(resourcesDir) + " is not referenced in the codebase."); + throw new IllegalStateException( + "Transport version definition file " + definition.path(resourcesDir) + " is not referenced in the codebase." + ); } // Validate that all Ids are in decending order: if (Comparators.isInOrder(definition.ids(), Comparator.reverseOrder()) == false) { - throw new IllegalStateException("Transport version definition file " - + definition.path(resourcesDir) + " does not have ordered ids"); + throw new IllegalStateException( + "Transport version definition file " + definition.path(resourcesDir) + " does not have ordered ids" + ); } // Validate that the name is in the correct format: @@ -83,8 +86,11 @@ public void validateTransportVersions() throws IOException { // Validate that there are no duplicate ids across any files: for (var id : definition.ids()) { if (seenIds.contains(id)) { - throw new IllegalStateException("Transport version definition file " - + definition.path(resourcesDir) + " contains an id also present in another file"); + throw new IllegalStateException( + "Transport version definition file " + + definition.path(resourcesDir) + + " contains an id also present in another file" + ); } seenIds.add(id); }